Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions .github/workflows/checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
- lib/fixtures
- lib/flattening
- lib/identifier
- tests-bdd
- tests
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
Expand All @@ -54,7 +54,7 @@ jobs:
protocol/go/go.sum
sdk/go.sum
service/go.sum
tests-bdd/go.sum
tests/go.sum
- if: env.IS_RELEASE_BRANCH == 'true'
name: prevent depending on unreleased upstream changes
run: ./.github/scripts/work-init.sh
Expand Down Expand Up @@ -85,8 +85,9 @@ jobs:
run: .github/scripts/init-temp-keys.sh
- run: go test ./... -short
working-directory: ${{ matrix.directory }}
- if: matrix.directory == 'service'
- if: matrix.directory == 'tests'
run: go test ./service/integration -race -failfast
working-directory: tests
- name: verify sqlc generate
if: matrix.directory == 'service'
run: |
Expand Down Expand Up @@ -420,7 +421,7 @@ jobs:
# use commit instead of ref so we can "go get" specific sdk version
platform-ref: ${{ github.event.pull_request.head.sha || github.sha }} lts

tests-bdd:
tests:
name: Cucumber BDD Tests
runs-on: ubuntu-22.04
strategy:
Expand All @@ -445,15 +446,15 @@ jobs:
persist-credentials: false
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version-file: ./tests-bdd/go.mod
go-version-file: ./tests/go.mod
cache: false

- name: Build local platform-cukes image for testing
run: docker build -t platform-cukes .

- name: Run BDD Tests
run: |
CUKES_LOG_HANDLER=console go test ./tests-bdd -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features
CUKES_LOG_HANDLER=console go test ./tests -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features

- name: Check for undefined steps
run: |
Expand Down Expand Up @@ -561,7 +562,7 @@ jobs:
- benchmark
- license
- platform-xtest
- tests-bdd
- tests
- otdfctl-test
runs-on: ubuntu-22.04
if: ${{ !cancelled() }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/pr-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ jobs:
ci
deps
docs
tests
sdk
policy
ers
Expand Down
3 changes: 3 additions & 0 deletions .golangci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -207,3 +207,6 @@ formatters:
- third_party$
- builtin$
- examples$
run:
skip-dirs:
- ^integration$
6 changes: 3 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ This repo is a Go workspace (`go.work`) containing multiple Go modules:
- `sdk/`: Go SDK and generated clients.
- `lib/*/`: shared libraries (e.g., `lib/ocrypto`, `lib/identifier`).
- `protocol/` and `service/`: protobuf sources; generated Go lives under `protocol/go/` and docs under `docs/grpc/` + `docs/openapi/`.
- `tests-bdd/`: BDD/integration-style tests (Godog) and feature files (`tests-bdd/features/`).
- `tests/`: BDD/integration-style tests (Godog) and feature files (`tests/features/`).
- `docs/`, `examples/`, `adr/`: documentation, example code, and architecture decisions.

## Build, Test, and Development Commands
Expand All @@ -18,7 +18,7 @@ Prefer `make` targets at repo root:
- `make toolcheck`: verifies required tooling (Buf, golangci-lint, generators).
- `make build`: regenerates protos/codegen and builds `opentdf` + `sdk` + `examples`.
- `make lint`: runs `buf lint`, `golangci-lint`, and `govulncheck` across modules.
- `make test`: runs `go test ./... -race` across core modules (does **not** include `tests-bdd/`).
- `make test`: runs `go test ./... -race` across core modules (does **not** include `tests/`).
- `docker compose up`: brings up local infra (Postgres + Keycloak). See `docs/Contributing.md`.

## Coding Style & Naming Conventions
Expand Down Expand Up @@ -52,7 +52,7 @@ Prefer `make` targets at repo root:
### Test Types

- **Unit tests**: `*_test.go` next to code; run `make test`.
- **BDD tests**: run `cd tests-bdd && go test ./...` (requires Docker; feature files are `tests-bdd/features/*.feature`).
- **BDD tests**: run `cd tests && go test ./...` (requires Docker; feature files are `tests/features/*.feature`).
- **Integration tests** may require the compose stack; follow module README(s) under `service/`.
- **README tests**: verify code examples in documentation compile and work correctly.

Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ COPY sdk/ sdk/
COPY lib/ lib/
COPY service/ service/
COPY examples/ examples/
COPY tests-bdd/ tests-bdd/
COPY tests/ tests/
COPY go.work ./
RUN cd service \
&& go mod download \
Expand Down
7 changes: 5 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# make
# To run all lint checks: `LINT_OPTIONS= make lint`

.PHONY: all build clean connect-wrapper-generate docker-build fix fmt go-lint license lint proto-generate proto-lint sdk/sdk test tidy toolcheck
.PHONY: all build clean connect-wrapper-generate docker-build fix fmt go-lint license lint proto-generate proto-lint sdk/sdk test test-integration tidy toolcheck

MODS=protocol/go lib/ocrypto lib/fixtures lib/flattening lib/identifier sdk service examples
MODS=protocol/go lib/ocrypto lib/fixtures lib/flattening lib/identifier sdk service examples tests
HAND_MODS=lib/ocrypto lib/fixtures lib/flattening lib/identifier sdk service examples
REQUIRED_BUF_VERSION=1.56.0

Expand Down Expand Up @@ -103,6 +103,9 @@ policy-erd-gen:
test:
for m in $(HAND_MODS); do (cd $$m && go test ./... -race) || exit 1; done

test-integration:
cd tests && go test ./...

fuzz:
cd sdk && go test ./... -fuzztime=2m

Expand Down
9 changes: 7 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
- [Multi-Strategy Entity Resolution Service](./ERS_TESTING.md)
- [Development](#for-contributors)
- [Policy Config Schema](./service/policy/db/schema_erd.md)
- [Policy Config Testing Diagram](./service/integration/testing_diagram.png)
- [Policy Config Testing Diagram](./tests/service/integration/testing_diagram.png)

### Prerequisites for Project Consumers & Contributors

Expand Down Expand Up @@ -95,7 +95,7 @@ Test the multi-strategy ERS functionality:

```sh
# Run integration tests (Docker services provide the backends automatically)
go test ./service/entityresolution/integration -run TestMultiStrategy -v
go test ./tests/service/entityresolution/integration -run TestMultiStrategy -v
```

#### Configuration Options
Expand All @@ -109,6 +109,11 @@ The multi-strategy ERS (preview) provides enterprise-grade identity resolution w

This section is focused on the development of the OpenTDF platform.

### Testing

- Unit/fast tests: `make test`
- Integration + BDD tests (containers): `make test-integration`

### Libraries

Libraries `./lib` are shared libraries that are used across the OpenTDF platform. These libraries are used to provide
Expand Down
52 changes: 52 additions & 0 deletions adr/ai/2026-02-09-centralized-tests-module.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Centralize Containerized and BDD Tests Under tests/

## Status
Accepted

## Context
We want to remove the `testcontainers` dependency from the `sdk` and `service` modules while keeping integration and BDD coverage. The existing layout splits BDD tests under `tests-bdd/` and containerized integration tests inside `sdk/` and `service/`, which ties those modules to `testcontainers` and increases dependency weight.

## Decision
- Create a single root `tests/` Go module that contains all BDD and containerized integration tests.
- Move SDK OAuth, Service integration, and ERS integration tests into `tests/`.
- Keep `lib/fixtures` versioned (for now) and move service fixture helpers into it for tests and quickstart workflows.
- Use `go.work` for local module resolution across `sdk`, `service`, `protocol/go`, and `lib/fixtures`.

## Consequences
- `sdk` and `service` drop `testcontainers` from their `go.mod` files.
- Integration and BDD tests are run via `cd tests && go test ./...`.
- `lib/fixtures` now depends on `service` and `protocol/go`, which increases coupling and should be monitored for release friction.
- CI and documentation must point to the new `tests/` location.

## Migration Checklist (Completed)

### 1) BDD migration (tests-bdd -> tests)
- [x] Move `tests-bdd/` to `tests/`
- [x] Update module path, imports, and docs
- [x] Update Dockerfile and root references

### 2) SDK OAuth integration tests
- [x] Move OAuth tests under `tests/sdk/auth/oauth`
- [x] Move testdata and update package imports
- [x] Ensure Keycloak container setup works

### 3) Service integration tests
- [x] Move `service/integration` tests under `tests/service/integration`
- [x] Replace fixture loading with `lib/fixtures`
- [x] Use shared container helpers

### 4) ERS integration tests
- [x] Move ERS integration tests under `tests/service/entityresolution/integration`
- [x] Update internal package imports
- [x] Update docs and docker-compose data mounts

### 5) Fixtures consolidation
- [x] Move `service/internal/fixtures` into `lib/fixtures`
- [x] Embed `policy_fixtures.yaml` and update loaders
- [x] Update `service/cmd/provisionFixtures.go`

### 6) Cleanup and wiring
- [x] Remove `testcontainers` from `sdk` and `service` go.mod
- [x] Update `go.work` to include `tests/`
- [x] Add `make test-integration` and document usage
- [x] Run `go mod tidy` and formatting
Original file line number Diff line number Diff line change
Expand Up @@ -591,7 +591,7 @@ mapping_strategies:

**Integration Tests**:
```
service/entityresolution/integration/
tests/service/entityresolution/integration/
├── multi_strategy_test.go # Multi-strategy core integration tests
├── multi_strategy_sql_test.go # SQL provider integration tests
├── multi_strategy_ldap_test.go # LDAP provider integration tests
Expand Down
8 changes: 4 additions & 4 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ services:
# docker-compose --profile ers-admin up -d # Include LDAP admin UI
# ERS_TEST_POSTGRES_URL="postgres://ers_test_user:ers_test_pass@localhost:5433/ers_test?sslmode=disable" \
# ERS_TEST_LDAP_URL="ldap://localhost:1389" \
# go test ./service/entityresolution/integration -run TestMultiStrategy -v
# go test ./tests/service/entityresolution/integration -run TestMultiStrategy -v
#
# Services:
# - ers-postgres: PostgreSQL 16 on port 5433 with test schema and data
Expand All @@ -145,7 +145,7 @@ services:
POSTGRES_INITDB_ARGS: "--encoding=UTF-8"
volumes:
- ers_postgres_data:/var/lib/postgresql/data
- ./service/entityresolution/integration/sql_test_data:/docker-entrypoint-initdb.d
- ./tests/service/entityresolution/integration/sql_test_data:/docker-entrypoint-initdb.d
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ers_test_user -d ers_test"]
interval: 5s
Expand Down Expand Up @@ -181,7 +181,7 @@ services:
volumes:
- ers_ldap_data:/var/lib/ldap
- ers_ldap_config:/etc/ldap/slapd.d
- ./service/entityresolution/integration/ldap_test_data:/container/service/slapd/assets/config/bootstrap/ldif/custom
- ./tests/service/entityresolution/integration/ldap_test_data:/container/service/slapd/assets/config/bootstrap/ldif/custom
healthcheck:
test: ["CMD-SHELL", "ldapsearch -x -H ldap://localhost:389 -b dc=opentdf,dc=test -D cn=admin,dc=opentdf,dc=test -w admin_password '(objectclass=*)' dn"]
interval: 10s
Expand Down Expand Up @@ -210,4 +210,4 @@ volumes:
ers_ldap_data:
name: ers_test_ldap_data
ers_ldap_config:
name: ers_test_ldap_config
name: ers_test_ldap_config
2 changes: 1 addition & 1 deletion go.work
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ use (
./protocol/go
./sdk
./service
./tests-bdd
./tests
)
4 changes: 2 additions & 2 deletions lib/fixtures/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Internal Module

These modules are used by other http://github.com/opentdf/ projects, and
are not intended for use elsewhere.
This module provides fixtures used by tests and quickstart workflows across OpenTDF repositories.
It is versioned and can be consumed like other internal libs.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package fixtures

import (
"context"
_ "embed"
"encoding/base64"
"encoding/json"
"log/slog"
Expand All @@ -15,6 +16,9 @@ import (
var (
fixtureFilename = "policy_fixtures.yaml"
fixtureData FixtureData

//go:embed testdata/policy_fixtures.yaml
defaultFixtureData []byte
)

type FixtureMetadata struct {
Expand Down Expand Up @@ -249,14 +253,26 @@ func LoadFixtureData(file string) {
panic(err)
}

if err := yaml.Unmarshal(c, &fixtureData); err != nil {
loadFixtureData(c, file)
}

func LoadFixtureDataFromDefault() {
if len(defaultFixtureData) == 0 {
panic("default fixture data is empty")
}
loadFixtureData(defaultFixtureData, "embedded:"+fixtureFilename)
}

func loadFixtureData(data []byte, source string) {
if err := yaml.Unmarshal(data, &fixtureData); err != nil {
slog.Error("could not unmarshal",
slog.String("fixture_file_name", fixtureFilename),
slog.String("fixture_source", source),
slog.Any("error", err),
)
panic(err)
}
slog.Info("fully loaded fixtures", slog.Any("fixture_data", fixtureData))
slog.Info("fully loaded fixtures", slog.String("fixture_source", source))
}

type Fixtures struct {
Expand Down
Loading
Loading