From 018249a7a10374282c21a2e418fe8f6250bcd987 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 12 Jun 2026 08:44:41 +1000 Subject: [PATCH 1/2] feature: Added V2 API endpoint for Connections. Added connections endpoint to /api/v2/connections supporting list, get, create, update, all using Bearer token auth. --- .changeset/external-api-connections.md | 5 + packages/api/openapi.json | 466 ++++++++++++++ packages/api/src/controllers/connection.ts | 4 +- packages/api/src/models/connection.ts | 2 + .../routers/api/__tests__/connections.test.ts | 26 + packages/api/src/routers/api/connections.ts | 6 +- .../__tests__/connections.test.ts | 436 +++++++++++++ .../routers/external-api/v2/connections.ts | 577 ++++++++++++++++++ .../api/src/routers/external-api/v2/index.ts | 8 + packages/api/src/utils/swagger.ts | 4 + 10 files changed, 1531 insertions(+), 3 deletions(-) create mode 100644 .changeset/external-api-connections.md create mode 100644 packages/api/src/routers/external-api/__tests__/connections.test.ts create mode 100644 packages/api/src/routers/external-api/v2/connections.ts diff --git a/.changeset/external-api-connections.md b/.changeset/external-api-connections.md new file mode 100644 index 0000000000..b430d6e8ca --- /dev/null +++ b/.changeset/external-api-connections.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/api": minor +--- + +Add connection management endpoints to the external API (`/api/v2/connections`), supporting list, get, create, update, and delete with Bearer token authentication. Passwords are write-only and never returned by the API. \ No newline at end of file diff --git a/packages/api/openapi.json b/packages/api/openapi.json index d043230d7e..76cf3c7d7a 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -24,6 +24,10 @@ "name": "Charts", "description": "Endpoints for querying chart data" }, + { + "name": "Connections", + "description": "Endpoints for managing ClickHouse connections" + }, { "name": "Sources", "description": "Endpoints for managing data sources" @@ -581,6 +585,167 @@ } } }, + "Connection": { + "type": "object", + "required": [ + "id", + "name", + "host", + "username" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique connection ID.", + "example": "507f1f77bcf86cd799439012" + }, + "name": { + "type": "string", + "description": "Display name for the connection.", + "example": "Production ClickHouse" + }, + "host": { + "type": "string", + "description": "ClickHouse HTTP endpoint URL.", + "example": "https://clickhouse.example.com:8443" + }, + "username": { + "type": "string", + "description": "ClickHouse username.", + "example": "default" + }, + "hyperdxSettingPrefix": { + "type": "string", + "description": "Optional prefix for HyperDX-specific ClickHouse settings. Must only contain alphanumeric characters and underscores.", + "nullable": true, + "example": "hyperdx_" + }, + "prometheusEndpoint": { + "type": "string", + "description": "Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint.", + "nullable": true, + "example": "http://prometheus:9090" + }, + "createdAt": { + "type": "string", + "format": "date-time", + "description": "Creation timestamp", + "example": "2025-01-01T00:00:00.000Z" + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "description": "Last update timestamp", + "example": "2025-06-15T10:30:00.000Z" + } + } + }, + "CreateConnectionRequest": { + "type": "object", + "required": [ + "name", + "host", + "username" + ], + "properties": { + "name": { + "type": "string", + "description": "Display name for the connection.", + "example": "Production ClickHouse" + }, + "host": { + "type": "string", + "description": "ClickHouse HTTP endpoint URL.", + "example": "https://clickhouse.example.com:8443" + }, + "username": { + "type": "string", + "description": "ClickHouse username.", + "example": "default" + }, + "password": { + "type": "string", + "writeOnly": true, + "description": "ClickHouse password. Never returned by the API.", + "example": "my-secret-password" + }, + "hyperdxSettingPrefix": { + "type": "string", + "description": "Optional prefix for HyperDX-specific ClickHouse settings. Must only contain alphanumeric characters and underscores.", + "nullable": true, + "example": "hyperdx_" + }, + "prometheusEndpoint": { + "type": "string", + "description": "Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint.", + "example": "http://prometheus:9090" + } + } + }, + "UpdateConnectionRequest": { + "type": "object", + "required": [ + "name", + "host", + "username" + ], + "properties": { + "name": { + "type": "string", + "description": "Display name for the connection.", + "example": "Production ClickHouse" + }, + "host": { + "type": "string", + "description": "ClickHouse HTTP endpoint URL.", + "example": "https://clickhouse.example.com:8443" + }, + "username": { + "type": "string", + "description": "ClickHouse username.", + "example": "default" + }, + "password": { + "type": "string", + "writeOnly": true, + "description": "ClickHouse password. If omitted or empty, the existing password is kept.", + "example": "my-new-secret-password" + }, + "hyperdxSettingPrefix": { + "type": "string", + "description": "Optional prefix for HyperDX-specific ClickHouse settings. Set to null or an empty string to clear the existing value. If omitted, the existing value is kept.", + "nullable": true, + "example": "hyperdx_" + }, + "prometheusEndpoint": { + "type": "string", + "description": "Optional Prometheus-compatible API endpoint. Set to null to clear the existing value. If omitted, the existing value is kept.", + "nullable": true, + "example": "http://prometheus:9090" + } + } + }, + "ConnectionResponseEnvelope": { + "type": "object", + "properties": { + "data": { + "$ref": "#/components/schemas/Connection", + "description": "The connection object." + } + } + }, + "ConnectionsListResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "description": "List of connection objects.", + "items": { + "$ref": "#/components/schemas/Connection" + } + } + } + }, "NumberFormatOutput": { "type": "string", "enum": [ @@ -4487,6 +4652,307 @@ } } }, + "/api/v2/connections": { + "get": { + "summary": "List Connections", + "description": "Retrieves a list of all ClickHouse connections for the authenticated team. Passwords are never returned.", + "operationId": "listConnections", + "tags": [ + "Connections" + ], + "responses": { + "200": { + "description": "Successfully retrieved connections", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConnectionsListResponse" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + }, + "post": { + "summary": "Create Connection", + "description": "Creates a new ClickHouse connection", + "operationId": "createConnection", + "tags": [ + "Connections" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateConnectionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successfully created connection", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConnectionResponseEnvelope" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Body validation failed: name: Required" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + } + } + } + }, + "/api/v2/connections/{id}": { + "get": { + "summary": "Get Connection", + "description": "Retrieves a specific ClickHouse connection by ID. Passwords are never returned.", + "operationId": "getConnection", + "tags": [ + "Connections" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connection ID", + "example": "507f1f77bcf86cd799439012" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved connection", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConnectionResponseEnvelope" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + }, + "404": { + "description": "Connection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Connection not found" + } + } + } + } + } + }, + "put": { + "summary": "Update Connection", + "description": "Updates an existing ClickHouse connection.\n\nField semantics: if `password` is omitted or empty the existing\npassword is kept. `hyperdxSettingPrefix` is cleared when set to null\nor an empty string, and `prometheusEndpoint` is cleared when set to\nnull; both are kept unchanged when omitted.\n", + "operationId": "updateConnection", + "tags": [ + "Connections" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connection ID", + "example": "507f1f77bcf86cd799439012" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateConnectionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successfully updated connection", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ConnectionResponseEnvelope" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Body validation failed: host: Required" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + }, + "404": { + "description": "Connection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Connection not found" + } + } + } + } + } + }, + "delete": { + "summary": "Delete Connection", + "description": "Deletes a ClickHouse connection", + "operationId": "deleteConnection", + "tags": [ + "Connections" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connection ID", + "example": "507f1f77bcf86cd799439012" + } + ], + "responses": { + "200": { + "description": "Successfully deleted connection", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmptyResponse" + }, + "example": {} + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Unauthorized access. API key is missing or invalid." + } + } + } + }, + "404": { + "description": "Connection not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + }, + "example": { + "message": "Connection not found" + } + } + } + } + } + } + }, "/api/v2/dashboards": { "get": { "summary": "List Dashboards", diff --git a/packages/api/src/controllers/connection.ts b/packages/api/src/controllers/connection.ts index 1bff0d6e79..3116d7b6ea 100644 --- a/packages/api/src/controllers/connection.ts +++ b/packages/api/src/controllers/connection.ts @@ -1,8 +1,10 @@ import Connection, { IConnection } from '@/models/connection'; +// Returns all connections across all teams. Only intended for instance-level +// operations (e.g. startup auto-provisioning); user-facing routes must use +// the team-scoped variants below. export function getConnections() { // Never return password back to the user - // Return all connections in current tenant return Connection.find({}); } diff --git a/packages/api/src/models/connection.ts b/packages/api/src/models/connection.ts index 324390df6c..9e46f6f935 100644 --- a/packages/api/src/models/connection.ts +++ b/packages/api/src/models/connection.ts @@ -18,6 +18,8 @@ export interface IConnection { prometheusEndpoint?: string; } +export type ConnectionDocument = mongoose.HydratedDocument; + export default mongoose.model( 'Connection', new Schema( diff --git a/packages/api/src/routers/api/__tests__/connections.test.ts b/packages/api/src/routers/api/__tests__/connections.test.ts index e168de4328..405ce5b240 100644 --- a/packages/api/src/routers/api/__tests__/connections.test.ts +++ b/packages/api/src/routers/api/__tests__/connections.test.ts @@ -1,3 +1,5 @@ +import { ObjectId } from 'mongodb'; + import * as config from '@/config'; import { getLoggedInAgent, getServer } from '@/fixtures'; import Connection from '@/models/connection'; @@ -17,6 +19,30 @@ describe('connections router', () => { await server.stop(); }); + it('only returns connections belonging to the current team through GET /connections', async () => { + const { agent, team } = await getLoggedInAgent(server); + + await Connection.create({ + team: team._id, + name: 'My Team Connection', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + }); + await Connection.create({ + team: new ObjectId(), + name: 'Other Team Connection', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + }); + + const res = await agent.get('/connections').expect(200); + + expect(res.body).toHaveLength(1); + expect(res.body[0].name).toBe('My Team Connection'); + }); + it('persists prometheusEndpoint through POST /connections', async () => { const { agent } = await getLoggedInAgent(server); diff --git a/packages/api/src/routers/api/connections.ts b/packages/api/src/routers/api/connections.ts index 97a62a5f3c..2fac55e9c4 100644 --- a/packages/api/src/routers/api/connections.ts +++ b/packages/api/src/routers/api/connections.ts @@ -6,7 +6,7 @@ import { createConnection, deleteConnection, getConnectionById, - getConnections, + getConnectionsByTeam, updateConnection, } from '@/controllers/connection'; import { getNonNullUserWithTeam } from '@/middleware/auth'; @@ -15,7 +15,9 @@ const router = express.Router(); router.get('/', async (req, res, next) => { try { - const connections = await getConnections(); + const { teamId } = getNonNullUserWithTeam(req); + + const connections = await getConnectionsByTeam(teamId.toString()); res.json(connections.map(c => c.toJSON({ virtuals: true }))); } catch (e) { diff --git a/packages/api/src/routers/external-api/__tests__/connections.test.ts b/packages/api/src/routers/external-api/__tests__/connections.test.ts new file mode 100644 index 0000000000..38037b559e --- /dev/null +++ b/packages/api/src/routers/external-api/__tests__/connections.test.ts @@ -0,0 +1,436 @@ +import { ObjectId } from 'mongodb'; +import request, { SuperAgentTest } from 'supertest'; + +import { getLoggedInAgent, getServer } from '../../../fixtures'; +import Connection from '../../../models/connection'; +import { ITeam } from '../../../models/team'; +import { IUser } from '../../../models/user'; + +const CONNECTIONS_BASE_URL = '/api/v2/connections'; + +const MOCK_CONNECTION = { + name: 'Test Connection', + host: 'https://clickhouse.example.com:8443', + username: 'default', + password: 'test-password', +}; + +describe('External API v2 Connections', () => { + const server = getServer(); + let agent: SuperAgentTest; + let team: ITeam; + let user: IUser; + + beforeAll(async () => { + await server.start(); + }); + + beforeEach(async () => { + const result = await getLoggedInAgent(server); + agent = result.agent; + team = result.team; + user = result.user; + }); + + afterEach(async () => { + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + const authRequest = ( + method: 'get' | 'post' | 'put' | 'delete', + url: string, + ) => { + return agent[method](url).set('Authorization', `Bearer ${user?.accessKey}`); + }; + + describe('GET /api/v2/connections', () => { + it('should return an empty list when no connections exist', async () => { + const response = await authRequest('get', CONNECTIONS_BASE_URL).expect( + 200, + ); + + expect(response.headers['content-type']).toMatch(/application\/json/); + expect(response.body).toEqual({ data: [] }); + }); + + it('should list connections without exposing passwords', async () => { + await Connection.create({ ...MOCK_CONNECTION, team: team._id }); + + const response = await authRequest('get', CONNECTIONS_BASE_URL).expect( + 200, + ); + + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0]).toMatchObject({ + id: expect.any(String), + name: MOCK_CONNECTION.name, + host: MOCK_CONNECTION.host, + username: MOCK_CONNECTION.username, + createdAt: expect.any(String), + updatedAt: expect.any(String), + }); + expect(response.body.data[0]).not.toHaveProperty('password'); + expect(response.body.data[0]).not.toHaveProperty('team'); + }); + + it('should not return connections belonging to another team', async () => { + await Connection.create({ ...MOCK_CONNECTION, team: team._id }); + + const otherTeamId = new ObjectId(); + await Connection.create({ + ...MOCK_CONNECTION, + name: 'Other Team Connection', + team: otherTeamId, + }); + + const response = await authRequest('get', CONNECTIONS_BASE_URL).expect( + 200, + ); + + expect(response.body.data).toHaveLength(1); + expect(response.body.data[0].name).toBe(MOCK_CONNECTION.name); + }); + + it('should require authentication', async () => { + await request(server.getHttpServer()) + .get(CONNECTIONS_BASE_URL) + .expect(401); + }); + }); + + describe('GET /api/v2/connections/:id', () => { + it('should return a connection by id without the password', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + const response = await authRequest( + 'get', + `${CONNECTIONS_BASE_URL}/${connection._id}`, + ).expect(200); + + expect(response.body.data).toMatchObject({ + id: connection._id.toString(), + name: MOCK_CONNECTION.name, + host: MOCK_CONNECTION.host, + username: MOCK_CONNECTION.username, + }); + expect(response.body.data).not.toHaveProperty('password'); + }); + + it('should return 404 for a non-existent connection', async () => { + await authRequest( + 'get', + `${CONNECTIONS_BASE_URL}/${new ObjectId()}`, + ).expect(404); + }); + + it('should return 404 for a connection belonging to another team', async () => { + const otherTeamConnection = await Connection.create({ + ...MOCK_CONNECTION, + team: new ObjectId(), + }); + + await authRequest( + 'get', + `${CONNECTIONS_BASE_URL}/${otherTeamConnection._id}`, + ).expect(404); + }); + + it('should return 400 for an invalid connection id', async () => { + await authRequest('get', `${CONNECTIONS_BASE_URL}/not-an-id`).expect(400); + }); + + it('should require authentication', async () => { + await request(server.getHttpServer()) + .get(`${CONNECTIONS_BASE_URL}/${new ObjectId()}`) + .expect(401); + }); + }); + + describe('POST /api/v2/connections', () => { + it('should create a connection and return it without the password', async () => { + const response = await authRequest('post', CONNECTIONS_BASE_URL) + .send(MOCK_CONNECTION) + .expect(200); + + expect(response.body.data).toMatchObject({ + id: expect.any(String), + name: MOCK_CONNECTION.name, + host: MOCK_CONNECTION.host, + username: MOCK_CONNECTION.username, + }); + expect(response.body.data).not.toHaveProperty('password'); + + const stored = await Connection.findById(response.body.data.id).select( + '+password', + ); + expect(stored).not.toBeNull(); + expect(stored?.password).toBe(MOCK_CONNECTION.password); + expect(stored?.team.toString()).toBe(team._id.toString()); + }); + + it('should create a connection without a password', async () => { + const { password, ...connectionWithoutPassword } = MOCK_CONNECTION; + + const response = await authRequest('post', CONNECTIONS_BASE_URL) + .send(connectionWithoutPassword) + .expect(200); + + const stored = await Connection.findById(response.body.data.id).select( + '+password', + ); + expect(stored?.password).toBe(''); + }); + + it('should create a connection with optional fields', async () => { + const response = await authRequest('post', CONNECTIONS_BASE_URL) + .send({ + ...MOCK_CONNECTION, + hyperdxSettingPrefix: 'hyperdx_', + prometheusEndpoint: 'http://prometheus:9090', + }) + .expect(200); + + expect(response.body.data).toMatchObject({ + hyperdxSettingPrefix: 'hyperdx_', + prometheusEndpoint: 'http://prometheus:9090', + }); + }); + + it('should reject a request with missing required fields', async () => { + await authRequest('post', CONNECTIONS_BASE_URL) + .send({ name: 'Missing host and username' }) + .expect(400); + }); + + it('should reject an invalid prometheusEndpoint', async () => { + await authRequest('post', CONNECTIONS_BASE_URL) + .send({ ...MOCK_CONNECTION, prometheusEndpoint: 'not-a-url' }) + .expect(400); + }); + + it('should require authentication', async () => { + await request(server.getHttpServer()) + .post(CONNECTIONS_BASE_URL) + .send(MOCK_CONNECTION) + .expect(401); + }); + }); + + describe('PUT /api/v2/connections/:id', () => { + it('should update a connection', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + const response = await authRequest( + 'put', + `${CONNECTIONS_BASE_URL}/${connection._id}`, + ) + .send({ + name: 'Updated Connection', + host: 'https://clickhouse-updated.example.com:8443', + username: 'updated-user', + }) + .expect(200); + + expect(response.body.data).toMatchObject({ + id: connection._id.toString(), + name: 'Updated Connection', + host: 'https://clickhouse-updated.example.com:8443', + username: 'updated-user', + }); + expect(response.body.data).not.toHaveProperty('password'); + }); + + it('should keep the existing password when omitted', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ + name: 'Updated Connection', + host: MOCK_CONNECTION.host, + username: MOCK_CONNECTION.username, + }) + .expect(200); + + const stored = await Connection.findById(connection._id).select( + '+password', + ); + expect(stored?.password).toBe(MOCK_CONNECTION.password); + }); + + it('should update the password when provided', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ ...MOCK_CONNECTION, password: 'new-password' }) + .expect(200); + + const stored = await Connection.findById(connection._id).select( + '+password', + ); + expect(stored?.password).toBe('new-password'); + }); + + it('should clear hyperdxSettingPrefix when set to an empty string', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + hyperdxSettingPrefix: 'hyperdx_', + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ ...MOCK_CONNECTION, hyperdxSettingPrefix: '' }) + .expect(200); + + const stored = await Connection.findById(connection._id); + expect(stored?.hyperdxSettingPrefix).toBeUndefined(); + }); + + it('should clear hyperdxSettingPrefix when set to null', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + hyperdxSettingPrefix: 'hyperdx_', + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ ...MOCK_CONNECTION, hyperdxSettingPrefix: null }) + .expect(200); + + const stored = await Connection.findById(connection._id); + expect(stored?.hyperdxSettingPrefix).toBeUndefined(); + }); + + it('should clear prometheusEndpoint when set to null', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + prometheusEndpoint: 'http://prometheus:9090', + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ ...MOCK_CONNECTION, prometheusEndpoint: null }) + .expect(200); + + const stored = await Connection.findById(connection._id); + expect(stored?.prometheusEndpoint).toBeUndefined(); + }); + + it('should keep optional fields unchanged when omitted', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + hyperdxSettingPrefix: 'hyperdx_', + prometheusEndpoint: 'http://prometheus:9090', + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ + name: 'Updated Connection', + host: MOCK_CONNECTION.host, + username: MOCK_CONNECTION.username, + }) + .expect(200); + + const stored = await Connection.findById(connection._id); + expect(stored?.hyperdxSettingPrefix).toBe('hyperdx_'); + expect(stored?.prometheusEndpoint).toBe('http://prometheus:9090'); + }); + + it('should return 404 for a non-existent connection', async () => { + await authRequest('put', `${CONNECTIONS_BASE_URL}/${new ObjectId()}`) + .send(MOCK_CONNECTION) + .expect(404); + }); + + it('should return 404 for a connection belonging to another team', async () => { + const otherTeamConnection = await Connection.create({ + ...MOCK_CONNECTION, + team: new ObjectId(), + }); + + await authRequest( + 'put', + `${CONNECTIONS_BASE_URL}/${otherTeamConnection._id}`, + ) + .send(MOCK_CONNECTION) + .expect(404); + }); + + it('should reject a request with missing required fields', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + await authRequest('put', `${CONNECTIONS_BASE_URL}/${connection._id}`) + .send({ name: 'Missing host and username' }) + .expect(400); + }); + + it('should require authentication', async () => { + await request(server.getHttpServer()) + .put(`${CONNECTIONS_BASE_URL}/${new ObjectId()}`) + .send(MOCK_CONNECTION) + .expect(401); + }); + }); + + describe('DELETE /api/v2/connections/:id', () => { + it('should delete a connection', async () => { + const connection = await Connection.create({ + ...MOCK_CONNECTION, + team: team._id, + }); + + await authRequest( + 'delete', + `${CONNECTIONS_BASE_URL}/${connection._id}`, + ).expect(200); + + expect(await Connection.findById(connection._id)).toBeNull(); + }); + + it('should return 404 for a non-existent connection', async () => { + await authRequest( + 'delete', + `${CONNECTIONS_BASE_URL}/${new ObjectId()}`, + ).expect(404); + }); + + it('should not delete a connection belonging to another team', async () => { + const otherTeamConnection = await Connection.create({ + ...MOCK_CONNECTION, + team: new ObjectId(), + }); + + await authRequest( + 'delete', + `${CONNECTIONS_BASE_URL}/${otherTeamConnection._id}`, + ).expect(404); + + expect(await Connection.findById(otherTeamConnection._id)).not.toBeNull(); + }); + + it('should require authentication', async () => { + await request(server.getHttpServer()) + .delete(`${CONNECTIONS_BASE_URL}/${new ObjectId()}`) + .expect(401); + }); + }); +}); diff --git a/packages/api/src/routers/external-api/v2/connections.ts b/packages/api/src/routers/external-api/v2/connections.ts new file mode 100644 index 0000000000..4093e118b5 --- /dev/null +++ b/packages/api/src/routers/external-api/v2/connections.ts @@ -0,0 +1,577 @@ +import { ConnectionSchema } from '@hyperdx/common-utils/dist/types'; +import express from 'express'; +import { z } from 'zod'; + +import { + createConnection, + deleteConnection, + getConnectionById, + getConnectionsByTeam, + updateConnection, +} from '@/controllers/connection'; +import { ConnectionDocument } from '@/models/connection'; +import { processRequestWithEnhancedErrors as validateRequest } from '@/utils/enhancedErrors'; +import logger from '@/utils/logger'; +import { objectIdSchema } from '@/utils/zod'; + +// External representation of a connection. The password is intentionally +// excluded so that it is never returned by the API. +const externalConnectionSchema = ConnectionSchema.omit({ + password: true, +}).extend({ + createdAt: z.string().optional(), + updatedAt: z.string().optional(), +}); + +const createConnectionBodySchema = ConnectionSchema.omit({ id: true }); + +// On update, hyperdxSettingPrefix additionally accepts '' and +// prometheusEndpoint additionally accepts null, both meaning "clear the +// existing value". The base ConnectionSchema rejects both. +const updateConnectionBodySchema = ConnectionSchema.omit({ id: true }).extend({ + hyperdxSettingPrefix: z + .string() + .regex(/^[a-z0-9_]+$/i) + .or(z.literal('')) + .optional() + .nullable(), + prometheusEndpoint: z.string().url().optional().nullable(), +}); + +function formatExternalConnection(connection: ConnectionDocument) { + // Convert to JSON so that any ObjectIds and Dates are converted to strings + const json = JSON.stringify(connection.toJSON({ virtuals: true })); + + // Parse using the externalConnectionSchema to strip out any fields not + // defined in the schema (e.g. password, team, _id, __v) + const parseResult = externalConnectionSchema.safeParse(JSON.parse(json)); + if (parseResult.success) { + return parseResult.data; + } + + // If parsing fails, log the error and return undefined + logger.error( + { connectionId: connection._id, error: parseResult.error }, + 'Failed to parse connection using externalConnectionSchema:', + ); + + return undefined; +} + +/** + * @openapi + * components: + * schemas: + * Connection: + * type: object + * required: + * - id + * - name + * - host + * - username + * properties: + * id: + * type: string + * description: Unique connection ID. + * example: 507f1f77bcf86cd799439012 + * name: + * type: string + * description: Display name for the connection. + * example: Production ClickHouse + * host: + * type: string + * description: ClickHouse HTTP endpoint URL. + * example: https://clickhouse.example.com:8443 + * username: + * type: string + * description: ClickHouse username. + * example: default + * hyperdxSettingPrefix: + * type: string + * description: Optional prefix for HyperDX-specific ClickHouse settings. Must only contain alphanumeric characters and underscores. + * nullable: true + * example: hyperdx_ + * prometheusEndpoint: + * type: string + * description: Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint. + * nullable: true + * example: http://prometheus:9090 + * createdAt: + * type: string + * format: date-time + * description: Creation timestamp + * example: "2025-01-01T00:00:00.000Z" + * updatedAt: + * type: string + * format: date-time + * description: Last update timestamp + * example: "2025-06-15T10:30:00.000Z" + * CreateConnectionRequest: + * type: object + * required: + * - name + * - host + * - username + * properties: + * name: + * type: string + * description: Display name for the connection. + * example: Production ClickHouse + * host: + * type: string + * description: ClickHouse HTTP endpoint URL. + * example: https://clickhouse.example.com:8443 + * username: + * type: string + * description: ClickHouse username. + * example: default + * password: + * type: string + * writeOnly: true + * description: ClickHouse password. Never returned by the API. + * example: my-secret-password + * hyperdxSettingPrefix: + * type: string + * description: Optional prefix for HyperDX-specific ClickHouse settings. Must only contain alphanumeric characters and underscores. + * nullable: true + * example: hyperdx_ + * prometheusEndpoint: + * type: string + * description: Optional Prometheus-compatible API endpoint. When set, PromQL queries are proxied to this endpoint. + * example: http://prometheus:9090 + * UpdateConnectionRequest: + * type: object + * required: + * - name + * - host + * - username + * properties: + * name: + * type: string + * description: Display name for the connection. + * example: Production ClickHouse + * host: + * type: string + * description: ClickHouse HTTP endpoint URL. + * example: https://clickhouse.example.com:8443 + * username: + * type: string + * description: ClickHouse username. + * example: default + * password: + * type: string + * writeOnly: true + * description: ClickHouse password. If omitted or empty, the existing password is kept. + * example: my-new-secret-password + * hyperdxSettingPrefix: + * type: string + * description: Optional prefix for HyperDX-specific ClickHouse settings. Set to null or an empty string to clear the existing value. If omitted, the existing value is kept. + * nullable: true + * example: hyperdx_ + * prometheusEndpoint: + * type: string + * description: Optional Prometheus-compatible API endpoint. Set to null to clear the existing value. If omitted, the existing value is kept. + * nullable: true + * example: http://prometheus:9090 + * ConnectionResponseEnvelope: + * type: object + * properties: + * data: + * $ref: '#/components/schemas/Connection' + * description: The connection object. + * ConnectionsListResponse: + * type: object + * properties: + * data: + * type: array + * description: List of connection objects. + * items: + * $ref: '#/components/schemas/Connection' + */ + +const router = express.Router(); + +/** + * @openapi + * /api/v2/connections: + * get: + * summary: List Connections + * description: Retrieves a list of all ClickHouse connections for the authenticated team. Passwords are never returned. + * operationId: listConnections + * tags: [Connections] + * responses: + * '200': + * description: Successfully retrieved connections + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ConnectionsListResponse' + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + * '403': + * description: Forbidden + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + */ +router.get('/', async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const connections = await getConnectionsByTeam(teamId.toString()); + + res.json({ + data: connections + .map(formatExternalConnection) + .filter(c => c !== undefined), + }); + } catch (e) { + next(e); + } +}); + +/** + * @openapi + * /api/v2/connections/{id}: + * get: + * summary: Get Connection + * description: Retrieves a specific ClickHouse connection by ID. Passwords are never returned. + * operationId: getConnection + * tags: [Connections] + * parameters: + * - name: id + * in: path + * required: true + * schema: + * type: string + * description: Connection ID + * example: "507f1f77bcf86cd799439012" + * responses: + * '200': + * description: Successfully retrieved connection + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ConnectionResponseEnvelope' + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + * '404': + * description: Connection not found + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Connection not found" + */ +router.get( + '/:id', + validateRequest({ + params: z.object({ + id: objectIdSchema, + }), + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const connection = await getConnectionById( + teamId.toString(), + req.params.id, + ); + + if (connection == null) { + return res.sendStatus(404); + } + + res.json({ + data: formatExternalConnection(connection), + }); + } catch (e) { + next(e); + } + }, +); + +/** + * @openapi + * /api/v2/connections: + * post: + * summary: Create Connection + * description: Creates a new ClickHouse connection + * operationId: createConnection + * tags: [Connections] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/CreateConnectionRequest' + * responses: + * '200': + * description: Successfully created connection + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ConnectionResponseEnvelope' + * '400': + * description: Bad request + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Body validation failed: name: Required" + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + */ +router.post( + '/', + validateRequest({ + body: createConnectionBodySchema, + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const connection = await createConnection(teamId.toString(), { + ...req.body, + password: req.body.password ?? '', + team: teamId, + hyperdxSettingPrefix: req.body.hyperdxSettingPrefix ?? undefined, + }); + + res.json({ + data: formatExternalConnection(connection), + }); + } catch (e) { + next(e); + } + }, +); + +/** + * @openapi + * /api/v2/connections/{id}: + * put: + * summary: Update Connection + * description: | + * Updates an existing ClickHouse connection. + * + * Field semantics: if `password` is omitted or empty the existing + * password is kept. `hyperdxSettingPrefix` is cleared when set to null + * or an empty string, and `prometheusEndpoint` is cleared when set to + * null; both are kept unchanged when omitted. + * operationId: updateConnection + * tags: [Connections] + * parameters: + * - name: id + * in: path + * required: true + * schema: + * type: string + * description: Connection ID + * example: "507f1f77bcf86cd799439012" + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/UpdateConnectionRequest' + * responses: + * '200': + * description: Successfully updated connection + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/ConnectionResponseEnvelope' + * '400': + * description: Bad request + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Body validation failed: host: Required" + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + * '404': + * description: Connection not found + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Connection not found" + */ +router.put( + '/:id', + validateRequest({ + params: z.object({ + id: objectIdSchema, + }), + body: updateConnectionBodySchema, + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const existingConnection = await getConnectionById( + teamId.toString(), + req.params.id, + true, + ); + + if (existingConnection == null) { + return res.sendStatus(404); + } + + const { hyperdxSettingPrefix, prometheusEndpoint, ...restBody } = + req.body; + + const unsetFields: string[] = []; + if (hyperdxSettingPrefix === null || hyperdxSettingPrefix === '') { + unsetFields.push('hyperdxSettingPrefix'); + } + if (prometheusEndpoint === null) { + unsetFields.push('prometheusEndpoint'); + } + + const updatedConnection = await updateConnection( + teamId.toString(), + req.params.id, + { + ...restBody, + team: teamId, + // Keep the existing password when none is provided + password: req.body.password + ? req.body.password + : existingConnection.password, + ...(hyperdxSettingPrefix ? { hyperdxSettingPrefix } : {}), + ...(prometheusEndpoint ? { prometheusEndpoint } : {}), + }, + unsetFields, + ); + + if (updatedConnection == null) { + return res.sendStatus(404); + } + + res.json({ + data: formatExternalConnection(updatedConnection), + }); + } catch (e) { + next(e); + } + }, +); + +/** + * @openapi + * /api/v2/connections/{id}: + * delete: + * summary: Delete Connection + * description: Deletes a ClickHouse connection + * operationId: deleteConnection + * tags: [Connections] + * parameters: + * - name: id + * in: path + * required: true + * schema: + * type: string + * description: Connection ID + * example: "507f1f77bcf86cd799439012" + * responses: + * '200': + * description: Successfully deleted connection + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/EmptyResponse' + * example: {} + * '401': + * description: Unauthorized + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Unauthorized access. API key is missing or invalid." + * '404': + * description: Connection not found + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + * example: + * message: "Connection not found" + */ +router.delete( + '/:id', + validateRequest({ + params: z.object({ + id: objectIdSchema, + }), + }), + async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + const deletedConnection = await deleteConnection( + teamId.toString(), + req.params.id, + ); + + if (deletedConnection == null) { + return res.sendStatus(404); + } + + res.json({}); + } catch (e) { + next(e); + } + }, +); + +export default router; diff --git a/packages/api/src/routers/external-api/v2/index.ts b/packages/api/src/routers/external-api/v2/index.ts index cb4baafc00..59fdf95dc1 100644 --- a/packages/api/src/routers/external-api/v2/index.ts +++ b/packages/api/src/routers/external-api/v2/index.ts @@ -3,6 +3,7 @@ import express from 'express'; import { validateUserAccessKey } from '@/middleware/auth'; import alertsRouter from '@/routers/external-api/v2/alerts'; import chartsRouter from '@/routers/external-api/v2/charts'; +import connectionsRouter from '@/routers/external-api/v2/connections'; import dashboardRouter from '@/routers/external-api/v2/dashboards'; import searchRouter from '@/routers/external-api/v2/search'; import sourcesRouter from '@/routers/external-api/v2/sources'; @@ -30,6 +31,13 @@ router.use('/alerts', defaultRateLimiter, validateUserAccessKey, alertsRouter); router.use('/charts', defaultRateLimiter, validateUserAccessKey, chartsRouter); +router.use( + '/connections', + defaultRateLimiter, + validateUserAccessKey, + connectionsRouter, +); + router.use( '/dashboards', defaultRateLimiter, diff --git a/packages/api/src/utils/swagger.ts b/packages/api/src/utils/swagger.ts index 6cf9192176..3d86f0d25a 100644 --- a/packages/api/src/utils/swagger.ts +++ b/packages/api/src/utils/swagger.ts @@ -32,6 +32,10 @@ export const swaggerOptions = { name: 'Charts', description: 'Endpoints for querying chart data', }, + { + name: 'Connections', + description: 'Endpoints for managing ClickHouse connections', + }, { name: 'Sources', description: 'Endpoints for managing data sources', From a83246e2319b350e19d1766f42f0e81024cdb908 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 12 Jun 2026 09:08:04 +1000 Subject: [PATCH 2/2] fix: resolved formatExternalConnection silently failing. Now returning serialization error --- .../api/src/routers/external-api/v2/connections.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/api/src/routers/external-api/v2/connections.ts b/packages/api/src/routers/external-api/v2/connections.ts index 4093e118b5..c850b8d504 100644 --- a/packages/api/src/routers/external-api/v2/connections.ts +++ b/packages/api/src/routers/external-api/v2/connections.ts @@ -49,13 +49,16 @@ function formatExternalConnection(connection: ConnectionDocument) { return parseResult.data; } - // If parsing fails, log the error and return undefined + // If parsing fails, log and throw so handlers return an explicit 500 + // instead of silently responding with `{}` or a partial list. logger.error( { connectionId: connection._id, error: parseResult.error }, 'Failed to parse connection using externalConnectionSchema:', ); - return undefined; + throw new Error( + `Failed to serialize connection ${connection._id} for external API`, + ); } /** @@ -231,9 +234,7 @@ router.get('/', async (req, res, next) => { const connections = await getConnectionsByTeam(teamId.toString()); res.json({ - data: connections - .map(formatExternalConnection) - .filter(c => c !== undefined), + data: connections.map(formatExternalConnection), }); } catch (e) { next(e);