chore: loki and opensearch improvements

.github/workflows/test.yaml

@@ -12,18 +12,8 @@ jobs:
       fail-fast: false
       matrix:
         postgres-version:
-          - {
-              tag: '14',
-              sha: 'sha256:bbcaba1d74865ee6d6318b5e297d0df73d1f6b6d995cd892b60a2cf1440b716a',
-            }
-          - {
-              tag: '15',
-              sha: 'sha256:f57a3bdbf044f0b213fdc99f35a0d21c401608bf41f063176ec00c51df9655f7',
-            }
-          - {
-              tag: '16',
-              sha: 'sha256:47053cd4ee3f096afc744e53e3280de7b29b3670d2f2196c2acc0c6470923c99',
-            }
+          - { tag: "15", sha: "sha256:f57a3bdbf044f0b213fdc99f35a0d21c401608bf41f063176ec00c51df9655f7" }
+          - { tag: "16", sha: "sha256:47053cd4ee3f096afc744e53e3280de7b29b3670d2f2196c2acc0c6470923c99" }
     services:
       postgres:
         image: postgres:${{ matrix.postgres-version.tag }}@${{ matrix.postgres-version.sha }}
@@ -59,21 +49,10 @@ jobs:
         run: make test
         env:
           DUTY_DB_URL: postgres://postgres:password@localhost:5432/test?sslmode=disable
-          DUTY_DB_CREATE: 'false'
-          DUTY_DB_DISABLE_RLS: ${{ matrix.postgres-version.tag == '14' && 'true' || 'false' }}
+          DUTY_DB_CREATE: "false"
 
   e2e:
     runs-on: ubuntu-latest
-    services:
-      loki:
-        image: grafana/loki:3.5.1@sha256:3a4e80320df38087263e9d450fd812010ee57e1e0b0679465672c0e227441f43
-        ports:
-          - 3100:3100
-        options: >-
-          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
     steps:
       - name: Install Go
         uses: buildjet/setup-go@555ce355a95ff01018ffcf8fbbd9c44654db8374 # v5.0.2
@@ -91,32 +70,28 @@ jobs:
           key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
           restore-keys: |
             cache-
+
+      - uses: flanksource/deps@v1.0.23
+        with:
+          tools: |
+            loki@3.5.1
+            golangci-lint@v2.7.2
+            postgrest
+
       - name: E2E Test
         run: |
-          make ginkgo
-          ginkgo -r tests/e2e/
+          make test-e2e
         env:
-          DUTY_DB_DISABLE_RLS: 'true'
-          LOKI_URL: http://localhost:3100
+          DUTY_DB_DISABLE_RLS: "true"
 
   migrate:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         postgres-version:
-          - {
-              tag: '14',
-              sha: 'sha256:bbcaba1d74865ee6d6318b5e297d0df73d1f6b6d995cd892b60a2cf1440b716a',
-            }
-          - {
-              tag: '15',
-              sha: 'sha256:f57a3bdbf044f0b213fdc99f35a0d21c401608bf41f063176ec00c51df9655f7',
-            }
-          - {
-              tag: '16',
-              sha: 'sha256:47053cd4ee3f096afc744e53e3280de7b29b3670d2f2196c2acc0c6470923c99',
-            }
+          - { tag: "15", sha: "sha256:f57a3bdbf044f0b213fdc99f35a0d21c401608bf41f063176ec00c51df9655f7" }
+          - { tag: "16", sha: "sha256:47053cd4ee3f096afc744e53e3280de7b29b3670d2f2196c2acc0c6470923c99" }
     env:
       migrate_command: >
         make -B tidy hack/migrate/go.mod &&

.gitignore

@@ -12,4 +12,4 @@ __debug_bin*
 
 # This is where shell tests clone the checkout repository
 exec-checkout/
-shell-bin-dir
+ginkgo-report*

Makefile

@@ -7,24 +7,14 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
 CONTROLLER_TOOLS_VERSION ?= v0.19.0
 GOLANGCI_LINT_VERSION ?= v2.7.2
 
-.PHONY: ginkgo
-ginkgo:
-	go install github.com/onsi/ginkgo/v2/ginkgo
 
-test: ginkgo
-	ginkgo -r -v --skip-package=tests/e2e
+test:
+		go run github.com/onsi/ginkgo/v2/ginkgo -r -v --skip-package=tests/e2e --skip-package=logs/loki
 
 .PHONY: test-e2e
-test-e2e: ginkgo
-	cd tests/e2e && docker-compose up -d && \
-	timeout 60 bash -c 'until curl -s http://localhost:3100/ready >/dev/null 2>&1; do sleep 2; done' && \
-	(ginkgo -v; TEST_EXIT_CODE=$$?; docker-compose down; exit $$TEST_EXIT_CODE)
-
-.PHONY: e2e-services
-e2e-services: ## Run e2e test services in foreground with automatic cleanup on exit
-	cd tests/e2e && \
-	trap 'docker-compose down -v && docker-compose rm -f' EXIT INT TERM && \
-	docker-compose up --remove-orphans
+test-e2e:
+	go run github.com/onsi/ginkgo/v2/ginkgo -v
+
 
 .PHONY: bench
 bench:

api/http.go

@@ -45,7 +45,8 @@ func WriteSuccess(c echo.Context, payload any) error {
 func WriteError(c echo.Context, err error) error {
 	var oopsErr oops.OopsError
 	if errors.As(err, &oopsErr) {
-		return c.JSON(ErrorStatusCode(oopsErr.Code()), oopsErr)
+		code, _ := oopsErr.Code().(string)
+		return c.JSON(ErrorStatusCode(code), oopsErr)
 	}
 
 	code, message, data := ErrorCode(err), ErrorMessage(err), ErrorData(err)

connection/gcs.go

@@ -7,6 +7,7 @@ import (
 	gcs "cloud.google.com/go/storage"
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/types"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/option"
 )
 
@@ -49,7 +50,11 @@ func (g *GCSConnection) Client(ctx context.Context) (*gcs.Client, error) {
 		if err != nil {
 			return nil, err
 		}
-		clientOpts = append(clientOpts, option.WithCredentialsJSON([]byte(credential)))
+		creds, err := google.CredentialsFromJSON(ctx, []byte(credential), gcs.ScopeReadWrite)
+		if err != nil {
+			return nil, err
+		}
+		clientOpts = append(clientOpts, option.WithCredentials(creds))
 	} else {
 		clientOpts = append(clientOpts, option.WithoutAuthentication())
 	}

connection/gke.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 
 	"github.com/flanksource/duty/context"
+	"golang.org/x/oauth2/google"
 	container "google.golang.org/api/container/v1"
 	"google.golang.org/api/option"
 	"k8s.io/client-go/kubernetes"
@@ -58,7 +59,11 @@ func (t *GKEConnection) Client(ctx context.Context) (*container.Service, error)
 		if err != nil {
 			return nil, err
 		}
-		clientOpts = append(clientOpts, option.WithCredentialsJSON([]byte(credential)))
+		creds, err := google.CredentialsFromJSON(ctx, []byte(credential), container.CloudPlatformScope)
+		if err != nil {
+			return nil, err
+		}
+		clientOpts = append(clientOpts, option.WithCredentials(creds))
 	} else {
 		clientOpts = append(clientOpts, option.WithoutAuthentication())
 	}