diff --git a/.github/actions/publish-website/action.yml b/.github/actions/publish-website/action.yml index baf650b94..f822065ca 100644 --- a/.github/actions/publish-website/action.yml +++ b/.github/actions/publish-website/action.yml @@ -39,7 +39,7 @@ runs: run: docker load --input /tmp/mokapi.tar shell: bash - name: Run mokapi image - run: docker run --name mokapi --rm -d -p 80:80 -p 8080:8080 -p 9092:9092 -p 8389:8389 -p 8025:8025 --mount type=bind,source=$(pwd)/webui/scripts/dashboard-demo/demo-configs,target=/data --env MOKAPI_Providers_File_Directory=/data ${{ inputs.image-name }} + run: docker run --name mokapi --rm -d -p 80:80 -p 1883:1883 -p 8080:8080 -p 9092:9092 -p 8389:8389 -p 8025:8025 --mount type=bind,source=$(pwd)/webui/scripts/dashboard-demo/demo-configs,target=/data --env MOKAPI_Providers_File_Directory=/data ${{ inputs.image-name }} shell: bash - name: Build Demo Dashboard working-directory: ./webui/scripts/dashboard-demo diff --git a/acceptance/cmd_test.go b/acceptance/cmd_test.go index 458193e00..513767e1a 100644 --- a/acceptance/cmd_test.go +++ b/acceptance/cmd_test.go @@ -65,10 +65,12 @@ func Start(cfg *static.Config) (*Cmd, error) { }) apiHandler := api.New(app, cfg.Api) - if u, err := api.BuildUrl(cfg.Api); err == nil { - err = http.AddInternalService("api", u, apiHandler) - if err != nil { - return nil, err + if urls, err := api.BuildUrl(cfg.Api); err == nil { + for _, u := range urls { + err = http.AddInternalService("api", u, apiHandler) + if err != nil { + return nil, err + } } } else { return nil, err diff --git a/acceptance/ldap_test.go b/acceptance/ldap_test.go index 575246ff9..e61a007cc 100644 --- a/acceptance/ldap_test.go +++ b/acceptance/ldap_test.go @@ -81,7 +81,7 @@ func (suite *LdapSuite) TestMetric() { } _, err := suite.Client.Search(search) require.NoError(suite.T(), err) - require.Equal(suite.T(), float64(1), suite.cmd.App.Monitor.Ldap.RequestCounter.Sum()) + require.Equal(suite.T(), float64(1), suite.cmd.App.Monitor.Ldap.RequestCounter.Sum(metrics.NewQuery())) q := metrics.NewQuery(metrics.ByNamespace("ldap"), metrics.ByName("request_timestamp")) require.Greater(suite.T(), suite.cmd.App.Monitor.Ldap.LastRequest.Value(q), float64(1)) } diff --git a/acceptance/petstore/asyncapi.yml b/acceptance/petstore/asyncapi.yml index 2ded4c390..c9c719fbe 100644 --- a/acceptance/petstore/asyncapi.yml +++ b/acceptance/petstore/asyncapi.yml @@ -6,6 +6,9 @@ servers: broker: url: 127.0.0.1:19092 protocol: kafka + mqtt: + url: 127.0.0.1:11883 + protocol: mqtt channels: petstore.order-event: subscribe: diff --git a/acceptance/petstore_test.go b/acceptance/petstore_test.go index 82f95c753..ed6fafb19 100644 --- a/acceptance/petstore_test.go +++ b/acceptance/petstore_test.go @@ -28,6 +28,7 @@ func (suite *PetStoreSuite) SetupSuite() { cfg.Providers.File.Directories = []static.FileConfig{{Path: "./petstore"}} cfg.Api.Search.Enabled = true cfg.Api.Search.InMemory = true + cfg.Api.Search.NumIndexWorker = 1 suite.initCmd(cfg) } @@ -54,78 +55,21 @@ func (suite *PetStoreSuite) TestApi() { suite.T().Run("get AsyncAPI service", func(t *testing.T) { expected := map[string]interface{}{ - "version": "1.0.0", - "name": "A sample AsyncApi Kafka streaming api", - "description": "", + "version": "1.0.0", + "name": "A sample AsyncApi Kafka streaming api", "servers": []interface{}{ map[string]interface{}{ - "host": "127.0.0.1:19092", - "name": "broker", - "protocol": "kafka", - "title": "", - "summary": "", - "description": "", + "host": "127.0.0.1:19092", + "name": "broker", + "protocol": "kafka", }, }, "topics": []interface{}{map[string]interface{}{ - "bindings": map[string]interface{}{"partitions": float64(2), "segmentMs": float64(30000), "valueSchemaValidation": true}, - "description": "", - "messages": map[string]interface{}{ - "order": map[string]interface{}{ - "name": "order", - "contentType": "application/json", - "header": map[string]interface{}{ - "schema": map[string]interface{}{ - "properties": map[string]interface{}{ - "number": map[string]interface{}{ - "type": "number", - }, "test": map[string]interface{}{ - "type": "string", - }, - }, "type": "object", - }, - }, - "key": map[string]interface{}{ - "schema": map[string]interface{}{ - "type": "string", - }, - }, - "payload": map[string]interface{}{ - "schema": map[string]interface{}{ - "properties": map[string]interface{}{ - "accepted": map[string]interface{}{ - "properties": map[string]interface{}{ - "timestamp": map[string]interface{}{"format": "date-time", "type": "string"}, - }, - "type": "object", - }, - "completed": map[string]interface{}{ - "properties": map[string]interface{}{ - "timestamp": map[string]interface{}{"format": "date-time", "type": "string"}, - }, - "type": "object", - }, - "id": map[string]interface{}{"type": "integer"}, - "placed": map[string]interface{}{ - "properties": map[string]interface{}{ - "petid": map[string]interface{}{"type": "integer"}, - "quantity": map[string]interface{}{"format": "int32", "type": "integer"}, - "ship-date": map[string]interface{}{"format": "date-time", "type": "string"}, - }, - "type": "object", - }, - }, - "required": []interface{}{"id"}, - "type": "object", - }, - }, - }, - }, "name": "petstore.order-event", - "partitions": []interface{}{ - map[string]interface{}{"id": float64(0), "offset": float64(1), "segments": float64(1), "startOffset": float64(0)}, - map[string]interface{}{"id": float64(1), "offset": float64(0), "segments": float64(0), "startOffset": float64(0)}, + "metrics": map[string]interface{}{ + // skip timestamp check "kafka_message_timestamp" + "kafka_messages_total": float64(1), }, }}, } @@ -220,7 +164,7 @@ func (suite *PetStoreSuite) TestKafka_TopicConfig() { require.Equal(suite.T(), "petstore.order-event", r.Topics[0].Name) require.Len(suite.T(), r.Topics[0].Partitions, 2) - require.Len(suite.T(), suite.cmd.App.ListHttp(), 1) + require.Equal(suite.T(), 1, suite.cmd.App.Http.Len()) } func (suite *PetStoreSuite) TestKafka_Produce_InvalidFormat() { @@ -303,8 +247,16 @@ func (suite *PetStoreSuite) TestEvents() { assert.True(t, ok, "event should be a map[string]any") assert.NotNil(t, evt) assert.Equal(t, "Event", evt["type"]) - assert.Equal(t, "GET http://127.0.0.1:18080/user/bob", evt["title"]) + assert.Equal(t, "http://127.0.0.1:18080/user/bob", evt["title"]) assert.Equal(t, "Swagger Petstore", evt["domain"]) + params := evt["params"].(map[string]any) + assert.Len(t, params, 6) + assert.Equal(t, "event", params["type"]) + assert.Equal(t, "http", params["traits.namespace"]) + assert.Equal(t, "Swagger Petstore", params["traits.name"]) + assert.Equal(t, "/user/{username}", params["traits.path"]) + assert.Equal(t, "GET", params["traits.method"]) + assert.Equal(t, "Swagger Petstore", params["traits.name"]) }), ) } @@ -389,7 +341,7 @@ func (suite *PetStoreSuite) TestSearch_Paging() { assert.Len(t, items, 10) evt := items[0].(map[string]interface{}) assert.Equal(t, "HTTP", evt["type"]) - assert.Equal(t, "GET /pet/{petId}", evt["title"]) + assert.Equal(t, "/pet/{petId}", evt["title"]) assert.Equal(t, "Swagger Petstore", evt["domain"]) }), ) diff --git a/api/handler.go b/api/handler.go index 5ddc704d9..c19c936db 100644 --- a/api/handler.go +++ b/api/handler.go @@ -7,7 +7,6 @@ import ( "io/fs" "mokapi/config/static" "mokapi/runtime" - "mokapi/runtime/metrics" "mokapi/webui" "net/http" "net/url" @@ -16,6 +15,7 @@ import ( "strconv" "strings" + "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -36,6 +36,7 @@ type handler struct { healthHandler http.Handler mcpPath string mcpHandler http.Handler + router *mux.Router } type info struct { @@ -56,15 +57,17 @@ var ( ServiceKafka serviceType = "kafka" ServiceMail serviceType = "mail" ServiceLdap serviceType = "ldap" + ServiceMqtt serviceType = "mqtt" ) type service struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Contact *contact `json:"contact,omitempty"` - Version string `json:"version,omitempty"` - Type serviceType `json:"type"` - Metrics []metrics.Metric `json:"metrics,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Contact *contact `json:"contact,omitempty"` + Version string `json:"version,omitempty"` + Type serviceType `json:"type"` + Status string `json:"status,omitempty"` + Metrics any `json:"metrics,omitempty"` } type contact struct { @@ -83,6 +86,7 @@ func New(app *runtime.App, config static.Api) Handler { path: config.Path, base: config.Base, app: app, + router: mux.NewRouter(), } if config.Dashboard { @@ -101,12 +105,26 @@ func New(app *runtime.App, config static.Api) Handler { h.fileServer = http.FileServer(http.FS(dist)) } + h.setupHttp() + h.setupKafka() + h.setupMqtt() + return h } -func BuildUrl(cfg static.Api) (*url.URL, error) { - s := fmt.Sprintf("http://:%v%v", cfg.Port, cfg.Path) - return url.Parse(s) +func BuildUrl(cfg static.Api) ([]*url.URL, error) { + var urls []*url.URL + u, err := url.Parse(fmt.Sprintf("http://:%v%v", cfg.Port, cfg.Path)) + if err != nil { + return urls, err + } + urls = append(urls, u) + u, err = url.Parse(fmt.Sprintf("http://localhost:%v%v", cfg.Port, cfg.Path)) + if err != nil { + return urls, err + } + urls = append(urls, u) + return urls, nil } func (h *handler) RegisterHealthHandler(path string, handler http.Handler) { @@ -134,10 +152,6 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.getInfo(w, r) case p == "/api/services": h.getServices(w, r) - case strings.HasPrefix(p, "/api/services/http/"): - h.getHttpService(w, r, h.app.Monitor) - case strings.HasPrefix(p, "/api/services/kafka"): - h.handleKafka(w, r) case strings.HasPrefix(p, "/api/services/mail/"): h.handleMailService(w, r) case strings.HasPrefix(p, "/api/services/ldap/"): @@ -164,6 +178,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.healthHandler.ServeHTTP(w, r) case strings.HasPrefix(p, h.mcpPath) && h.mcpHandler != nil: h.mcpHandler.ServeHTTP(w, r) + case strings.HasPrefix(p, "/api/"): + h.router.ServeHTTP(w, r) case h.fileServer != nil: if r.Method != "GET" { http.Error(w, fmt.Sprintf("method %v is not allowed", r.Method), http.StatusMethodNotAllowed) @@ -198,14 +214,28 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (h *handler) getServices(w http.ResponseWriter, _ *http.Request) { - services := make([]interface{}, 0) - services = append(services, getHttpServices(h.app.ListHttp(), h.app.Monitor)...) - services = append(services, getKafkaServices(h.app.Kafka, h.app.Monitor)...) - services = append(services, getMailServices(h.app.Mail, h.app.Monitor)...) - services = append(services, getLdapServices(h.app.Ldap, h.app.Monitor)...) - slices.SortFunc(services, func(a interface{}, b interface{}) int { - return compareService(a, b) +func (h *handler) getServices(w http.ResponseWriter, r *http.Request) { + services := make([]service, 0) + + typ := r.URL.Query().Get("type") + + if typ == "" || typ == "http" { + services = append(services, getHttpServices(h.app.Http, h.app.Monitor)...) + } + if typ == "" || typ == "kafka" { + services = append(services, getKafkaServices(h.app.Kafka, h.app.Monitor)...) + } + if typ == "" || typ == "mail" { + services = append(services, getMailServices(h.app.Mail, h.app.Monitor)...) + } + if typ == "" || typ == "ldap" { + services = append(services, getLdapServices(h.app.Ldap, h.app.Monitor)...) + } + if typ == "" || typ == "mqtt" { + services = append(services, getMqttServices(h.app.Mqtt, h.app.Monitor)...) + } + slices.SortFunc(services, func(a service, b service) int { + return strings.Compare(a.Name, b.Name) }) w.Header().Set("Content-Type", "application/json") writeJsonBody(w, services) @@ -230,22 +260,30 @@ func (h *handler) getInfo(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") i := info{Version: h.app.Version, BuildTime: h.app.BuildTime, Search: searchInfo{Enabled: h.config.Search.Enabled}} - if len(h.app.ListHttp()) > 0 { + if h.app.Http.Len() > 0 { i.ActiveServices = append(i.ActiveServices, "http") } - if len(h.app.Kafka.List()) > 0 { + if h.app.Kafka.Len() > 0 { i.ActiveServices = append(i.ActiveServices, "kafka") } - if len(h.app.Mail.List()) > 0 { + if h.app.Mail.Len() > 0 { i.ActiveServices = append(i.ActiveServices, "mail") } - if len(h.app.Ldap.List()) > 0 { + if h.app.Ldap.Len() > 0 { i.ActiveServices = append(i.ActiveServices, "ldap") } + if h.app.Mqtt.Len() > 0 { + i.ActiveServices = append(i.ActiveServices, "mqtt") + } writeJsonBody(w, i) } +func write(w http.ResponseWriter, data any) { + w.Header().Set("Content-Type", "application/json") + writeJsonBody(w, data) +} + func writeJsonBody(w http.ResponseWriter, v interface{}) { var buf bytes.Buffer enc := json.NewEncoder(&buf) @@ -278,24 +316,6 @@ func isImage(path string) bool { } } -func compareService(a, b interface{}) int { - return strings.Compare(getServiceName(a), getServiceName(b)) -} - -func getServiceName(a interface{}) string { - switch v := a.(type) { - case *httpSummary: - return v.Name - case *kafkaSummary: - return v.Name - case *ldapSummary: - return v.Name - case *mailSummary: - return v.Name - } - return "" -} - func getPageInfo(r *http.Request) (index int, limit int, err error) { limit = 10 diff --git a/api/handler_config.go b/api/handler_config.go index 180e005f9..d79336996 100644 --- a/api/handler_config.go +++ b/api/handler_config.go @@ -12,22 +12,22 @@ import ( log "github.com/sirupsen/logrus" ) -type config struct { - Id string `json:"id"` - Url string `json:"url"` - Provider string `json:"provider"` - Time time.Time `json:"time"` - Refs []configRef `json:"refs,omitempty"` - Tags []string `json:"tags,omitempty"` -} - -type configRef struct { +type configInfo struct { Id string `json:"id"` Url string `json:"url"` Provider string `json:"provider"` Time time.Time `json:"time"` } +type config struct { + Id string `json:"id"` + Url string `json:"url"` + Provider string `json:"provider"` + Time time.Time `json:"time"` + Refs []configInfo `json:"refs,omitempty"` + Tags []string `json:"tags,omitempty"` +} + func (h *handler) handleConfig(w http.ResponseWriter, r *http.Request) { segments := strings.Split(r.URL.Path, "/") if len(segments) == 3 { @@ -104,6 +104,19 @@ func (h *handler) getConfigData(w http.ResponseWriter, r *http.Request, key stri } } +func getConfigInfos(list []*dynamic.Config) []configInfo { + var result []configInfo + for _, c := range list { + result = append(result, configInfo{ + Id: c.Info.Key(), + Url: filepath.ToSlash(c.Info.Path()), + Provider: c.Info.Provider, + Time: c.Info.Time, + }) + } + return result +} + func getConfigs(src []*dynamic.Config) (dst []config) { for _, cfg := range src { dst = append(dst, toConfig(cfg)) @@ -112,9 +125,9 @@ func getConfigs(src []*dynamic.Config) (dst []config) { } func toConfig(cfg *dynamic.Config) config { - var refs []configRef + var refs []configInfo for _, ref := range cfg.Refs.List(false) { - refs = append(refs, configRef{ + refs = append(refs, configInfo{ Id: ref.Info.Key(), Url: filepath.ToSlash(ref.Info.Path()), Provider: ref.Info.Provider, diff --git a/api/handler_events_test.go b/api/handler_events_test.go index a85dd3450..ce885079f 100644 --- a/api/handler_events_test.go +++ b/api/handler_events_test.go @@ -95,7 +95,11 @@ func TestHandler_Events(t *testing.T) { } r = r.WithContext(openapi.NewContext(context.Background(), params)) - _, err := openapi.NewLogEventContext(r, false, sm, events.NewTraits().WithNamespace("http")) + traits := events.NewTraits().WithNamespace("http") + ctx, err := openapi.NewLogEventContext(r, false, traits) + require.NoError(t, err) + l, _ := openapi.LogEventFromContext(ctx) + err = sm.Push(l, traits) require.NoError(t, err) try.Handler(t, http.MethodGet, @@ -148,7 +152,11 @@ func TestHandler_Events(t *testing.T) { } r = r.WithContext(openapi.NewContext(context.Background(), params)) - _, err := openapi.NewLogEventContext(r, false, sm, events.NewTraits().WithNamespace("http")) + traits := events.NewTraits().WithNamespace("http") + ctx, err := openapi.NewLogEventContext(r, false, traits) + require.NoError(t, err) + l, _ := openapi.LogEventFromContext(ctx) + err = sm.Push(l, traits) require.NoError(t, err) try.Handler(t, http.MethodGet, diff --git a/api/handler_fileserver_test.go b/api/handler_fileserver_test.go index 8fc9033eb..8d707b29c 100644 --- a/api/handler_fileserver_test.go +++ b/api/handler_fileserver_test.go @@ -148,7 +148,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."))}) + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."))}) h := api.New(app, static.Api{Path: "/mokapi", Dashboard: true}) try.Handler(t, http.MethodGet, @@ -168,7 +168,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."), openapitest.WithPath("/pet/{petId}"), )}, @@ -192,7 +192,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."), openapitest.WithPath("/pet/{petId}", openapitest.WithPathInfo("foo", "bar"), @@ -218,7 +218,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."), openapitest.WithPath("/pet/{petId}", openapitest.WithPathInfo("", "bar"), @@ -243,7 +243,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."), openapitest.WithPath("/pet/{petId}", openapitest.WithOperation("GET"), @@ -268,7 +268,7 @@ func TestOpenGraphInDashboard(t *testing.T) { test: func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - app.AddHttp(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", + app.Http.Add(&dynamic.Config{Info: dynamic.ConfigInfo{Url: mustParse("https://foo.bar")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server."), openapitest.WithPath("/pet/{petId}", openapitest.WithOperation("GET"), diff --git a/api/handler_http.go b/api/handler_http.go index f6adb8646..20acdefb1 100644 --- a/api/handler_http.go +++ b/api/handler_http.go @@ -1,40 +1,54 @@ package api import ( + "maps" "mokapi/providers/openapi" "mokapi/providers/openapi/schema" "mokapi/runtime" "mokapi/runtime/metrics" "mokapi/runtime/monitor" "net/http" + "slices" "strings" -) -type httpSummary struct { - service -} + "github.com/gorilla/mux" +) type httpInfo struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Version string `json:"version,omitempty"` - Contact *contact `json:"contact,omitempty"` - Servers []server `json:"servers,omitempty"` - Paths []pathItem `json:"paths,omitempty"` - Tags []tag `json:"tags,omitempty"` - Metrics []metrics.Metric `json:"metrics,omitempty"` - Configs []config `json:"configs,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version,omitempty"` + Contact *contact `json:"contact,omitempty"` + Servers []server `json:"servers,omitempty"` + Paths []pathItem `json:"paths,omitempty"` + Tags []tag `json:"tags,omitempty"` + Configs []config `json:"configs,omitempty"` } type pathItem struct { - Path string `json:"path"` + Path string `json:"path"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Status string `json:"status"` + Errors []errorData `json:"errors,omitempty"` + Operations []operationInfo `json:"operations,omitempty"` +} + +type operationInfo struct { + Method string `json:"method"` Summary string `json:"summary,omitempty"` Description string `json:"description,omitempty"` - Operations []operation `json:"operations,omitempty"` + OperationId string `json:"operationId,omitempty"` + Deprecated bool `json:"deprecated"` + Tags []string `json:"tags,omitempty"` + Status string `json:"status"` + Errors []errorData `json:"errors,omitempty"` + Metrics httpMetrics `json:"metrics"` } type operation struct { Method string `json:"method"` + Path string `json:"path"` Summary string `json:"summary,omitempty"` Description string `json:"description,omitempty"` OperationId string `json:"operationId,omitempty"` @@ -44,6 +58,13 @@ type operation struct { Responses []response `json:"responses,omitempty"` Security []securityRequirement `json:"security,omitempty"` Tags []string `json:"tags,omitempty"` + Status string `json:"status"` + Errors []errorData `json:"errors,omitempty"` + Metrics httpMetrics `json:"metrics"` +} + +type errorData struct { + Message string `json:"message"` } type param struct { @@ -102,24 +123,28 @@ type tag struct { Kind string `yaml:"kind" json:"kind"` } -func getHttpServices(list []*runtime.HttpInfo, m *monitor.Monitor) []interface{} { - result := make([]interface{}, 0, len(list)) +type httpMetrics struct { + Requests float64 `json:"http_requests_total"` + RequestErrors float64 `json:"http_requests_errors_total"` + LastRequest float64 `json:"http_request_timestamp"` +} + +func getHttpServices(s *runtime.HttpStore, m *monitor.Monitor) []service { + list := s.List() + result := make([]service, 0, len(list)) for _, hs := range list { s := service{ Name: hs.Info.Name, Description: hs.Info.Description, Version: hs.Info.Version, Type: ServiceHttp, + Status: hs.GetStatus().String(), } if hs.Info.Summary != "" { s.Description = hs.Info.Summary } - if m != nil { - s.Metrics = m.FindAll(metrics.ByNamespace("http"), metrics.ByLabel("service", hs.Info.Name)) - } - if hs.Info.Contact != nil { c := hs.Info.Contact s.Contact = &contact{ @@ -129,31 +154,63 @@ func getHttpServices(list []*runtime.HttpInfo, m *monitor.Monitor) []interface{} } } - result = append(result, &httpSummary{service: s}) + s.Metrics = httpMetrics{ + Requests: m.Http.RequestCounter.Sum(metrics.NewQuery(metrics.ByLabel("service", hs.Info.Name))), + RequestErrors: m.Http.RequestErrorCounter.Sum(metrics.NewQuery(metrics.ByLabel("service", hs.Info.Name))), + LastRequest: m.Http.LastRequest.Max(metrics.NewQuery(metrics.ByLabel("service", hs.Info.Name))), + } + + result = append(result, s) } return result } -func (h *handler) getHttpService(w http.ResponseWriter, r *http.Request, m *monitor.Monitor) { - segments := strings.Split(r.URL.Path, "/") - name := segments[4] +func (h *handler) setupHttp() { + r := h.router.PathPrefix("/api/services/http").Subrouter() + + r.HandleFunc("", h.getHttpServices).Methods(http.MethodGet) + r.HandleFunc("/{api}", h.getHttpApi).Methods(http.MethodGet) + r.HandleFunc("/{api}/operations", h.getHttpOperations).Methods(http.MethodGet) +} + +func (h *handler) getHttpServices(w http.ResponseWriter, _ *http.Request) { + services := getKafkaServices(h.app.Kafka, h.app.Monitor) + write(w, services) +} + +func (h *handler) getHttpApi(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + if s := h.app.Http.Get(vars["api"]); s != nil { + result := h.getHttpService(s) + write(w, result) + } else { + w.WriteHeader(404) + } +} + +func (h *handler) getHttpOperations(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) - s := h.app.GetHttp(name) + s := h.app.Http.Get(vars["api"]) if s == nil { w.WriteHeader(404) return } + method := r.URL.Query().Get("method") + p := r.URL.Query().Get("path") + result := getOperations(s, p, method, h.app.Monitor.Http) + write(w, result) +} + +func (h *handler) getHttpService(s *runtime.HttpInfo) httpInfo { result := httpInfo{ Name: s.Info.Name, Description: s.Info.Description, Version: s.Info.Version, } - if m != nil { - result.Metrics = m.FindAll(metrics.ByNamespace("http"), metrics.ByLabel("service", s.Info.Name)) - } - if s.Info.Contact != nil { result.Contact = &contact{ Name: s.Info.Contact.Name, @@ -177,6 +234,7 @@ func (h *handler) getHttpService(w http.ResponseWriter, r *http.Request, m *moni Path: path, Summary: p.Value.Summary, Description: p.Value.Description, + Status: p.Value.Status.String(), } if len(p.Summary) > 0 { pi.Summary = p.Summary @@ -185,14 +243,99 @@ func (h *handler) getHttpService(w http.ResponseWriter, r *http.Request, m *moni pi.Description = p.Description } - for m, o := range p.Value.Operations() { + for _, err := range p.Value.Errors { + pi.Errors = append(pi.Errors, errorData{Message: err.Message}) + } + + for method, o := range p.Value.Operations() { + data := httpMetrics{ + Requests: h.app.Monitor.Http.RequestCounter.Sum(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", method), + )), + RequestErrors: h.app.Monitor.Http.RequestErrorCounter.Sum(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", method), + )), + LastRequest: h.app.Monitor.Http.LastRequest.Max(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", method), + )), + } + + pi.Operations = append(pi.Operations, operationInfo{ + Method: strings.ToLower(method), + Summary: o.Summary, + Description: o.Description, + OperationId: o.OperationId, + Deprecated: o.Deprecated, + Tags: o.Tags, + Status: o.Status.String(), + Errors: getErrors(o.Errors), + Metrics: data, + }) + } + result.Paths = append(result.Paths, pi) + } + + for _, t := range s.Tags { + result.Tags = append(result.Tags, tag{ + Name: t.Name, + Summary: t.Summary, + Description: t.Description, + Parent: t.Parent, + Kind: t.Kind, + }) + } + + result.Configs = getConfigs(s.Configs()) + + return result +} + +func getOperations(s *runtime.HttpInfo, path, method string, monitor *monitor.Http) []operation { + var paths []string + if path != "" { + paths = append(paths, path) + } else { + keys := maps.Keys(s.Paths) + paths = slices.Sorted(keys) + } + + operations := make([]operation, 0, len(paths)) + for _, ps := range paths { + + p, ok := s.Paths[ps] + if !ok || p.Value == nil { + continue + } + + var methods []string + if method != "" { + methods = append(methods, method) + } else { + keys := maps.Keys(p.Value.Operations()) + methods = slices.Sorted(keys) + } + + for _, m := range methods { + o := p.Value.Operation(m) + if o == nil { + continue + } + op := operation{ Method: strings.ToLower(m), + Path: ps, Summary: o.Summary, Description: o.Description, OperationId: o.OperationId, Deprecated: o.Deprecated, Tags: o.Tags, + Status: o.Status.String(), } if o.RequestBody != nil && o.RequestBody.Value != nil { op.RequestBody = &requestBody{ @@ -202,9 +345,6 @@ func (h *handler) getHttpService(w http.ResponseWriter, r *http.Request, m *moni if len(o.RequestBody.Summary) > 0 { op.Summary = o.RequestBody.Summary } - if len(o.RequestBody.Description) > 0 { - pi.Description = o.RequestBody.Description - } for ct, rb := range o.RequestBody.Value.Content { op.RequestBody.Contents = append(op.RequestBody.Contents, mediaType{ @@ -271,26 +411,34 @@ func (h *handler) getHttpService(w http.ResponseWriter, r *http.Request, m *moni } op.Security = append(op.Security, req) } + if o.Errors != nil { + for _, err := range o.Errors { + op.Errors = append(op.Errors, errorData{Message: err.Message}) + } + } - pi.Operations = append(pi.Operations, op) - } - result.Paths = append(result.Paths, pi) - } + op.Metrics = httpMetrics{ + Requests: monitor.RequestCounter.Sum(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", m), + )), + RequestErrors: monitor.RequestErrorCounter.Sum(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", m), + )), + LastRequest: monitor.LastRequest.Max(metrics.NewQuery( + metrics.ByLabel("service", s.Info.Name), + metrics.ByLabel("endpoint", p.Value.Path), + metrics.ByLabel("method", m), + )), + } - for _, t := range s.Tags { - result.Tags = append(result.Tags, tag{ - Name: t.Name, - Summary: t.Summary, - Description: t.Description, - Parent: t.Parent, - Kind: t.Kind, - }) + operations = append(operations, op) + } } - - result.Configs = getConfigs(s.Configs()) - - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, result) + return operations } func getParameters(params openapi.Parameters) (result []param) { @@ -318,3 +466,11 @@ func getParameters(params openapi.Parameters) (result []param) { } return } + +func getErrors(err []openapi.Error) []errorData { + var errData []errorData + for _, e := range err { + errData = append(errData, errorData{Message: e.Message}) + } + return errData +} diff --git a/api/handler_http_test.go b/api/handler_http_test.go index 93f015f2a..e9227a8fa 100644 --- a/api/handler_http_test.go +++ b/api/handler_http_test.go @@ -39,7 +39,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"http"}`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "summary takes precedence over description", @@ -52,7 +52,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"summary","version":"1.0","type":"http"}`, + responseBody: `[{"name":"foo","description":"summary","version":"1.0","type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}`, }, { name: "get http services with contact", @@ -64,7 +64,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"type":"http"}]`, + responseBody: `[{"name":"foo","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "get specific http service", @@ -86,7 +86,7 @@ func TestHandler_Http(t *testing.T) { ), } cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") - app.AddHttp(cfg) + app.Http.Add(cfg) return app }, requestUrl: "http://foo.api/api/services/http/foo", @@ -118,6 +118,51 @@ func TestHandler_Http(t *testing.T) { requestUrl: "http://foo.api/api/services/http/foo", responseBody: `{"name":"foo","servers":[{"url":"https://foo.bar","description":"a foo description"}]`, }, + { + name: "get http service with parameters", + app: func() *runtime.App { + return runtimetest.NewHttpApp( + openapitest.NewConfig("3.0.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo/{bar}", + openapitest.WithPathErrors(openapi.Error{Message: "error"}), + ), + ), + ) + }, + requestUrl: "http://foo.api/api/services/http/foo", + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"invalid","errors":[{"message":"error"}]}]`, + }, + { + name: "path does not exist", + app: func() *runtime.App { + return runtimetest.NewHttpApp( + openapitest.NewConfig("3.0.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo", + openapitest.WithPathErrors(openapi.Error{Message: "error"}), + ), + ), + ) + }, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/bar", + responseBody: `[]`, + }, + { + name: "operation does not exist", + app: func() *runtime.App { + return runtimetest.NewHttpApp( + openapitest.NewConfig("3.0.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo", + openapitest.WithPathErrors(openapi.Error{Message: "error"}), + ), + ), + ) + }, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo", + responseBody: `[]`, + }, { name: "get http service with parameters", app: func() *runtime.App { @@ -131,8 +176,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"parameters":[{"name":"bar","type":"path","required":true,"deprecated":false,"explode":false,"allowReserved":false,"schema":{"type":"string"}}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"parameters":[{"name":"bar","type":"path","required":true,"deprecated":false,"explode":false,"allowReserved":false,"schema":{"type":"string"}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "get http service with requestBody", @@ -149,8 +194,8 @@ func TestHandler_Http(t *testing.T) { )), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"requestBody":{"description":"foo","contents":[{"type":"application/json","schema":{"type":"string"}}],"required":true}}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"requestBody":{"description":"foo","contents":[{"type":"application/json","schema":{"type":"string"}}],"required":true},"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "get http service with security", @@ -171,8 +216,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo","operations":[{"method":"get","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo", + responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "get http service with global security", @@ -192,8 +237,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo","operations":[{"method":"get","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo", + responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "get http service with response", @@ -214,8 +259,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "reference override summary/description", @@ -252,8 +297,8 @@ func TestHandler_Http(t *testing.T) { return runtimetest.NewHttpApp(c) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","summary":"Summary","description":"Description","operations":[{"method":"get","deprecated":false,"responses":[{"statusCode":"200","description":"Description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"Description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "schema with string or number", @@ -274,8 +319,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":["string","number"]}}]}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":["string","number"]}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "schema with default", @@ -295,8 +340,8 @@ func TestHandler_Http(t *testing.T) { ), ) }, - requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string","default":"foobar"}}]}]}]}]`, + requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string","default":"foobar"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, }, { name: "tags", @@ -314,7 +359,26 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","operations":[{"method":"get","deprecated":false,"tags":["foo"]}]}],"tags":[{"name":"foo","summary":"sum","description":"desc","parent":"","kind":""}]`, + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"tags":["foo"],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]}],"tags":[{"name":"foo","summary":"sum","description":"desc","parent":"","kind":""}]`, + }, + { + name: "operation with errors", + app: func() *runtime.App { + return runtimetest.NewHttpApp( + openapitest.NewConfig("3.0.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo/{bar}", + openapitest.WithOperation("get", + openapitest.WithOperationErrors(openapi.Error{ + Message: "error", + }), + ), + ), + ), + ) + }, + requestUrl: "http://foo.api/api/services/http/foo", + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"invalid","errors":[{"message":"error"}],"metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]}]`, }, } @@ -364,18 +428,27 @@ func TestHandler_Http_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewHttpApp(openapitest.NewConfig("3.0.0", openapitest.WithInfo("foo", "", ""))), requestUrl: "http://foo.api/api/services", - responseBody: `{"name":"foo","type":"http","metrics":[{"name":"http_requests_total{service=\"foo\",endpoint=\"bar\",method=\"GET\"}","value":1}]}`, + responseBody: `{"name":"foo","type":"http","status":"valid","metrics":{"http_requests_total":1,"http_requests_errors_total":3,"http_request_timestamp":123456}}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Http.RequestCounter.WithLabel("foo", "bar", http.MethodGet).Add(1) + monitor.Http.RequestErrorCounter.WithLabel("foo", "bar", http.MethodGet).Add(3) + monitor.Http.LastRequest.WithLabel("foo", "bar", http.MethodGet).Set(float64(123456)) }, }, { - name: "specific with metric", - app: runtimetest.NewHttpApp(openapitest.NewConfig("3.0.0", openapitest.WithInfo("foo", "", ""))), + name: "specific with metric", + app: runtimetest.NewHttpApp( + openapitest.NewConfig("3.0.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo/{bar}", openapitest.WithOperation("get")), + ), + ), requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `"metrics":[{"name":"http_requests_total{service=\"foo\",endpoint=\"bar\",method=\"POST\"}","value":1}]`, + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"valid","metrics":{"http_requests_total":1,"http_requests_errors_total":3,"http_request_timestamp":123456}}]}],`, addMetrics: func(monitor *monitor.Monitor) { - monitor.Http.RequestCounter.WithLabel("foo", "bar", http.MethodPost).Add(1) + monitor.Http.RequestCounter.WithLabel("foo", "/foo/{bar}", http.MethodGet).Add(1) + monitor.Http.RequestErrorCounter.WithLabel("foo", "/foo/{bar}", http.MethodGet).Add(3) + monitor.Http.LastRequest.WithLabel("foo", "/foo/{bar}", http.MethodGet).Set(float64(123456)) }, }, } diff --git a/api/handler_kafka.go b/api/handler_kafka.go index 94e8426db..57831c7b0 100644 --- a/api/handler_kafka.go +++ b/api/handler_kafka.go @@ -11,72 +11,75 @@ import ( "mokapi/runtime/metrics" "mokapi/runtime/monitor" "net/http" + "slices" "sort" "strconv" "strings" "time" + "github.com/gorilla/mux" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) -type kafkaSummary struct { - service -} - -type cluster struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Contact *contact `json:"contact,omitempty"` - Version string `json:"version,omitempty"` -} - type kafkaInfo struct { - Name string `json:"name"` - Description string `json:"description"` - Version string `json:"version"` - Contact *contact `json:"contact,omitempty"` - Servers []kafkaServer `json:"servers,omitempty"` - Topics []topic `json:"topics,omitempty"` - Groups []group `json:"groups,omitempty"` - Metrics []metrics.Metric `json:"metrics,omitempty"` - Configs []config `json:"configs,omitempty"` - Clients []client `json:"clients,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version"` + Contact *contact `json:"contact,omitempty"` + Servers []kafkaServer `json:"servers,omitempty"` + Topics []kafkaTopicInfo `json:"topics,omitempty"` + Groups []kafkaGroupInfo `json:"groups,omitempty"` + Configs []configInfo `json:"configs,omitempty"` + Clients []kafkaClientInfo `json:"clients,omitempty"` } type kafkaServer struct { Name string `json:"name"` Host string `json:"host"` Protocol string `json:"protocol"` - Title string `json:"title"` - Summary string `json:"summary"` - Description string `json:"description"` + Title string `json:"title,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` Configs map[string]any `json:"configs,omitempty"` Tags []kafkaTag `json:"tags,omitempty"` } type kafkaTag struct { Name string `json:"name"` - Description string `json:"description"` + Description string `json:"description,omitempty"` +} + +type kafkaGroupInfo struct { + Name string `json:"name"` + Generation int `json:"generation"` + State string `json:"state"` + Protocol string `json:"protocol"` + Members int `json:"members"` + Metrics kafkaGroupMetrics `json:"metrics"` } type group struct { - Name string `json:"name"` - Generation int `json:"generation"` - Members []member `json:"members"` - Leader string `json:"leader"` - State string `json:"state"` - AssignmentStrategy string `json:"protocol"` - Topics []string `json:"topics"` + Name string `json:"name"` + Generation int `json:"generation"` + State string `json:"state"` + Protocol string `json:"protocol"` + Members []kafkaMember `json:"members"` + Leader string `json:"leader"` + Topics []string `json:"topics"` + Metrics kafkaGroupMetrics `json:"metrics"` } -type client struct { - ClientId string `json:"clientId"` - Address string `json:"address"` - BrokerAddress string `json:"brokerAddress"` - ClientSoftwareName string `json:"clientSoftwareName"` - ClientSoftwareVersion string `json:"clientSoftwareVersion"` - Groups []clientGroupMember `json:"groups"` +type kafkaClientInfo struct { + ClientId string `json:"clientId"` + Address string `json:"address"` + Software string `json:"software"` +} + +type kafkaClient struct { + kafkaClientInfo + BrokerAddress string `json:"brokerAddress"` + Groups []clientGroupMember `json:"groups"` } type clientGroupMember struct { @@ -84,26 +87,35 @@ type clientGroupMember struct { Group string `json:"group"` } -type member struct { - Name string `json:"name"` - ClientId string `json:"clientId"` - Addr string `json:"addr"` - ClientSoftwareName string `json:"clientSoftwareName"` - ClientSoftwareVersion string `json:"clientSoftwareVersion"` - Heartbeat time.Time `json:"heartbeat"` - Partitions map[string][]int `json:"partitions"` +type kafkaMember struct { + Name string `json:"name"` + ClientId string `json:"clientId"` + Addr string `json:"addr"` + Software string `json:"software"` + Heartbeat time.Time `json:"heartbeat"` + Partitions map[string][]int `json:"partitions"` +} + +type kafkaTopicInfo struct { + Name string `json:"name"` + Summary string `json:"summary,omitempty"` + Tags []kafkaTag `json:"tags,omitempty"` + Metrics kafkaTopicMetric `json:"metrics"` } -type topic struct { +type kafkaTopic struct { Name string `json:"name"` - Description string `json:"description"` - Partitions []partition `json:"partitions"` + Title string `json:"title,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Partitions []kafkaPartition `json:"partitions"` Messages map[string]messageConfig `json:"messages,omitempty"` - Bindings bindings `json:"bindings,omitempty"` + Bindings kafkaBindings `json:"bindings,omitempty"` Tags []kafkaTag `json:"tags,omitempty"` + Groups []kafkaGroupInfo `json:"groups,omitempty"` } -type partition struct { +type kafkaPartition struct { Id int `json:"id"` StartOffset int64 `json:"startOffset"` Offset int64 `json:"offset"` @@ -121,7 +133,7 @@ type messageConfig struct { ContentType string `json:"contentType"` } -type bindings struct { +type kafkaBindings struct { Partitions int `json:"partitions,omitempty"` RetentionBytes int64 `json:"retentionBytes,omitempty"` RetentionMs int64 `json:"retentionMs,omitempty"` @@ -131,34 +143,54 @@ type bindings struct { KeySchemaValidation bool `json:"keySchemaValidation,omitempty"` } -type produceRequest struct { +type kafkaProduceRequest struct { Records []store.Record `json:"records"` } -type ProduceResponse struct { - Offsets []RecordResult `json:"offsets"` +type KafkaProduceResponse struct { + Offsets []KafkaRecordResult `json:"offsets"` } -type RecordResult struct { +type KafkaRecordResult struct { Partition int Offset int64 Error string } -func getKafkaServices(store *runtime.KafkaStore, m *monitor.Monitor) []interface{} { +type kafkaTopicMetric struct { + NumMessages float64 `json:"kafka_messages_total"` + LastMessageTime float64 `json:"kafka_message_timestamp"` +} + +type kafkaGroupMetrics struct { + LastRebalancing float64 `json:"kafka_rebalance_timestamp"` + Topics map[string][]kafkaTopicGroupMetrics `json:"topics,omitempty"` +} + +type kafkaTopicGroupMetrics struct { + Partition int `json:"partition"` + Lags float64 `json:"kafka_consumer_group_lag"` + Commit float64 `json:"kafka_consumer_group_commit"` +} + +func getKafkaServices(store *runtime.KafkaStore, m *monitor.Monitor) []service { list := store.List() - result := make([]interface{}, 0, len(list)) - for _, hs := range list { + result := make([]service, 0, len(list)) + for _, ki := range list { s := service{ - Name: hs.Info.Name, - Description: hs.Info.Description, - Version: hs.Info.Version, + Name: ki.Info.Name, + Description: ki.Info.Description, + Version: ki.Info.Version, Type: ServiceKafka, - Metrics: m.FindAll(metrics.ByNamespace("kafka"), metrics.ByLabel("service", hs.Info.Name)), } - if hs.Info.Contact != nil { - c := hs.Info.Contact + s.Metrics = kafkaTopicMetric{ + NumMessages: m.Kafka.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name))), + LastMessageTime: m.Kafka.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name))), + } + + if ki.Info.Contact != nil { + c := ki.Info.Contact s.Contact = &contact{ Name: c.Name, Url: c.Url, @@ -166,265 +198,60 @@ func getKafkaServices(store *runtime.KafkaStore, m *monitor.Monitor) []interface } } - result = append(result, kafkaSummary{service: s}) + result = append(result, s) } return result } -func (h *handler) handleKafka(w http.ResponseWriter, r *http.Request) { - segments := strings.Split(strings.Trim(r.URL.Path, "/"), "/") - switch { - // /api/services/kafka - case len(segments) == 3: - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, getKafkaClusters(h.app)) - return - // /api/services/kafka/{cluster} - case len(segments) == 4: - name := segments[3] - if s := h.app.Kafka.Get(name); s != nil { - k := getKafka(s) - k.Metrics = h.app.Monitor.FindAll(metrics.ByNamespace("kafka"), metrics.ByLabel("service", name)) - - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, k) - } else { - w.WriteHeader(http.StatusNotFound) - } - return - // /api/services/kafka/{cluster}/topics - case len(segments) == 5 && segments[4] == "topics": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - } else { - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, getTopics(k)) - } - return - // /api/services/kafka/{cluster}/topics/{topic} - case len(segments) == 6 && segments[4] == "topics": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - return - } - topicName := segments[5] - - if r.Method == "GET" { - t := getTopic(k, topicName) - if t == nil { - w.WriteHeader(http.StatusNotFound) - return - } else { - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, t) - return - } - } else if r.Method == "POST" { - records, err := getProduceRecords(r) - if err != nil { - writeError(w, err, http.StatusBadRequest) - return - } - c := store.NewClient(k.Store, h.app.Monitor.Kafka) - ct := media.ParseContentType(r.Header.Get("Content-Type")) - result, err := c.Write(topicName, records, ct) - if err != nil { - if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { - writeError(w, err, http.StatusNotFound) - } else { - writeError(w, err, http.StatusBadRequest) - } - } - res := ProduceResponse{} - for _, rec := range result { - res.Offsets = append(res.Offsets, RecordResult{ - Partition: rec.Partition, - Offset: rec.Offset, - Error: rec.Error, - }) - } - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, res) - return - } - // /api/services/kafka/{cluster}/topics/{topic}/partitions - case len(segments) == 7 && segments[6] == "partitions": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - return - } - topicName := segments[5] - t := k.Store.Topic(topicName) - if t == nil { - w.WriteHeader(http.StatusNotFound) - } else { - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, getPartitions(t)) - } - return - // /api/services/kafka/{cluster}/topics/{topic}/partitions/{id} - case len(segments) == 8 && segments[6] == "partitions": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - return - } - topicName := segments[5] - t := k.Store.Topic(topicName) - if t == nil { - w.WriteHeader(http.StatusNotFound) - return - } - idValue := segments[7] - id, err := strconv.Atoi(idValue) - if err != nil { - writeError(w, fmt.Errorf("error partition ID is not an integer"), http.StatusBadRequest) - return - } - p := t.Partition(id) - if p == nil { - w.WriteHeader(http.StatusNotFound) - return - } - if r.Method == "GET" { - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, newPartition(p)) - } else { - records, err := getProduceRecords(r) - if err != nil { - writeError(w, err, http.StatusBadRequest) - return - } - for _, record := range records { - record.Partition = id - } - c := store.NewClient(k.Store, h.app.Monitor.Kafka) - ct := media.ParseContentType(r.Header.Get("Content-Type")) - result, err := c.Write(topicName, records, ct) - if err != nil { - if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { - writeError(w, err, http.StatusNotFound) - } else { - writeError(w, err, http.StatusBadRequest) - } - } - res := ProduceResponse{} - for _, rec := range result { - res.Offsets = append(res.Offsets, RecordResult{ - Partition: rec.Partition, - Offset: rec.Offset, - Error: rec.Error, - }) - } - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, res) - return - } - return - // /api/services/kafka/{cluster}/topics/{topic}/partitions/{id}/offsets - case len(segments) == 9 && segments[8] == "offsets": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - return - } - topicName := segments[5] - idValue := segments[7] - id, err := strconv.Atoi(idValue) - if err != nil { - writeError(w, fmt.Errorf("error partition ID is not an integer"), http.StatusBadRequest) - return - } - offsetValue := r.URL.Query().Get("offset") - offset := -1 - if offsetValue != "" { - offset, err = strconv.Atoi(offsetValue) - if err != nil { - writeError(w, fmt.Errorf("error offset is not an integer"), http.StatusBadRequest) - return - } - } +func (h *handler) setupKafka() { + r := h.router.PathPrefix("/api/services/kafka").Subrouter() + + r.HandleFunc("", h.getKafkaClusters).Methods(http.MethodGet) + r.HandleFunc("/{cluster}", h.getKafkaInfo).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics", h.getKafkaTopics).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics/{topic}", h.getKafkaTopic).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics/{topic}", h.produceKafkaMessage).Methods(http.MethodPost) + r.HandleFunc("/{cluster}/topics/{topic}/partitions", h.getKafkaPartitions).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics/{topic}/partitions/{partition}", h.getKafkaPartition).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics/{topic}/partitions/{partition}", h.produceKafkaMessage).Methods(http.MethodPost) + r.HandleFunc("/{cluster}/topics/{topic}/partitions/{partition}/offsets", h.getKafkaMessages).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics/{topic}/partitions/{partition}/offsets/{offset}", h.getKafkaMessage).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/groups/{group}", h.getKafkaGroup).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/clients/{client}", h.getKafkaClient).Methods(http.MethodGet) +} - c := store.NewClient(k.Store, h.app.Monitor.Kafka) - ct := media.ParseContentType(r.Header.Get("Accept")) - records, err := c.Read(topicName, id, int64(offset), &ct) - if err != nil { - if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { - writeError(w, err, http.StatusNotFound) - } else { - writeError(w, err, http.StatusInternalServerError) - } - return - } - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, records) - return - // /api/services/kafka/{cluster}/topics/{topic}/partitions/{id}/offsets/0 - case len(segments) == 10 && segments[8] == "offsets": - k := h.app.Kafka.Get(segments[3]) - if k == nil { - w.WriteHeader(http.StatusNotFound) - return - } - topicName := segments[5] - idValue := segments[7] - id, err := strconv.Atoi(idValue) - if err != nil { - writeError(w, fmt.Errorf("error partition ID is not an integer"), http.StatusBadRequest) - return - } - offsetValue := segments[9] - offset, err := strconv.Atoi(offsetValue) - if err != nil { - writeError(w, fmt.Errorf("error offset is not an integer"), http.StatusBadRequest) - return - } +func (h *handler) getKafkaClusters(w http.ResponseWriter, _ *http.Request) { + services := getKafkaServices(h.app.Kafka, h.app.Monitor) + write(w, services) +} - c := store.NewClient(k.Store, h.app.Monitor.Kafka) - ct := media.ParseContentType(r.Header.Get("Accept")) - records, err := c.Read(topicName, id, int64(offset), &ct) - if err != nil { - if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { - writeError(w, err, http.StatusNotFound) - } else { - writeError(w, err, http.StatusBadRequest) - } - return - } - if len(records) == 0 { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, records[0]) +func (h *handler) getKafkaInfo(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) return } - w.WriteHeader(http.StatusBadRequest) -} -func getKafka(info *runtime.KafkaInfo) kafkaInfo { k := kafkaInfo{ - Name: info.Config.Info.Name, - Description: info.Config.Info.Description, - Version: info.Config.Info.Version, - Groups: make([]group, 0), + Name: ki.Config.Info.Name, + Description: ki.Config.Info.Description, + Version: ki.Config.Info.Version, } - if info.Config.Info.Contact != nil { + if ki.Config.Info.Contact != nil { k.Contact = &contact{ - Name: info.Config.Info.Contact.Name, - Url: info.Config.Info.Contact.Url, - Email: info.Config.Info.Contact.Email, + Name: ki.Config.Info.Contact.Name, + Url: ki.Config.Info.Contact.Url, + Email: ki.Config.Info.Contact.Email, } } - for it := info.Servers.Iter(); it.Next(); { + for it := ki.Servers.Iter(); it.Next(); { name := it.Key() s := it.Value() - if s == nil || s.Value == nil { + if s == nil || s.Value == nil || strings.ToLower(s.Value.Protocol) != "kafka" { continue } @@ -453,78 +280,327 @@ func getKafka(info *runtime.KafkaInfo) kafkaInfo { return strings.Compare(k.Servers[i].Name, k.Servers[j].Name) < 0 }) - k.Topics = getTopics(info) + k.Topics = getTopicInfos(ki, h.app.Monitor.Kafka) + k.Groups = getGroupInfos(ki, "", h.app.Monitor.Kafka) + k.Configs = getConfigInfos(ki.Configs()) + + for _, ctx := range ki.Store.Clients() { + c := kafkaClientInfo{ + ClientId: ctx.ClientId, + Address: ctx.Addr, + Software: getSoftware(ctx.ClientSoftwareName, ctx.ClientSoftwareVersion), + } - for _, g := range info.Store.Groups() { - k.Groups = append(k.Groups, newGroup(g)) + k.Clients = append(k.Clients, c) } - sort.Slice(k.Groups, func(i, j int) bool { - return strings.Compare(k.Groups[i].Name, k.Groups[j].Name) < 0 - }) - k.Configs = getConfigs(info.Configs()) + write(w, k) +} - for _, ctx := range info.Store.Clients() { - c := client{ - ClientId: ctx.ClientId, - Address: ctx.Addr, - BrokerAddress: ctx.ServerAddress, - ClientSoftwareName: ctx.ClientSoftwareName, - ClientSoftwareVersion: ctx.ClientSoftwareVersion, - } - for groupName, memberId := range ctx.Member { - c.Groups = append(c.Groups, clientGroupMember{ - MemberId: memberId, - Group: groupName, - }) - } +func (h *handler) getKafkaTopics(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) - k.Clients = append(k.Clients, c) + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + return } - return k + topics := getTopicInfos(ki, h.app.Monitor.Kafka) + write(w, topics) } -func getTopics(info *runtime.KafkaInfo) []topic { - topics := make([]topic, 0, len(info.Config.Channels)) - for name, ch := range info.Config.Channels { +func (h *handler) getKafkaTopic(w http.ResponseWriter, r *http.Request) { + ki, t, ok := h.resolveKafkaTopic(w, r) + if !ok { + return + } + + for n, ch := range ki.Channels { if ch.Value == nil { continue } addr := ch.Value.Address if addr == "" { - addr = name + addr = n + } + if addr == t.Name { + r := newTopic(t, ki, ch.Value, h.app.Monitor.Kafka) + write(w, r) + return } - t := info.Store.Topic(addr) - topics = append(topics, newTopic(t, ch.Value, info.Config)) + } - sort.Slice(topics, func(i, j int) bool { - return strings.Compare(topics[i].Name, topics[j].Name) < 0 + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka topic not found")) +} + +func (h *handler) getKafkaPartitions(w http.ResponseWriter, r *http.Request) { + _, t, ok := h.resolveKafkaTopic(w, r) + if !ok { + return + } + + var partitions []kafkaPartition + for _, p := range t.Partitions { + partitions = append(partitions, newPartition(p)) + } + sort.Slice(partitions, func(i, j int) bool { + return partitions[i].Id < partitions[j].Id }) - return topics + write(w, partitions) +} + +func (h *handler) getKafkaPartition(w http.ResponseWriter, r *http.Request) { + _, _, p, ok := h.resolveKafkaPartition(w, r) + if !ok { + return + } + + write(w, newPartition(p)) +} + +func (h *handler) getKafkaMessages(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + ki, t, p, ok := h.resolveKafkaPartition(w, r) + if !ok { + return + } + + startOffset := -1 + if startOffsetValue, ok := vars["startOffset"]; ok { + var err error + startOffset, err = strconv.Atoi(startOffsetValue) + if err != nil { + writeError(w, fmt.Errorf("error startOffset is not an integer"), http.StatusBadRequest) + return + } + } + + c := store.NewClient(ki.Store, h.app.Monitor.Kafka) + ct := media.ParseContentType(r.Header.Get("Accept")) + records, err := c.Read(t.Name, p.Index, int64(startOffset), &ct) + if err != nil { + if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { + writeError(w, err, http.StatusNotFound) + } else { + writeError(w, err, http.StatusInternalServerError) + } + return + } + write(w, records) } -func getTopic(info *runtime.KafkaInfo, name string) *topic { - for n, ch := range info.Config.Channels { +func (h *handler) getKafkaMessage(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + ki, t, p, ok := h.resolveKafkaPartition(w, r) + if !ok { + return + } + + offset := -1 + offsetValue := vars["offset"] + var err error + offset, err = strconv.Atoi(offsetValue) + if err != nil { + writeError(w, fmt.Errorf("error offset is not an integer"), http.StatusBadRequest) + return + } + + c := store.NewClient(ki.Store, h.app.Monitor.Kafka) + ct := media.ParseContentType(r.Header.Get("Accept")) + record, err := c.Offset(t.Name, p.Index, int64(offset), &ct) + if err != nil { + if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) || errors.Is(err, store.OffsetOutOfRange) { + writeError(w, err, http.StatusNotFound) + } else { + writeError(w, err, http.StatusInternalServerError) + } + return + } + write(w, record) +} + +func (h *handler) getKafkaGroup(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka cluster not found")) + return + } + + g, ok := ki.Group(vars["group"]) + if !ok { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka group not found")) + return + } + + grp := group{ + Name: g.Name, + State: g.State.String(), + Metrics: getGroupMetrics(g.Name, ki, h.app.Monitor.Kafka), + } + + if g.Generation != nil { + grp.Generation = g.Generation.Id + grp.Leader = g.Generation.LeaderId + grp.Protocol = g.Generation.Protocol + + for id, m := range g.Generation.Members { + grp.Members = append(grp.Members, kafkaMember{ + Name: id, + ClientId: m.Client.ClientId, + Addr: m.Client.Addr, + Software: getSoftware(m.Client.ClientSoftwareName, m.Client.ClientSoftwareVersion), + Heartbeat: m.Client.Heartbeat, + Partitions: m.Partitions, + }) + } + sort.Slice(grp.Members, func(i, j int) bool { + return strings.Compare(grp.Members[i].Name, grp.Members[j].Name) < 0 + }) + } else { + grp.Generation = -1 + } + for topicName := range g.Commits { + grp.Topics = append(grp.Topics, topicName) + } + sort.Slice(grp.Topics, func(i, j int) bool { + return strings.Compare(grp.Topics[i], grp.Topics[j]) < 0 + }) + + write(w, grp) +} + +func (h *handler) getKafkaClient(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka cluster not found")) + return + } + + c, ok := ki.Client(vars["client"]) + if !ok { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka client not found")) + return + } + + client := kafkaClient{ + kafkaClientInfo: kafkaClientInfo{ + ClientId: c.ClientId, + Address: c.Addr, + Software: getSoftware(c.ClientSoftwareName, c.ClientSoftwareVersion), + }, + BrokerAddress: c.ServerAddress, + } + + for groupName, memberId := range c.Member { + client.Groups = append(client.Groups, clientGroupMember{ + MemberId: memberId, + Group: groupName, + }) + } + + write(w, client) +} + +func (h *handler) produceKafkaMessage(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka cluster not found")) + return + } + + if t := ki.Topic(vars["topic"]); t == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("kafka topic not found")) + return + } + + records, err := getProduceRecords(r) + if err != nil { + writeError(w, err, http.StatusBadRequest) + return + } + + if idValue, ok := vars["partition"]; ok { + var id int + id, err = strconv.Atoi(idValue) + if err != nil { + writeError(w, fmt.Errorf("error partition ID is not an integer"), http.StatusBadRequest) + return + } + for _, record := range records { + record.Partition = id + } + } + + c := store.NewClient(ki.Store, h.app.Monitor.Kafka) + ct := media.ParseContentType(r.Header.Get("Content-Type")) + result, err := c.Write(vars["topic"], records, ct) + if err != nil { + if errors.Is(err, store.TopicNotFound) || errors.Is(err, store.PartitionNotFound) { + writeError(w, err, http.StatusNotFound) + } else { + writeError(w, err, http.StatusBadRequest) + } + } + res := KafkaProduceResponse{} + for _, rec := range result { + res.Offsets = append(res.Offsets, KafkaRecordResult{ + Partition: rec.Partition, + Offset: rec.Offset, + Error: rec.Error, + }) + } + write(w, res) +} + +func getTopicInfos(ki *runtime.KafkaInfo, m *monitor.Kafka) []kafkaTopicInfo { + topics := make([]kafkaTopicInfo, 0, len(ki.Config.Channels)) + for name, ch := range ki.Config.Channels { if ch.Value == nil { continue } + if !ch.Value.IsChannelAvailable("kafka") { + continue + } addr := ch.Value.Address if addr == "" { - addr = n + addr = name } - if addr == name { - t := info.Store.Topic(addr) - r := newTopic(t, ch.Value, info.Config) - return &r + + ti := kafkaTopicInfo{ + Name: addr, + Summary: ch.Value.Summary, + Tags: getKafkaTags(ch.Value), } + ti.Metrics = kafkaTopicMetric{ + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name))), + } + + topics = append(topics, ti) } - return nil + sort.Slice(topics, func(i, j int) bool { + return strings.Compare(topics[i].Name, topics[j].Name) < 0 + }) + return topics } -func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topic { - var partitions []partition +func newTopic(t *store.Topic, ki *runtime.KafkaInfo, ch *asyncapi3.Channel, m *monitor.Kafka) kafkaTopic { + var partitions []kafkaPartition for _, p := range t.Partitions { partitions = append(partitions, newPartition(p)) } @@ -532,11 +608,13 @@ func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topi return partitions[i].Id < partitions[j].Id }) - result := topic{ + result := kafkaTopic{ Name: t.Name, + Title: ch.Title, + Summary: ch.Summary, Description: ch.Description, Partitions: partitions, - Bindings: bindings{ + Bindings: kafkaBindings{ Partitions: t.Config.Bindings.Kafka.Partitions, RetentionBytes: t.Config.Bindings.Kafka.RetentionBytes, RetentionMs: t.Config.Bindings.Kafka.RetentionMs, @@ -545,8 +623,125 @@ func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topi ValueSchemaValidation: t.Config.Bindings.Kafka.ValueSchemaValidation, KeySchemaValidation: t.Config.Bindings.Kafka.KeySchemaValidation, }, + Tags: getKafkaTags(ch), + Groups: getGroupInfos(ki, t.Name, m), + } + + result.Messages = getMessageConfigs(ch, ki.Config) + + return result +} + +func getGroupInfos(ki *runtime.KafkaInfo, topic string, m *monitor.Kafka) []kafkaGroupInfo { + groups := ki.Groups() + var result []kafkaGroupInfo +Groups: + for _, g := range groups { + if topic != "" { + for topicName := range g.Commits { + if topicName != topic { + continue Groups + } + } + } + + gi := kafkaGroupInfo{ + Name: g.Name, + Metrics: getGroupMetrics(g.Name, ki, m), + } + if g.Generation != nil { + gi.Generation = g.Generation.Id + gi.State = g.State.String() + gi.Protocol = g.Generation.Protocol + gi.Members = len(g.Generation.Members) + } + result = append(result, gi) + } + slices.SortFunc(result, func(a, b kafkaGroupInfo) int { + return strings.Compare(a.Name, b.Name) + }) + return result +} + +func getGroupMetrics(groupName string, ki *runtime.KafkaInfo, m *monitor.Kafka) kafkaGroupMetrics { + result := kafkaGroupMetrics{ + LastRebalancing: m.LastRebalancing.Value(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name))), + } + lags := m.Lags.FindAll( + metrics.NewQuery( + metrics.ByLabel("service", ki.Info.Name), + metrics.ByLabel("group", groupName), + ), + ) + if len(lags) > 0 { + result.Topics = map[string][]kafkaTopicGroupMetrics{} + for _, lag := range lags { + topic := lag.Info().GetLabel("topic") + if _, ok := result.Topics[topic]; !ok { + result.Topics[topic] = []kafkaTopicGroupMetrics{} + } + partition := lag.Info().GetLabel("partition") + id, _ := strconv.Atoi(partition) + mCommit, _ := m.Commits.FindOne(metrics.NewQuery( + metrics.ByLabel("service", ki.Info.Name), + metrics.ByLabel("group", groupName), + metrics.ByLabel("topic", topic), + metrics.ByLabel("partition", partition), + )) + commit := float64(0) + if mCommit != nil { + commit = mCommit.Value() + } + + result.Topics[topic] = append(result.Topics[topic], kafkaTopicGroupMetrics{ + Partition: id, + Lags: lag.Value(), + Commit: commit, + }) + } + } + return result +} + +func newPartition(p *store.Partition) kafkaPartition { + return kafkaPartition{ + Id: p.Index, + StartOffset: p.StartOffset(), + Offset: p.Offset(), + Segments: len(p.Segments), } +} +func getProduceRecords(r *http.Request) ([]store.Record, error) { + var pr kafkaProduceRequest + b, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("error reading body") + } + err = json.Unmarshal(b, &pr) + if err != nil { + return nil, fmt.Errorf("error parsing body") + } + return pr.Records, nil +} + +func (r *KafkaRecordResult) MarshalJSON() ([]byte, error) { + aux := &struct { + Partition int `json:"partition"` + Offset int64 `json:"offset"` + Error *string `json:"error,omitempty"` + }{ + Partition: r.Partition, + Offset: r.Offset, + } + if r.Error != "" { + aux.Error = &r.Error + } + return json.Marshal(aux) +} + +func getMessageConfigs(ch *asyncapi3.Channel, cfg *asyncapi3.Config) map[string]messageConfig { + var result map[string]messageConfig for messageId, ref := range ch.Messages { if ref.Value == nil { continue @@ -571,7 +766,7 @@ func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topi } s, err := msg.Payload.GetSchema() if err != nil { - log.Errorf("failed to get schema for message in topic '%s': %v", t.Name, err) + log.Errorf("failed to get schema for message in topic '%s': %v", ch.Name, err) } m.Payload = &schemaInfo{Schema: s, Format: format} } @@ -582,7 +777,7 @@ func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topi } s, err := msg.Headers.GetSchema() if err != nil { - log.Errorf("failed to get schema for headers in topic '%s': %v", t.Name, err) + log.Errorf("failed to get schema for headers in topic '%s': %v", ch.Name, err) } m.Header = &schemaInfo{Schema: s, Format: format} } @@ -594,131 +789,80 @@ func newTopic(t *store.Topic, ch *asyncapi3.Channel, cfg *asyncapi3.Config) topi if msg.Bindings.Kafka.Key != nil { s, err := msg.Bindings.Kafka.Key.GetSchema() if err != nil { - log.Errorf("failed to get schema for key in topic '%s': %v", t.Name, err) + log.Errorf("failed to get schema for key in topic '%s': %v", ch.Name, err) } m.Key = &schemaInfo{Schema: s} } - if result.Messages == nil { - result.Messages = map[string]messageConfig{} + if result == nil { + result = map[string]messageConfig{} } - result.Messages[messageId] = m + result[messageId] = m } + return result +} +func getKafkaTags(ch *asyncapi3.Channel) []kafkaTag { + var result []kafkaTag for _, tRef := range ch.Tags { if tRef.Value == nil { continue } - result.Tags = append(result.Tags, kafkaTag{ + result = append(result, kafkaTag{ Name: tRef.Value.Name, Description: tRef.Value.Description, }) } - return result } -func getPartitions(t *store.Topic) []partition { - var partitions []partition - for _, p := range t.Partitions { - partitions = append(partitions, newPartition(p)) - } - sort.Slice(partitions, func(i, j int) bool { - return partitions[i].Id < partitions[j].Id - }) - return partitions -} +func (h *handler) resolveKafkaTopic(w http.ResponseWriter, r *http.Request) (*runtime.KafkaInfo, *store.Topic, bool) { + vars := mux.Vars(r) -func newGroup(g *store.Group) group { - grp := group{ - Name: g.Name, - State: g.State.String(), + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + http.Error(w, "kafka cluster not found", http.StatusNotFound) + return nil, nil, false } - if g.Generation != nil { - grp.Generation = g.Generation.Id - grp.Leader = g.Generation.LeaderId - grp.AssignmentStrategy = g.Generation.Protocol - for id, m := range g.Generation.Members { - grp.Members = append(grp.Members, member{ - Name: id, - ClientId: m.Client.ClientId, - Addr: m.Client.Addr, - ClientSoftwareName: m.Client.ClientSoftwareName, - ClientSoftwareVersion: m.Client.ClientSoftwareVersion, - Heartbeat: m.Client.Heartbeat, - Partitions: m.Partitions, - }) - } - sort.Slice(grp.Members, func(i, j int) bool { - return strings.Compare(grp.Members[i].Name, grp.Members[j].Name) < 0 - }) - } else { - grp.Generation = -1 - } - for topicName := range g.Commits { - grp.Topics = append(grp.Topics, topicName) + t := ki.Topic(vars["topic"]) + if t == nil { + http.Error(w, "kafka topic not found", http.StatusNotFound) + return nil, nil, false } - sort.Slice(grp.Topics, func(i, j int) bool { - return strings.Compare(grp.Topics[i], grp.Topics[j]) < 0 - }) - return grp + return ki, t, true } -func newPartition(p *store.Partition) partition { - return partition{ - Id: p.Index, - StartOffset: p.StartOffset(), - Offset: p.Offset(), - Segments: len(p.Segments), - } -} +func (h *handler) resolveKafkaPartition(w http.ResponseWriter, r *http.Request) (*runtime.KafkaInfo, *store.Topic, *store.Partition, bool) { + vars := mux.Vars(r) -func getKafkaClusters(app *runtime.App) []cluster { - var clusters []cluster - for _, k := range app.Kafka.List() { - var c *contact - if k.Info.Contact != nil { - c = &contact{ - Name: k.Info.Contact.Name, - Url: k.Info.Contact.Url, - Email: k.Info.Contact.Email, - } - } - clusters = append(clusters, cluster{ - Name: k.Info.Name, - Description: k.Info.Description, - Contact: c, - Version: k.Info.Version, - }) + ki, t, ok := h.resolveKafkaTopic(w, r) + if !ok { + return nil, nil, nil, false } - return clusters -} -func getProduceRecords(r *http.Request) ([]store.Record, error) { - var pr produceRequest - b, err := io.ReadAll(r.Body) + id, err := strconv.Atoi(vars["partition"]) if err != nil { - return nil, fmt.Errorf("error reading body") + writeError(w, fmt.Errorf("partition ID is not an integer"), http.StatusBadRequest) + return nil, nil, nil, false } - err = json.Unmarshal(b, &pr) - if err != nil { - return nil, fmt.Errorf("error parsing body") + + p := t.Partition(id) + if p == nil { + http.Error(w, "kafka partition not found", http.StatusNotFound) + return nil, nil, nil, false } - return pr.Records, nil + + return ki, t, p, true } -func (r *RecordResult) MarshalJSON() ([]byte, error) { - aux := &struct { - Partition int `json:"partition"` - Offset int64 `json:"offset"` - Error *string `json:"error,omitempty"` - }{ - Partition: r.Partition, - Offset: r.Offset, - } - if r.Error != "" { - aux.Error = &r.Error +func getSoftware(name, version string) string { + software := "" + if name != "" { + software = name + if version != "" { + software += " " + version + } } - return json.Marshal(aux) + return software } diff --git a/api/handler_kafka_test.go b/api/handler_kafka_test.go index d1be7baf0..e09a3ccbf 100644 --- a/api/handler_kafka_test.go +++ b/api/handler_kafka_test.go @@ -52,7 +52,7 @@ func TestHandler_Kafka(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"kafka"}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, }, { name: "get kafka services", @@ -65,7 +65,7 @@ func TestHandler_Kafka(t *testing.T) { asyncapi3test.WithContact("mokapi", "https://mokapi.io", "info@mokapi.io"), ), }, enginetest.NewEngine()) - app.AddHttp(&dynamic.Config{ + app.Http.Add(&dynamic.Config{ Info: dynamic.ConfigInfo{Url: try.MustUrl("http.yaml")}, Data: openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "bar", "1.0"), @@ -74,7 +74,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka", - responseBody: `[{"name":"foo","description":"bar","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0"}]`, + responseBody: `[{"name":"foo","description":"bar","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, }, { name: "get kafka services with contact", @@ -86,10 +86,10 @@ func TestHandler_Kafka(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"test","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"version":"1.0","type":"kafka"}]`, + responseBody: `[{"name":"test","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, }, { - name: "get specific", + name: "get cluster info", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) cfg := &dynamic.Config{ @@ -104,10 +104,10 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"mokapi","host":":9092","protocol":"kafka","title":"Mokapi Default Broker","summary":"Automatically added broker because no servers are defined in the AsyncAPI spec","description":""}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"mokapi","host":":9092","protocol":"kafka","title":"Mokapi Default Broker","summary":"Automatically added broker because no servers are defined in the AsyncAPI spec"}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, }, { - name: "get specific with contact", + name: "get cluster with contact", app: func() *runtime.App { return runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", &runtime.KafkaInfo{ Config: asyncapi3test.NewConfig( @@ -121,21 +121,22 @@ func TestHandler_Kafka(t *testing.T) { responseBody: `{"name":"foo","description":"bar","version":"1.0","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"}}`, }, { - name: "get specific with server", + name: "get cluster with server", app: func() *runtime.App { return runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", &runtime.KafkaInfo{ Config: asyncapi3test.NewConfig( asyncapi3test.WithInfo("foo", "bar", "1.0"), asyncapi3test.WithServer("foo", "kafka", "foo.bar", asyncapi3test.WithServerDescription("bar")), + asyncapi3test.WithServer("bar", "mqtt", "foo.bar", asyncapi3test.WithServerDescription("bar")), ), Store: &store.Store{}, })) }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","title":"","summary":"","description":"bar"}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","description":"bar"}]}`, }, { - name: "server with tags", + name: "get cluster with server and tags", app: func() *runtime.App { return runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", &runtime.KafkaInfo{ Config: asyncapi3test.NewConfig( @@ -151,20 +152,24 @@ func TestHandler_Kafka(t *testing.T) { })) }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","title":"","summary":"","description":"bar","tags":[{"name":"env:test","description":"This environment is for running internal tests"}]}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","description":"bar","tags":[{"name":"env:test","description":"This environment is for running internal tests"}]}]}`, }, { - name: "get specific with topic", + name: "get cluster info with one Kafka topic and one MQTT topic", app: func() *runtime.App { c := asyncapi3test.NewConfig( asyncapi3test.WithInfo("foo", "bar", "1.0"), asyncapi3test.WithChannel("foo", - asyncapi3test.WithChannelDescription("bar"), + asyncapi3test.WithChannelSummary("bar"), asyncapi3test.WithMessage("foo", asyncapi3test.WithPayload(schematest.New("string")), asyncapi3test.WithContentType("application/json"), ), ), + asyncapi3test.WithServer("bar", "mqtt", "foo.bar"), + asyncapi3test.WithChannel("bar", + asyncapi3test.AssignToServer("#/servers/bar"), + ), ) s := store.New(c, enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewKafka()) @@ -174,10 +179,10 @@ func TestHandler_Kafka(t *testing.T) { })) }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","summary":"bar","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]}`, }, { - name: "get specific with topic with tag", + name: "get cluster info with topic and tag", app: func() *runtime.App { c := asyncapi3test.NewConfig( asyncapi3test.WithInfo("foo", "bar", "1.0"), @@ -193,10 +198,10 @@ func TestHandler_Kafka(t *testing.T) { })) }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","description":"","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"bindings":{"partitions":1,"valueSchemaValidation":true},"tags":[{"name":"env:test","description":"bar"}]}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","tags":[{"name":"env:test","description":"bar"}],"metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]}`, }, { - name: "get specific with topic and multi schema format", + name: "get topic and multi schema format", app: func() *runtime.App { c := asyncapi3test.NewConfig( asyncapi3test.WithInfo("foo", "bar", "1.0"), @@ -215,11 +220,11 @@ func TestHandler_Kafka(t *testing.T) { Store: s, })) }, - requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}]}`, + requestUrl: "http://foo.api/api/services/kafka/foo/topics/foo", + responseBody: `{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}`, }, { - name: "get specific with group", + name: "get cluster info with group", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) app.Kafka.Set("foo", getKafkaInfoWithGroup(asyncapi3test.NewConfig( @@ -241,10 +246,35 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","title":"","summary":"","description":""}],"groups":[{"name":"foo","generation":3,"members":null,"leader":"","state":"PreparingRebalance","protocol":"range","topics":null}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka"}],"groups":[{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":0,"metrics":{"kafka_rebalance_timestamp":0}}]}`, + }, + { + name: "get group", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + app.Kafka.Set("foo", getKafkaInfoWithGroup(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "kafka", "foo.bar"), + ), + &store.Group{ + Name: "foo", + State: store.PreparingRebalance, + Generation: &store.Generation{ + Id: 3, + Protocol: "range", + LeaderId: "", + RebalanceTimeoutMs: 0, + }, + Commits: nil, + }, + )) + return app + }, + requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", + responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, }, { - name: "get specific with group no generation", + name: "get group without generation", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) app.Kafka.Set("foo", getKafkaInfoWithGroup(asyncapi3test.NewConfig( @@ -259,8 +289,8 @@ func TestHandler_Kafka(t *testing.T) { )) return app }, - requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","title":"","summary":"","description":""}],"groups":[{"name":"foo","generation":-1,"members":null,"leader":"","state":"PreparingRebalance","protocol":"","topics":null}]}`, + requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", + responseBody: `{"name":"foo","generation":-1,"state":"PreparingRebalance","protocol":"","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, }, { name: "get specific with group containing members", @@ -315,11 +345,11 @@ func TestHandler_Kafka(t *testing.T) { return app }, - requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka","title":"","summary":"","description":""}],"groups":[{"name":"foo","generation":3,"members":[{"name":"m1","clientId":"client1","addr":"192.168.0.100","clientSoftwareName":"mokapi","clientSoftwareVersion":"1.0","heartbeat":"2024-04-22T15:04:05+07:00","partitions":{"topic":[1,2,5]}},{"name":"m2","clientId":"client2","addr":"192.168.0.200","clientSoftwareName":"mokapi","clientSoftwareVersion":"1.0","heartbeat":"2024-04-22T15:04:10+07:00","partitions":{"topic":[3,4,6]}}],"leader":"m1","state":"PreparingRebalance","protocol":"range","topics":null}]}`, + requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", + responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":[{"name":"m1","clientId":"client1","addr":"192.168.0.100","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:05+07:00","partitions":{"topic":[1,2,5]}},{"name":"m2","clientId":"client2","addr":"192.168.0.200","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:10+07:00","partitions":{"topic":[3,4,6]}}],"leader":"m1","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, }, { - name: "get specific with topic and openapi schema", + name: "get topic with openapi schema", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) app.Kafka.Set("foo", getKafkaInfo(asyncapi3test.NewConfig( @@ -334,8 +364,8 @@ func TestHandler_Kafka(t *testing.T) { ))) return app }, - requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}]}`, + requestUrl: "http://foo.api/api/services/kafka/foo/topics/foo", + responseBody: `{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}`, }, } @@ -417,7 +447,7 @@ func TestHandler_KafkaAPI(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[{"name":"topic-1","description":"foobar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":null,"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true}}]`), + try.HasBody(`[{"name":"topic-1","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`), ) }, }, @@ -607,10 +637,10 @@ func TestHandler_KafkaAPI(t *testing.T) { h, try.HasStatusCode(http.StatusOK), try.AssertBody(func(t *testing.T, body string) { - var data api.ProduceResponse + var data api.KafkaProduceResponse _ = json.Unmarshal([]byte(body), &data) - require.Equal(t, api.ProduceResponse{ - Offsets: []api.RecordResult{ + require.Equal(t, api.KafkaProduceResponse{ + Offsets: []api.KafkaRecordResult{ { Partition: -1, Offset: -1, @@ -892,10 +922,10 @@ func TestHandler_KafkaAPI(t *testing.T) { h, try.HasStatusCode(http.StatusOK), try.AssertBody(func(t *testing.T, body string) { - var data api.ProduceResponse + var data api.KafkaProduceResponse _ = json.Unmarshal([]byte(body), &data) - require.Equal(t, api.ProduceResponse{ - Offsets: []api.RecordResult{ + require.Equal(t, api.KafkaProduceResponse{ + Offsets: []api.KafkaRecordResult{ { Partition: -1, Offset: -1, @@ -1058,18 +1088,55 @@ func TestHandler_Kafka_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", getKafkaInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","version":"1.0","type":"kafka","metrics":[{"name":"kafka_messages_total{service=\"foo\",topic=\"topic\"}","value":1}]}]`, + responseBody: `[{"name":"foo","version":"1.0","type":"kafka","metrics":{"kafka_messages_total":1,"kafka_message_timestamp":12345678}}]`, addMetrics: func(monitor *monitor.Monitor) { monitor.Kafka.Messages.WithLabel("foo", "topic").Add(1) + monitor.Kafka.LastMessage.WithLabel("foo", "topic").Set(12345678) }, }, { - name: "specific with metric", - app: runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", getKafkaInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), + name: "cluster with metric", + app: runtimetest.NewApp( + runtimetest.WithKafkaInfo("foo", getKafkaInfo( + asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"), + asyncapi3test.WithChannel("foo"), + ), + ))), requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"","version":"1.0","metrics":[{"name":"kafka_messages_total{service=\"foo\",topic=\"topic\"}","value":1}]}`, + responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"foo","metrics":{"kafka_messages_total":1,"kafka_message_timestamp":12345678}}]}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Kafka.Messages.WithLabel("foo", "topic").Add(1) + monitor.Kafka.LastMessage.WithLabel("foo", "topic").Set(12345678) + }, + }, + { + name: "group with metric", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + app.Kafka.Set("foo", getKafkaInfoWithGroup(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "kafka", "foo.bar"), + ), + &store.Group{ + Name: "group-1", + State: store.PreparingRebalance, + Generation: &store.Generation{ + Id: 3, + Protocol: "range", + LeaderId: "", + RebalanceTimeoutMs: 0, + }, + Commits: nil, + }, + )) + return app + }(), + requestUrl: "http://foo.api/api/services/kafka/foo/groups/group-1", + responseBody: `{"name":"group-1","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":12345678,"topics":{"topic-foo":[{"partition":0,"kafka_consumer_group_lag":123,"kafka_consumer_group_commit":11}]}}}`, + addMetrics: func(monitor *monitor.Monitor) { + monitor.Kafka.Lags.WithLabel("foo", "group-1", "topic-foo", "0").Set(123) + monitor.Kafka.Commits.WithLabel("foo", "group-1", "topic-foo", "0").Set(11) + monitor.Kafka.LastRebalancing.WithLabel("foo", "group-1").Set(12345678) }, }, } diff --git a/api/handler_ldap.go b/api/handler_ldap.go index dc16267c1..c57dcb1b1 100644 --- a/api/handler_ldap.go +++ b/api/handler_ldap.go @@ -8,10 +8,6 @@ import ( "strings" ) -type ldapSummary struct { - service -} - type ldapInfo struct { Name string `json:"name"` Description string `json:"description,omitempty"` @@ -20,22 +16,28 @@ type ldapInfo struct { Configs []config `json:"configs,omitempty"` } -func getLdapServices(store *runtime.LdapStore, m *monitor.Monitor) []interface{} { +type ldapMetrics struct { + NumRequests float64 `json:"ldap_requests_total"` + LastRequest float64 `json:"ldap_request_timestamp"` +} + +func getLdapServices(store *runtime.LdapStore, m *monitor.Monitor) []service { list := store.List() - result := make([]interface{}, 0, len(list)) - for _, hs := range list { + result := make([]service, 0, len(list)) + for _, li := range list { s := service{ - Name: hs.Info.Name, - Description: hs.Info.Description, - Version: hs.Info.Version, + Name: li.Info.Name, + Description: li.Info.Description, + Version: li.Info.Version, Type: ServiceLdap, } - if m != nil { - s.Metrics = m.FindAll(metrics.ByNamespace("ldap"), metrics.ByLabel("service", hs.Info.Name)) + s.Metrics = ldapMetrics{ + NumRequests: m.Ldap.RequestCounter.Sum(metrics.NewQuery(metrics.ByLabel("service", li.Info.Name))), + LastRequest: m.Ldap.LastRequest.Max(metrics.NewQuery(metrics.ByLabel("service", li.Info.Name))), } - result = append(result, &ldapSummary{service: s}) + result = append(result, s) } return result } diff --git a/api/handler_ldap_test.go b/api/handler_ldap_test.go index 3dc3b9049..df69dedb2 100644 --- a/api/handler_ldap_test.go +++ b/api/handler_ldap_test.go @@ -40,7 +40,7 @@ func TestHandler_Ldap(t *testing.T) { }, requestUrl: "http://foo.api/api/services", contentType: "application/json", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"ldap"}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"ldap","metrics":{"ldap_requests_total":0,"ldap_request_timestamp":0}}]`, }, { name: "get ldap service", diff --git a/api/handler_mail.go b/api/handler_mail.go index b80fa0a46..46915b98e 100644 --- a/api/handler_mail.go +++ b/api/handler_mail.go @@ -18,10 +18,6 @@ import ( log "github.com/sirupsen/logrus" ) -type mailSummary struct { - service -} - type mailInfo struct { Name string `json:"name"` Description string `json:"description,omitempty"` @@ -120,29 +116,35 @@ type attachment struct { ContentId string `json:"contentId,omitempty"` } -func getMailServices(store *runtime.MailStore, m *monitor.Monitor) []interface{} { +type mailMetrics struct { + NumMails float64 `json:"mail_mails_total"` + LastMail float64 `json:"mail_mail_timestamp"` +} + +func getMailServices(store *runtime.MailStore, m *monitor.Monitor) []service { list := store.List() - result := make([]interface{}, 0, len(list)) - for _, hs := range list { + result := make([]service, 0, len(list)) + for _, mi := range list { s := service{ - Name: hs.Info.Name, - Description: hs.Info.Description, + Name: mi.Info.Name, + Description: mi.Info.Description, Type: ServiceMail, - Version: hs.Info.Version, + Version: mi.Info.Version, } - if hs.Info.Contact != nil { + if mi.Info.Contact != nil { s.Contact = &contact{ - Name: hs.Info.Contact.Name, - Url: hs.Info.Contact.Url, - Email: hs.Info.Contact.Email, + Name: mi.Info.Contact.Name, + Url: mi.Info.Contact.Url, + Email: mi.Info.Contact.Email, } } - if m != nil { - s.Metrics = m.FindAll(metrics.ByNamespace("mail"), metrics.ByLabel("service", hs.Info.Name)) + s.Metrics = mailMetrics{ + NumMails: m.Mail.Mails.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + LastMail: m.Mail.LastMail.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), } - result = append(result, &mailSummary{service: s}) + result = append(result, s) } return result } diff --git a/api/handler_mail_test.go b/api/handler_mail_test.go index 0edfc85ce..13bf3c1bc 100644 --- a/api/handler_mail_test.go +++ b/api/handler_mail_test.go @@ -43,7 +43,7 @@ func TestHandler_Smtp(t *testing.T) { }, requestUrl: "http://foo.api/api/services", contentType: "application/json", - responseBody: `[{"name":"foo","description":"bar","version":"2.1","type":"mail"}]`, + responseBody: `[{"name":"foo","description":"bar","version":"2.1","type":"mail","metrics":{"mail_mails_total":0,"mail_mail_timestamp":0}}]`, }, { name: "get /api/services/mail", diff --git a/api/handler_mqtt.go b/api/handler_mqtt.go new file mode 100644 index 000000000..c912b6f11 --- /dev/null +++ b/api/handler_mqtt.go @@ -0,0 +1,253 @@ +package api + +import ( + "mokapi/providers/asyncapi3" + "mokapi/runtime" + "mokapi/runtime/metrics" + "mokapi/runtime/monitor" + "net/http" + "slices" + "sort" + "strings" + + "github.com/gorilla/mux" +) + +type mqttInfo struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version"` + Contact *contact `json:"contact,omitempty"` + Servers []mqttServer `json:"servers,omitempty"` + Topics []mqttTopic `json:"topics,omitempty"` + Configs []config `json:"configs,omitempty"` + Clients []mqttClient `json:"clients,omitempty"` +} + +type mqttServer struct { + Name string `json:"name"` + Host string `json:"host"` + Protocol string `json:"protocol"` + Title string `json:"title"` + Summary string `json:"summary"` + Description string `json:"description"` + Configs map[string]any `json:"configs,omitempty"` + Tags []kafkaTag `json:"tags,omitempty"` +} + +type mqttTopic struct { + Name string `json:"name"` + Title string `json:"title,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Messages map[string]messageConfig `json:"messages,omitempty"` + Tags []kafkaTag `json:"tags,omitempty"` + Instances []mqttTopicInstance `json:"instances,omitempty"` + Metrics mqttTopicMetrics `json:"metrics,omitempty"` +} + +type mqttTopicInstance struct { + Name string `json:"name"` + Parameters map[string]string `json:"parameters,omitempty"` +} + +type mqttClient struct { + ClientId string `json:"clientId"` + Address string `json:"address"` + BrokerAddress string `json:"brokerAddress"` + ProtocolVersion byte `json:"protocolVersion"` +} + +type mqttTopicMetrics struct { + NumMessages float64 `json:"mqtt_messages_total"` + LastMessageTime float64 `json:"mqtt_message_timestamp"` +} + +func getMqttServices(store *runtime.MqttStore, m *monitor.Monitor) []service { + list := store.List() + result := make([]service, 0, len(list)) + for _, mi := range list { + s := service{ + Name: mi.Info.Name, + Description: mi.Info.Description, + Version: mi.Info.Version, + Type: ServiceMqtt, + } + + if mi.Info.Contact != nil { + c := mi.Info.Contact + s.Contact = &contact{ + Name: c.Name, + Url: c.Url, + Email: c.Email, + } + } + + s.Metrics = mqttTopicMetrics{ + NumMessages: m.Mqtt.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + LastMessageTime: m.Mqtt.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + } + + result = append(result, s) + } + return result +} + +func (h *handler) setupMqtt() { + r := h.router.PathPrefix("/api/services/mqtt").Subrouter() + + r.HandleFunc("", h.getMqttClusters).Methods(http.MethodGet) + r.HandleFunc("/{cluster}", h.getMqttInfo).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/topics", h.getMqttTopics).Methods(http.MethodGet) +} + +func (h *handler) getMqttClusters(w http.ResponseWriter, _ *http.Request) { + services := getMqttServices(h.app.Mqtt, h.app.Monitor) + write(w, services) +} + +func (h *handler) getMqttInfo(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + mi := h.app.Mqtt.Get(vars["cluster"]) + if mi == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + m := mqttInfo{ + Name: mi.Config.Info.Name, + Description: mi.Config.Info.Description, + Version: mi.Config.Info.Version, + } + + if mi.Config.Info.Contact != nil { + m.Contact = &contact{ + Name: mi.Config.Info.Contact.Name, + Url: mi.Config.Info.Contact.Url, + Email: mi.Config.Info.Contact.Email, + } + } + + for it := mi.Servers.Iter(); it.Next(); { + name := it.Key() + s := it.Value() + if s == nil || s.Value == nil || strings.ToLower(s.Value.Protocol) != "mqtt" { + continue + } + + ms := mqttServer{ + Name: name, + Host: s.Value.Host, + Title: s.Value.Title, + Summary: s.Value.Summary, + Description: s.Value.Description, + Protocol: s.Value.Protocol, + } + + for _, r := range s.Value.Tags { + if r.Value == nil { + continue + } + t := r.Value + ms.Tags = append(ms.Tags, kafkaTag{ + Name: t.Name, + Description: t.Description, + }) + } + m.Servers = append(m.Servers, ms) + } + sort.Slice(m.Servers, func(i, j int) bool { + return strings.Compare(m.Servers[i].Name, m.Servers[j].Name) < 0 + }) + + m.Topics = getMqttTopics(mi, h.app.Monitor.Mqtt) + + for _, client := range mi.Store.Clients() { + c := mqttClient{ + ClientId: client.Id, + Address: client.Addr(), + BrokerAddress: client.ServerAddress(), + ProtocolVersion: client.ProtocolVersion(), + } + m.Clients = append(m.Clients, c) + } + + m.Configs = getConfigs(mi.Configs()) + + write(w, m) +} + +func (h *handler) getMqttTopics(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Mqtt.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + write(w, getMqttTopics(ki, h.app.Monitor.Mqtt)) +} + +func getMqttTopics(mi *runtime.MqttInfo, m *monitor.Mqtt) []mqttTopic { + topics := make([]mqttTopic, 0, len(mi.Config.Channels)) + for name, ch := range mi.Config.Channels { + if ch.Value == nil { + continue + } + if !ch.Value.IsChannelAvailable("kafka") { + continue + } + addr := name + if ch.Value.Address != "" { + addr = ch.Value.Address + } + + data := newMqttTopic(addr, ch.Value, mi.Config) + if len(ch.Value.Parameters) > 0 { + for _, t := range mi.Topics { + if err := ch.Value.IsNameValid(t.Name); err == nil { + params, _ := ch.Value.ExtractParams(t.Name) + data.Instances = append(data.Instances, mqttTopicInstance{ + Name: t.Name, + Parameters: params, + }) + } + } + } + + data.Metrics = mqttTopicMetrics{ + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + } + + topics = append(topics, data) + } + slices.SortFunc(topics, func(a, b mqttTopic) int { + return strings.Compare(a.Name, b.Name) + }) + return topics +} + +func newMqttTopic(name string, ch *asyncapi3.Channel, cfg *asyncapi3.Config) mqttTopic { + result := mqttTopic{ + Name: name, + Title: ch.Title, + Summary: ch.Summary, + Description: ch.Description, + Messages: getMessageConfigs(ch, cfg), + } + + for _, tRef := range ch.Tags { + if tRef.Value == nil { + continue + } + result.Tags = append(result.Tags, kafkaTag{ + Name: tRef.Value.Name, + Description: tRef.Value.Description, + }) + } + + return result +} diff --git a/api/handler_mqtt_test.go b/api/handler_mqtt_test.go new file mode 100644 index 000000000..2e43f822b --- /dev/null +++ b/api/handler_mqtt_test.go @@ -0,0 +1,206 @@ +package api_test + +import ( + "fmt" + "mokapi/api" + "mokapi/config/dynamic" + "mokapi/config/dynamic/dynamictest" + "mokapi/config/static" + "mokapi/engine/enginetest" + "mokapi/providers/asyncapi3" + "mokapi/providers/asyncapi3/asyncapi3test" + "mokapi/providers/asyncapi3/mqtt/store" + "mokapi/runtime" + "mokapi/runtime/events/eventstest" + "mokapi/runtime/monitor" + "mokapi/runtime/runtimetest" + "mokapi/try" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestHandler_Mqtt(t *testing.T) { + mustTime := func(s string) time.Time { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + panic(err) + } + return t + } + + testcases := []struct { + name string + app func() *runtime.App + requestUrl string + responseBody string + }{ + { + name: "get services", + app: func() *runtime.App { + return runtimetest.NewApp( + runtimetest.WithMqtt( + asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + ), + ), + ) + }, + requestUrl: "http://foo.api/api/services", + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + }, + { + name: "get MQTT services", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + _, _ = app.Mqtt.Add(&dynamic.Config{ + Info: dynamic.ConfigInfo{Url: try.MustUrl("mqtt.yaml")}, + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "mqtt", "1.0"), + asyncapi3test.WithContact("mokapi", "https://mokapi.io", "info@mokapi.io"), + ), + }, enginetest.NewEngine()) + _, _ = app.Kafka.Add(&dynamic.Config{ + Info: dynamic.ConfigInfo{Url: try.MustUrl("kafka.yaml")}, + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "kafka", "1.0"), + asyncapi3test.WithContact("mokapi", "https://mokapi.io", "info@mokapi.io"), + ), + }, enginetest.NewEngine()) + return app + }, + requestUrl: "http://foo.api/api/services/mqtt", + responseBody: `[{"name":"foo","description":"mqtt","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + }, + { + name: "get specific", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + ), + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + + _, _ = app.Mqtt.Add(cfg, enginetest.NewEngine()) + return app + }, + requestUrl: "http://foo.api/api/services/mqtt/foo", + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"mokapi","host":":1883","protocol":"mqtt","title":"Mokapi Default Broker","summary":"Automatically added broker because no servers are defined in the AsyncAPI spec","description":""}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, + }, + { + name: "topic with parameter", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + addr := fmt.Sprintf(":%v", try.GetFreePort()) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "mqtt", addr), + asyncapi3test.WithChannel("sensors/{sensorId}/data", + asyncapi3test.WithParameter("sensorId", &asyncapi3.Parameter{}), + ), + ), + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + + mi, err := app.Mqtt.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + mi.Topics["sensors/1234z/data"] = &store.Topic{Name: "sensors/1234z/data"} + + return app + }, + requestUrl: "http://foo.api/api/services/mqtt/foo/topics", + responseBody: `[{"name":"sensors/{sensorId}/data","instances":[{"name":"sensors/1234z/data","parameters":{"sensorId":"1234z"}}],"metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := api.New(tc.app(), static.Api{}) + + try.Handler(t, + http.MethodGet, + tc.requestUrl, + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.HasBody(tc.responseBody)) + }) + } +} + +func TestHandler_Mqtt_Metrics(t *testing.T) { + testcases := []struct { + name string + app *runtime.App + requestUrl string + responseBody string + addMetrics func(monitor *monitor.Monitor) + }{ + { + name: "service list with metric", + app: runtimetest.NewApp(runtimetest.WithMqttInfo("foo", getMqttInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), + requestUrl: "http://foo.api/api/services", + responseBody: `[{"name":"foo","version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":1,"mqtt_message_timestamp":12345678}}]`, + addMetrics: func(monitor *monitor.Monitor) { + monitor.Mqtt.Messages.WithLabel("foo", "topic").Add(1) + monitor.Mqtt.LastMessage.WithLabel("foo", "topic").Set(12345678) + }, + }, + { + name: "cluster with metric", + app: runtimetest.NewApp( + runtimetest.WithMqttInfo("foo", getMqttInfo( + asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"), + asyncapi3test.WithChannel("foo"), + ), + ))), + requestUrl: "http://foo.api/api/services/mqtt/foo", + responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"foo","metrics":{"mqtt_messages_total":1,"mqtt_message_timestamp":12345678}}]}`, + addMetrics: func(monitor *monitor.Monitor) { + monitor.Mqtt.Messages.WithLabel("foo", "topic").Add(1) + monitor.Mqtt.LastMessage.WithLabel("foo", "topic").Set(12345678) + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := api.New(tc.app, static.Api{}) + tc.addMetrics(tc.app.Monitor) + + try.Handler(t, + http.MethodGet, + tc.requestUrl, + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.HasBody(tc.responseBody)) + }) + } +} + +func getMqttInfo(config *asyncapi3.Config) *runtime.MqttInfo { + return &runtime.MqttInfo{ + Config: config, + Store: store.New(config, enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewMqtt()), + } +} diff --git a/api/handler_schema.go b/api/handler_schema.go index 8c8095078..c9e9cc129 100644 --- a/api/handler_schema.go +++ b/api/handler_schema.go @@ -354,11 +354,11 @@ func getSpecs(app *runtime.App, spec, name string) []interface{} { switch spec { case "openapi": if name == "" { - for _, s := range app.ListHttp() { + for _, s := range app.Http.List() { results = append(results, s) } } - s := app.GetHttp(name) + s := app.Http.Get(name) if s != nil { results = append(results, s) } @@ -372,10 +372,10 @@ func getSpecs(app *runtime.App, spec, name string) []interface{} { } default: if name == "" { - for _, s := range app.ListHttp() { + for _, s := range app.Http.List() { results = append(results, s) } - } else if s := app.GetHttp(name); s != nil { + } else if s := app.Http.Get(name); s != nil { results = append(results, s) } if name == "" { diff --git a/api/handler_search_test.go b/api/handler_search_test.go index 4c81473df..3c1a1865b 100644 --- a/api/handler_search_test.go +++ b/api/handler_search_test.go @@ -50,7 +50,7 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) return app }, @@ -70,7 +70,7 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) return app }, @@ -90,9 +90,9 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) cfg = openapitest.NewConfig("3.0", openapitest.WithInfo("bar", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) return app }, @@ -112,9 +112,9 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) cfg = openapitest.NewConfig("3.0", openapitest.WithInfo("bar", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) return app }, @@ -142,7 +142,7 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) h := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(h)) + app.Http.Add(toConfig(h)) k := asyncapi3test.NewConfig(asyncapi3test.WithInfo("foo", "", "")) _, err := app.Kafka.Add(toConfig(k), enginetest.NewEngine()) require.NoError(t, err) @@ -172,7 +172,7 @@ func TestHandler_SearchQuery(t *testing.T) { }}}, &dynamictest.Reader{}) h := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(h)) + app.Http.Add(toConfig(h)) k := asyncapi3test.NewConfig(asyncapi3test.WithInfo("foo", "", "")) _, err := app.Kafka.Add(toConfig(k), enginetest.NewEngine()) require.NoError(t, err) @@ -191,7 +191,10 @@ func TestHandler_SearchQuery(t *testing.T) { app := tc.app() pool := safe.NewPool(context.Background()) app.Start(pool) - defer pool.Stop() + defer func() { + app.Stop() + pool.Stop() + }() h := New(app, static.Api{}) diff --git a/api/handler_system_test.go b/api/handler_system_test.go index 204c5e425..1bcbda53c 100644 --- a/api/handler_system_test.go +++ b/api/handler_system_test.go @@ -16,7 +16,7 @@ func TestHandler_System(t *testing.T) { fn func(t *testing.T, h http.Handler, sm *events.StoreManager) }{ { - name: "no event stores", + name: "default store with size 1", fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { try.Handler(t, http.MethodGet, @@ -24,7 +24,9 @@ func TestHandler_System(t *testing.T) { nil, "", h, - try.HasStatusCode(404)) + try.HasStatusCode(200), + try.HasBody(`[{"traits":{},"size":1,"numEvents":0}]`), + ) }, }, { @@ -40,7 +42,7 @@ func TestHandler_System(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[{"traits":{"namespace":"foo"},"size":1,"numEvents":0}]`)) + try.HasBody(`[{"traits":{},"size":1,"numEvents":0},{"traits":{"namespace":"foo"},"size":1,"numEvents":0}]`)) }, }, { @@ -58,7 +60,7 @@ func TestHandler_System(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[{"traits":{"namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo"},"size":1,"numEvents":0}]`)) + try.HasBody(`[{"traits":{},"size":1,"numEvents":0},{"traits":{"namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo"},"size":1,"numEvents":0}]`)) }, }, { @@ -76,7 +78,7 @@ func TestHandler_System(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[{"traits":{"namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo","topic":"foo"},"size":1,"numEvents":0}]`)) + try.HasBody(`[{"traits":{},"size":1,"numEvents":0},{"traits":{"namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo"},"size":1,"numEvents":0},{"traits":{"name":"Kafka Testserver","namespace":"foo","topic":"foo"},"size":1,"numEvents":0}]`)) }, }, } @@ -84,7 +86,7 @@ func TestHandler_System(t *testing.T) { for _, tc := range testcases { tc := tc t.Run(tc.name, func(t *testing.T) { - cfg := &static.Config{} + cfg := &static.Config{Event: static.Event{Store: map[string]static.Store{"default": {Size: 1}}}} app := runtime.New(cfg, &dynamictest.Reader{}) h := New(app, static.Api{}) diff --git a/api/handler_test.go b/api/handler_test.go index 2d80ec935..83d371b49 100644 --- a/api/handler_test.go +++ b/api/handler_test.go @@ -171,6 +171,21 @@ func TestHandler_Api_Info(t *testing.T) { try.HasBody(`{"version":"0.0.0","buildTime":"","activeServices":["ldap"],"search":{"enabled":false}}`)) }, }, + { + name: "mqtt active", + app: runtimetest.NewApp(runtimetest.WithMqttInfo("foo", &runtime.MqttInfo{})), + fn: func(t *testing.T, h http.Handler) { + try.Handler(t, + http.MethodGet, + "http://foo.api/api/info", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.HasBody(`{"version":"0.0.0","buildTime":"","activeServices":["mqtt"],"search":{"enabled":false}}`)) + }, + }, } t.Parallel() diff --git a/api/replace_html.go b/api/replace_html.go index dd033fe89..abb983759 100644 --- a/api/replace_html.go +++ b/api/replace_html.go @@ -2,12 +2,13 @@ package api import ( "fmt" - log "github.com/sirupsen/logrus" "mokapi/providers/openapi" "mokapi/runtime" "net/url" "regexp" "strings" + + log "github.com/sirupsen/logrus" ) var ( @@ -59,7 +60,7 @@ func getHttpMetaInfo(segments []string, app *runtime.App) (title, description st return } - c := app.GetHttp(name) + c := app.Http.Get(name) if c == nil { return } diff --git a/config/dynamic/provider/file/file.go b/config/dynamic/provider/file/file.go index 0cd8cac8e..fc2e845c8 100644 --- a/config/dynamic/provider/file/file.go +++ b/config/dynamic/provider/file/file.go @@ -166,7 +166,7 @@ func (p *Provider) watch(pool *safe.Pool) error { events = nil t = nil mu.Unlock() - p.processEvents(e) + go p.processEvents(e) }) } t.Reset(time.Second) diff --git a/config/dynamic/provider/npm/npm.go b/config/dynamic/provider/npm/npm.go index fae306865..beb3fc049 100644 --- a/config/dynamic/provider/npm/npm.go +++ b/config/dynamic/provider/npm/npm.go @@ -10,6 +10,7 @@ import ( "net/url" "path/filepath" "strings" + "sync" log "github.com/sirupsen/logrus" ) @@ -21,6 +22,7 @@ type Provider struct { reader file.FSReader f *file.Provider + m sync.Mutex } func New(cfg static.NpmProvider) *Provider { @@ -72,6 +74,9 @@ func (p *Provider) Read(u *url.URL) (*dynamic.Config, error) { } func (p *Provider) Start(ch chan dynamic.ConfigEvent, pool *safe.Pool) error { + p.m.Lock() + defer p.m.Unlock() + workDir, err := p.reader.GetWorkingDir() if err != nil { return err @@ -116,6 +121,7 @@ func (p *Provider) forward(ch chan dynamic.ConfigEvent, ctx context.Context) { } } + p.m.Lock() for dir, pkg := range p.config { if strings.HasPrefix(path, dir) { relative := path[len(dir)+1:] @@ -143,6 +149,7 @@ func (p *Provider) forward(ch chan dynamic.ConfigEvent, ctx context.Context) { ch <- e } } + p.m.Unlock() } } } diff --git a/config/static/static_config.go b/config/static/static_config.go index fc22f4620..83f8aa1a1 100644 --- a/config/static/static_config.go +++ b/config/static/static_config.go @@ -40,6 +40,7 @@ func NewConfig() *Config { cfg.Api.Port = 8080 cfg.Api.Dashboard = true cfg.Api.Search.Enabled = true + cfg.Api.Search.NumIndexWorker = 2 cfg.Health.Enabled = true cfg.Health.Port = 8080 @@ -80,9 +81,10 @@ type Api struct { } type Search struct { - Enabled bool - IndexPath string `yaml:"indexPath" json:"indexPath" flag:"index-path"` - InMemory bool `yaml:"inMemory" json:"inMemory" flag:"in-memory"` + Enabled bool + IndexPath string `yaml:"indexPath" json:"indexPath" flag:"index-path"` + InMemory bool `yaml:"inMemory" json:"inMemory" flag:"in-memory"` + NumIndexWorker int `yaml:"-" json:"-"` } type FileProvider struct { diff --git a/engine/common/host.go b/engine/common/host.go index ef67cf24b..41a30ad54 100644 --- a/engine/common/host.go +++ b/engine/common/host.go @@ -176,9 +176,16 @@ func (e *JobExecution) AppendLog(level, message string) { } func (e *JobExecution) Title() string { + if e.Error != nil { + return fmt.Sprintf("Error in: %v", e.Tags["name"]) + } return e.Tags["name"] } +func (e *JobExecution) Domain() string { + return "Job Execution" +} + type FakerNode interface { Name() string Fake(r *generator.Request) (interface{}, error) diff --git a/engine/host.go b/engine/host.go index 0690843ba..269115f39 100644 --- a/engine/host.go +++ b/engine/host.go @@ -102,8 +102,9 @@ func (sh *scriptHost) Cron(expr string, handler func(), opt common.JobOptions) ( } func (sh *scriptHost) newJobFunc(handler func(), opt common.JobOptions, schedule string, id int) func() { + name := filepath.Base(sh.name) tags := map[string]string{ - "name": sh.name, + "name": name, "file": sh.name, "fileKey": sh.file.Info.Key(), } diff --git a/engine/kafka_test.go b/engine/kafka_test.go index 05ba77f8e..96c838ba3 100644 --- a/engine/kafka_test.go +++ b/engine/kafka_test.go @@ -16,6 +16,7 @@ import ( opSchema "mokapi/providers/openapi/schema" opSchematest "mokapi/providers/openapi/schema/schematest" "mokapi/runtime" + "mokapi/runtime/metrics" "mokapi/schema/json/generator" "mokapi/schema/json/schema/schematest" "net/url" @@ -329,7 +330,7 @@ func TestKafkaClient(t *testing.T) { require.Equal(t, "key2", kafka.BytesToString(b.Records[1].Key)) require.Equal(t, `"bar"`, kafka.BytesToString(b.Records[1].Value)) - require.Equal(t, float64(2), app.Monitor.Kafka.Messages.Sum()) + require.Equal(t, float64(2), app.Monitor.Kafka.Messages.Sum(metrics.NewQuery())) }, }, { diff --git a/go.mod b/go.mod index b7484626d..a1a2d8cdc 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,20 @@ go 1.25.5 require ( github.com/Masterminds/sprig v2.22.0+incompatible - github.com/blevesearch/bleve/v2 v2.5.7 - github.com/blevesearch/bleve_index_api v1.3.10 + github.com/blevesearch/bleve/v2 v2.6.0 + github.com/blevesearch/bleve_index_api v1.3.11 github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 github.com/brianvoe/gofakeit/v6 v6.28.0 github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c github.com/evanw/esbuild v0.28.0 - github.com/fsnotify/fsnotify v1.9.0 + github.com/fsnotify/fsnotify v1.10.1 github.com/go-co-op/gocron v1.37.0 github.com/go-git/go-git/v5 v5.19.0 github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.1 github.com/jinzhu/inflection v1.0.0 - github.com/modelcontextprotocol/go-sdk v1.5.0 + github.com/modelcontextprotocol/go-sdk v1.6.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.4 github.com/stretchr/testify v1.11.1 @@ -34,24 +35,25 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.1.6 // indirect - github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect - github.com/bits-and-blooms/bitset v1.22.0 // indirect - github.com/blevesearch/geo v0.2.4 // indirect - github.com/blevesearch/go-faiss v1.0.26 // indirect + github.com/RoaringBitmap/roaring/v2 v2.14.5 // indirect + github.com/bits-and-blooms/bitset v1.24.2 // indirect + github.com/blevesearch/geo v0.2.5 // indirect + github.com/blevesearch/go-faiss v1.1.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect - github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.3.13 // indirect + github.com/blevesearch/mmap-go v1.2.0 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.4.7 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect - github.com/blevesearch/vellum v1.1.0 // indirect - github.com/blevesearch/zapx/v11 v11.4.2 // indirect - github.com/blevesearch/zapx/v12 v12.4.2 // indirect - github.com/blevesearch/zapx/v13 v13.4.2 // indirect - github.com/blevesearch/zapx/v14 v14.4.2 // indirect - github.com/blevesearch/zapx/v15 v15.4.2 // indirect - github.com/blevesearch/zapx/v16 v16.2.8 // indirect + github.com/blevesearch/vellum v1.2.0 // indirect + github.com/blevesearch/zapx/v11 v11.4.3 // indirect + github.com/blevesearch/zapx/v12 v12.4.3 // indirect + github.com/blevesearch/zapx/v13 v13.4.3 // indirect + github.com/blevesearch/zapx/v14 v14.4.3 // indirect + github.com/blevesearch/zapx/v15 v15.4.3 // indirect + github.com/blevesearch/zapx/v16 v16.3.4 // indirect + github.com/blevesearch/zapx/v17 v17.1.2 // indirect github.com/cloudflare/circl v1.6.3 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -61,10 +63,10 @@ require ( github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/google/go-github/v84 v84.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect - github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.14 // indirect diff --git a/go.sum b/go.sum index 9cf2790f0..e4960fda9 100644 --- a/go.sum +++ b/go.sum @@ -13,51 +13,52 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= -github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= -github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/RoaringBitmap/roaring/v2 v2.14.5 h1:ckd0o545JqDPeVJDgeFoaM21eBixUnlWfYgjE5VnyWw= +github.com/RoaringBitmap/roaring/v2 v2.14.5/go.mod h1:eq4wdNXxtJIS/oikeCzdX1rBzek7ANzbth041hrU8Q4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= -github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/blevesearch/bleve/v2 v2.5.7 h1:2d9YrL5zrX5EBBW++GOaEKjE+NPWeZGaX77IM26m1Z8= -github.com/blevesearch/bleve/v2 v2.5.7/go.mod h1:yj0NlS7ocGC4VOSAedqDDMktdh2935v2CSWOCDMHdSA= -github.com/blevesearch/bleve_index_api v1.3.10 h1:a7G+IOMa2xuO6f8vtutbTsqjVLpLuCuH3uoTZHkGiYg= -github.com/blevesearch/bleve_index_api v1.3.10/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko= -github.com/blevesearch/geo v0.2.4 h1:ECIGQhw+QALCZaDcogRTNSJYQXRtC8/m8IKiA706cqk= -github.com/blevesearch/geo v0.2.4/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= -github.com/blevesearch/go-faiss v1.0.26 h1:4dRLolFgjPyjkaXwff4NfbZFdE/dfywbzDqporeQvXI= -github.com/blevesearch/go-faiss v1.0.26/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= +github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blevesearch/bleve/v2 v2.6.0 h1:Cyd3dd4q5tCbOV8MnKUVRUDYMHOir9xn12NZzXVSEd4= +github.com/blevesearch/bleve/v2 v2.6.0/go.mod h1:gLmI8lWgHgrIYf7UpUX7JISI1CaqC6VScu46mHThuAY= +github.com/blevesearch/bleve_index_api v1.3.11 h1:x29vbV8OjWfLcrDVd7Lr1q+BkLNS0JWNEig0MCVnKH4= +github.com/blevesearch/bleve_index_api v1.3.11/go.mod h1:xvd48t5XMeeioWQ5/jZvgLrV98flT2rdvEJ3l/ki4Ko= +github.com/blevesearch/geo v0.2.5 h1:yJg9FX1oRwLnjXSXF+ECHfXFTF4diF02Ca/qUGVjJhE= +github.com/blevesearch/geo v0.2.5/go.mod h1:Jhq7WE2K6mJTx1xS44M2pUO6Io+wjCSHh1+co3YOgH4= +github.com/blevesearch/go-faiss v1.1.0 h1:xM7Jc0ZUCv5lssG9Ohj3Jv0SdTpxcUABU1dDt9XVsc4= +github.com/blevesearch/go-faiss v1.1.0/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= -github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= -github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.3.13 h1:ZPjv/4VwWvHJZKeMSgScCapOy8+DdmsmRyLmSB88UoY= -github.com/blevesearch/scorch_segment_api/v2 v2.3.13/go.mod h1:ENk2LClTehOuMS8XzN3UxBEErYmtwkE7MAArFTXs9Vc= +github.com/blevesearch/mmap-go v1.2.0 h1:l33nNKPFcBjJUMwem6sAYJPUzhUCABoK9FxZDGiFNBI= +github.com/blevesearch/mmap-go v1.2.0/go.mod h1:Vd6+20GBhEdwJnU1Xohgt88XCD/CTWcqbCNxkZpyBo0= +github.com/blevesearch/scorch_segment_api/v2 v2.4.7 h1:GlMzW08hcsM3DnLUxhyF/1PcDal1qtvvIuytuph5djw= +github.com/blevesearch/scorch_segment_api/v2 v2.4.7/go.mod h1://IJ7tG3QCf0cWW/aVSXqy77tc1AvLu3fcJLYEvOAFs= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= -github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= -github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= -github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= -github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= -github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= -github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= -github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= -github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= -github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= -github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= -github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= -github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= -github.com/blevesearch/zapx/v16 v16.2.8 h1:SlnzF0YGtSlrsOE3oE7EgEX6BIepGpeqxs1IjMbHLQI= -github.com/blevesearch/zapx/v16 v16.2.8/go.mod h1:murSoCJPCk25MqURrcJaBQ1RekuqSCSfMjXH4rHyA14= +github.com/blevesearch/vellum v1.2.0 h1:xkDiOEsHc2t3Cp0NsNZZ36pvc130sCzcGKOPMzXe+e0= +github.com/blevesearch/vellum v1.2.0/go.mod h1:uEcfBJz7mAOf0Kvq6qoEKQQkLODBF46SINYNkZNae4k= +github.com/blevesearch/zapx/v11 v11.4.3 h1:PTZOO5loKpHC/x/GzmPZNa9cw7GZIQxd5qRjwij9tHY= +github.com/blevesearch/zapx/v11 v11.4.3/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= +github.com/blevesearch/zapx/v12 v12.4.3 h1:eElXvAaAX4m04t//CGBQAtHNPA+Q6A1hHZVrN3LSFYo= +github.com/blevesearch/zapx/v12 v12.4.3/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= +github.com/blevesearch/zapx/v13 v13.4.3 h1:qsdhRhaSpVnqDFlRiH9vG5+KJ+dE7KAW9WyZz/KXAiE= +github.com/blevesearch/zapx/v13 v13.4.3/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= +github.com/blevesearch/zapx/v14 v14.4.3 h1:GY4Hecx0C6UTmiNC2pKdeA2rOKiLR5/rwpU9WR51dgM= +github.com/blevesearch/zapx/v14 v14.4.3/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= +github.com/blevesearch/zapx/v15 v15.4.3 h1:iJiMJOHrz216jyO6lS0m9RTCEkprUnzvqAI2lc/0/CU= +github.com/blevesearch/zapx/v15 v15.4.3/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= +github.com/blevesearch/zapx/v16 v16.3.4 h1:hDAqA8qusZTNbPEL7//w5P65UZ2de6yhSeUaTbp0Po0= +github.com/blevesearch/zapx/v16 v16.3.4/go.mod h1:zqkPPqs9GS9FzVWzCO3Wf1X044yWAV17+4zb+FTiEHg= +github.com/blevesearch/zapx/v17 v17.1.2 h1:avbOk2igaASNoiy0BE/jPgcxAnRI2PGeydeP4hg7Ikk= +github.com/blevesearch/zapx/v17 v17.1.2/go.mod h1:WQObxKrqUX7cd0G1GMvDfc/bmZzQvoy7APOPimx7DiI= github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 h1:WPqnN6NS9XvYlOgZQAIseN7Z1uAiE+UxgDKlW7FvFuU= github.com/bradleyfalzon/ghinstallation/v2 v2.18.0/go.mod h1:gpoSwwWc4biE49F7n+roCcpkEkZ1Qr9soZ2ESvMiouU= github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= @@ -83,8 +84,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/evanw/esbuild v0.28.0 h1:V96ghtc5p5JnNUQIUsc5H3kr+AcFcMqOJll2ZmJW6Lo= github.com/evanw/esbuild v0.28.0/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= @@ -105,8 +106,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -116,13 +117,15 @@ github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfh github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.14 h1:fOqeC1+nCuuk6PKQdg9YmosXX7Y7mHX6R/0ZldI9iHo= @@ -150,8 +153,8 @@ github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMK github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/modelcontextprotocol/go-sdk v1.5.0 h1:CHU0FIX9kpueNkxuYtfYQn1Z0slhFzBZuq+x6IiblIU= -github.com/modelcontextprotocol/go-sdk v1.5.0/go.mod h1:gggDIhoemhWs3BGkGwd1umzEXCEMMvAnhTrnbXJKKKA= +github.com/modelcontextprotocol/go-sdk v1.6.0 h1:PPLS3kn7WtOEnR+Af4X5H96SG0qSab8R/ZQT/HkhPkY= +github.com/modelcontextprotocol/go-sdk v1.6.0/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= @@ -186,7 +189,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -221,7 +223,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= @@ -250,7 +251,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= layeh.com/gopher-luar v1.0.11 h1:8zJudpKI6HWkoh9eyyNFaTM79PY6CAPcIr6X/KTiliw= diff --git a/imap/append.go b/imap/append.go index bb5817ec5..8d38ca609 100644 --- a/imap/append.go +++ b/imap/append.go @@ -5,7 +5,6 @@ import ( "bytes" "encoding/base64" "fmt" - log "github.com/sirupsen/logrus" "io" "mime/multipart" "mime/quotedprintable" @@ -15,6 +14,8 @@ import ( "net/textproto" "strings" "time" + + log "github.com/sirupsen/logrus" ) type AppendOptions struct { @@ -36,7 +37,7 @@ func (c *conn) handleAppend(tag string, d *Decoder) error { } d.SP() - opt := AppendOptions{} + opt := AppendOptions{Date: time.Now()} if d.IsList() { err = d.List(func() error { f, err := d.ReadFlag() @@ -52,6 +53,14 @@ func (c *conn) handleAppend(tag string, d *Decoder) error { d.SP() } + if d.is("\"") { + opt.Date, err = d.Date() + if err != nil { + return err + } + d.SP() + } + if err := d.expect("{"); err != nil { return err } @@ -63,7 +72,7 @@ func (c *conn) handleAppend(tag string, d *Decoder) error { return err } - c.tpc.PrintfLine("+ Ready for literal data") + _ = c.tpc.PrintfLine("+ Ready for literal data") r := io.LimitReader(c.tpc.R, size) data, err := io.ReadAll(r) diff --git a/imap/append_test.go b/imap/append_test.go index 5d096d19a..24d181e11 100644 --- a/imap/append_test.go +++ b/imap/append_test.go @@ -2,12 +2,14 @@ package imap_test import ( "fmt" - "github.com/stretchr/testify/require" "mokapi/imap" "mokapi/imap/imaptest" "mokapi/smtp" "mokapi/try" "testing" + "time" + + "github.com/stretchr/testify/require" ) func TestAppend(t *testing.T) { @@ -37,7 +39,7 @@ func TestAppend(t *testing.T) { }() c := imap.NewClient(fmt.Sprintf("localhost:%v", p)) - defer c.Close() + defer func() { _ = c.Close() }() _, err := c.Dial() require.NoError(t, err) @@ -73,6 +75,7 @@ func TestAppend_Flags(t *testing.T) { handler := &imaptest.Handler{ AppendFunc: func(mailbox string, msg *smtp.Message, opt imap.AppendOptions) error { require.Equal(t, []imap.Flag{imap.FlagSeen}, opt.Flags) + require.Greater(t, opt.Date, time.Time{}) return nil }, } @@ -89,7 +92,7 @@ func TestAppend_Flags(t *testing.T) { }() c := imap.NewClient(fmt.Sprintf("localhost:%v", p)) - defer c.Close() + defer func() { _ = c.Close() }() _, err := c.Dial() require.NoError(t, err) @@ -119,3 +122,56 @@ func TestAppend_Flags(t *testing.T) { require.Equal(t, "A003 OK APPEND completed", res2) } + +func TestAppend_Flags_And_Time(t *testing.T) { + + handler := &imaptest.Handler{ + AppendFunc: func(mailbox string, msg *smtp.Message, opt imap.AppendOptions) error { + require.Equal(t, []imap.Flag{imap.FlagSeen}, opt.Flags) + require.Equal(t, "2026-04-26T14:20:31+02:00", opt.Date.Format(time.RFC3339)) + return nil + }, + } + + p := try.GetFreePort() + s := &imap.Server{ + Addr: fmt.Sprintf(":%v", p), + Handler: handler, + } + defer s.Close() + go func() { + err := s.ListenAndServe() + require.ErrorIs(t, err, imap.ErrServerClosed) + }() + + c := imap.NewClient(fmt.Sprintf("localhost:%v", p)) + defer func() { _ = c.Close() }() + + _, err := c.Dial() + require.NoError(t, err) + + err = c.PlainAuth("", "", "") + require.NoError(t, err) + + res, err := c.SendRaw("A003 APPEND Sent (\\Seen) \"26-Apr-2026 14:20:31 +0200\" {310}") + require.NoError(t, err) + require.Equal(t, "+ Ready for literal data", res) + + request := []string{ + "Date: Mon, 7 Feb 1994 21:52:25 -0800 (PST)", + "From: Fred Foobar ", + "Subject: afternoon meeting", + "To: mooch@owatagu.siam.edu", + "Message-Id: ", + "MIME-Version: 1.0", + "Content-Type: TEXT/PLAIN; CHARSET=US-ASCII", + "", + "Hello Joe, do you think we can meet at 3:30 tomorrow?", + "", + } + + res2, err := c.SendRawLines(request) + require.NoError(t, err) + require.Equal(t, "A003 OK APPEND completed", res2) + +} diff --git a/mcp/data/automation-core.md b/mcp/data/automation-core.md index 7e35fc17c..55b15d81b 100644 --- a/mcp/data/automation-core.md +++ b/mcp/data/automation-core.md @@ -44,6 +44,35 @@ interface Mokapi { * getEvents({ apiType: 'http', name: 'Swagger Petstore', path: '/pets' }) */ getEvents(traits: HttpTraits | KafkaTraits, limit?: number): Event[]; + + /** + * Returns a specific event from Mokapi based on the id + * @param id The id of the event + * @example + * getEvent('6cbca4f8-ea25-4f66-861c-44d8b15321b1') + */ + getEvent(id: string): Event[]; + + /** + * Advanced Search across all APIs, Topics, and Documentation. + * By default, multiple terms are combined with OR (results contain at least one term). Use prefixes to enforce stricter matches. + * + * QUERY SYNTAX: + * - Simple: `petstore` (Search in all fields) + * Type: `type:kafka` (Search only Kafka) + * - Exact: `"Swagger Petstore"` (Use quotes for phrases) + * - Exclude: `petstore -kafka` (Exclude terms with '-') + * - Wildcard: `pet*` (Matches "pet", "pets", "petstore") + * - Fuzy: `pet~` (Matches "pets", "pest" or slight typos) + * - Fields: `name:petstore`, `path:/pets`, `description:dog` + * - Boosting: `path:/pets^2` (Make path matches more important) + * - Numeric Ranges: `statusCode:>=200 + * + * @param queryText The search term or complex query string. + * @param index Page index for pagination (starts at 0). + * @param limit Number of results (default 10). + */ + search(queryText: string, index?: number, limit?: number): SearchResult } type ApiType = 'http' | 'kafka' | 'ldap' | 'mail' @@ -53,6 +82,33 @@ interface ApiSummary { type: ApiType; } +interface SearchResult { + items: SearchResultItem[] + total: number +} + +interface SearchResultItem { + /** The type of the resource (e.g., 'http', 'kafka', 'event') */ + type: string + /** The name of the API */ + domain: string + /** A human-readable title or summary */ + title: string + /** + * Relevant text snippets showing where the match was found. + * Useful to see context like 'GET /pets' or 'Topic: orders'. + */ + fragments: string[] + /** + * Additional context specific to the result type. + * - HTTP: includes 'method', 'path', 'service' + * - Kafka: includes 'topic', 'service' + */ + metadata: Record + + time?: string +} + /** * JSON Schema defines a JSON-based format for describing the structure of JSON data * @example @@ -201,4 +257,5 @@ interface Schema { } type SchemaType = "object" | "array" | "number" | "integer" | "string" | "boolean" | "null"; + ``` \ No newline at end of file diff --git a/mcp/data/automation-kafka.md b/mcp/data/automation-kafka.md index 46e75d23b..d57d5ff1a 100644 --- a/mcp/data/automation-kafka.md +++ b/mcp/data/automation-kafka.md @@ -33,13 +33,20 @@ interface KafkaTopic extends KafkaTopicSummary { /** * Use 'produce' to send a message to this topic. - * Check 'operations' with action 'send' for valid payloads. + * + * Use 'operations' with action 'send' to inspect the expected message schema + * and valid payload examples for this topic. + * + * Mokapi automatically encodes the payload based on the topic configuration: + * - JSON topics expect a JSON-compatible object + * - Avro topics expect a value matching the Avro schema + * * @param partition The target partition index. MUST be one of the indices listed in the 'partitions' array. - * @param value The message payload. If the operation specifies a JSON schema, provide this as a stringified object. + * @param value The message payload. Provide a value that matches the topic message schema. * @param key Optional message key. * @param headers Optional metadata headers. */ - produce(partition: number, value: string, key?: string, headers?: KafkaHeader): void + produce(partition: number, value: any, key?: string, headers?: KafkaHeader): void /** * INSPECT: Retrieves a specific record for analysis or verification. diff --git a/mcp/data/automation.md b/mcp/data/automation.md index 9be899ad3..b96ab8eec 100644 --- a/mcp/data/automation.md +++ b/mcp/data/automation.md @@ -39,4 +39,32 @@ mokapi.getEvents() Get latest HTTP events for a specific API ```typescript mokapi.getEvents({ type: 'http', name: 'Petstore' }) -``` \ No newline at end of file +``` + +Search for all POST endpoints related to "payments". +```typescript +// 1. Search for a specific capability +const searchResponse = mokapi.search("method:POST AND path:*payments*"); +let op = null +if (searchResponse.items.length > 0) { + const match = searchResponse.items[0]; + + // 2. Use metadata to load the full API specification + const api = mokapi.getApi(match.metadata.service); + + // 3. Find the specific operation to see allowed status codes/schemas + // Tip: Use the path from metadata to identify the correct operation + const opSummary = api.getOperations().find(x => x.path === match.metadata.path && x.method === match.metadata.method); + + if (opSummary) { + op = api.getOperation(opSummary.id); + } +} +op +``` + +Search for errors (404 or 500) related to the Petstore +```typescript +const errors = mokapi.search('+api:"Petstore" +response.statusCode:>=400 +type:event') +errors.items.map(x => mokapi.getEvent(x.metadata.id)) +``` diff --git a/mcp/generate_http_mock_response.go b/mcp/generate_http_mock_response.go index fc7fc91bd..8ac46d4ef 100644 --- a/mcp/generate_http_mock_response.go +++ b/mcp/generate_http_mock_response.go @@ -109,7 +109,7 @@ The returned object should be assigned to: func (s *Service) GenerateHttpMockResponse(_ context.Context, in GenerateHttpMockResponseInput) (GenerateHttpMockResponseOutput, error) { result := GenerateHttpMockResponseOutput{StatusCode: in.StatusCode, Headers: make(map[string]any)} - info := s.app.GetHttp(in.ApiName) + info := s.app.Http.Get(in.ApiName) if info == nil { return result, fmt.Errorf("http api not found") } diff --git a/mcp/get_api_spec.go b/mcp/get_api_spec.go index 3f455d8df..2d4a3be9a 100644 --- a/mcp/get_api_spec.go +++ b/mcp/get_api_spec.go @@ -95,7 +95,7 @@ func (s *Service) GetApiSpec(_ context.Context, in GetApiSpecInput) (GetApiSpecO if in.Name == "" { if in.Type == "http" || len(in.Type) == 0 { - for _, api := range s.app.ListHttp() { + for _, api := range s.app.Http.List() { if api.Info.Name == "" { log.Warnf("mcp tool mokapi_get_api_spec: skip empty HTTTP API name") continue @@ -149,7 +149,7 @@ func (s *Service) GetApiSpec(_ context.Context, in GetApiSpecInput) (GetApiSpecO } if in.Type == "http" || len(in.Type) == 0 { - info := s.app.GetHttp(in.Name) + info := s.app.Http.Get(in.Name) if info != nil { result = append(result, ApiSpec{ Name: in.Name, diff --git a/mcp/get_http_response_schema.go b/mcp/get_http_response_schema.go index 7173c3448..324dc1cb7 100644 --- a/mcp/get_http_response_schema.go +++ b/mcp/get_http_response_schema.go @@ -60,7 +60,7 @@ All mock responses must strictly conform to this schema. Do not omit required fi } func (s *Service) GetHttpResponseSchema(_ context.Context, in GetHttpResponseSchemaInput) (any, error) { - info := s.app.GetHttp(in.ApiName) + info := s.app.Http.Get(in.ApiName) if info == nil { return nil, fmt.Errorf("http api not found") } diff --git a/mcp/run.go b/mcp/run.go index 3479b7847..649d780cc 100644 --- a/mcp/run.go +++ b/mcp/run.go @@ -9,6 +9,7 @@ import ( "mokapi/js/faker" "mokapi/providers/openapi" "mokapi/runtime" + "mokapi/runtime/search" "mokapi/schema/json/generator" "reflect" "slices" @@ -136,11 +137,13 @@ func (m *mokapi) init(obj *goja.Object) { _ = obj.Set("getApi", m.getApi) _ = obj.Set("fake", m.fake) _ = obj.Set("getEvents", m.getEvents) + _ = obj.Set("getEvent", m.getEvent) + _ = obj.Set("search", m.search) } func (m *mokapi) getApis() []ApiSummary { var result []ApiSummary - for _, api := range m.app.ListHttp() { + for _, api := range m.app.Http.List() { if api.Info.Name == "" { log.Warnf("mcp tool mokapi_execute_code: skip empty HTTTP API name") continue @@ -184,6 +187,50 @@ func (m *mokapi) fake(v goja.Value) (any, error) { return generator.New(&generator.Request{Schema: js}) } +type SearchResult struct { + Items []SearchResultItem `json:"items"` + Total uint64 `json:"total"` +} + +type SearchResultItem struct { + Type string `json:"type"` + Domain string `json:"domain,omitempty"` + Title string `json:"title"` + Fragments []string `json:"fragments,omitempty"` + Metadata map[string]string `json:"metadata"` + Time string `json:"time,omitempty"` +} + +func (m *mokapi) search(queryText string, index int, limit int) (SearchResult, error) { + if limit == 0 { + limit = 10 + } + r := search.Request{ + QueryText: queryText, + Index: index, + Limit: limit, + } + + sr, err := m.app.Search(r) + if err != nil { + return SearchResult{}, err + } + result := SearchResult{ + Total: sr.Total, + } + for _, item := range sr.Results { + result.Items = append(result.Items, SearchResultItem{ + Type: item.Type, + Domain: item.Domain, + Title: item.Title, + Fragments: item.Fragments, + Metadata: item.Params, + Time: item.Time, + }) + } + return result, nil +} + type customFieldNameMapper struct { } diff --git a/mcp/run_events.go b/mcp/run_events.go index f926f4b23..9ed110f72 100644 --- a/mcp/run_events.go +++ b/mcp/run_events.go @@ -32,6 +32,18 @@ func (m *mokapi) getEvents(vTraits goja.Value, vLimit goja.Value) ([]events.Even } } +func (m *mokapi) getEvent(id string) (events.Event, error) { + if id == "" { + return events.Event{}, fmt.Errorf("expected id parameter in GUID format, got '%v'", id) + } + + e := m.app.Events.GetEvent(id) + if e.Id == "" { + return e, fmt.Errorf("event %s not found. Use `mokapi.search('type:event ...')` to search for existing events", id) + } + return e, nil +} + func parseTraits(v goja.Value, vm *goja.Runtime) (events.Traits, error) { traits := events.Traits{} diff --git a/mcp/run_events_test.go b/mcp/run_events_test.go index c1f222439..b4f18fad7 100644 --- a/mcp/run_events_test.go +++ b/mcp/run_events_test.go @@ -24,7 +24,7 @@ func TestEvents(t *testing.T) { name string app *runtime.App code string - test func(t *testing.T, evts []events.Event, err error) + test func(t *testing.T, result any, err error) }{ { name: "without params should not error", @@ -32,8 +32,10 @@ func TestEvents(t *testing.T) { app: runtimetest.NewApp( runtimetest.WithEvent(events.NewTraits().WithNamespace("http"), &testEvent{Name: "test-1"}), ), - test: func(t *testing.T, evts []events.Event, err error) { + test: func(t *testing.T, result any, err error) { require.NoError(t, err) + require.IsType(t, []events.Event{}, result) + evts := result.([]events.Event) require.Len(t, evts, 1) require.Equal(t, &testEvent{Name: "test-1"}, evts[0].Data) }, @@ -45,8 +47,10 @@ func TestEvents(t *testing.T) { runtimetest.WithEvent(events.NewTraits().WithNamespace("kafka"), &testEvent{Name: "test-1"}), runtimetest.WithEvent(events.NewTraits().WithNamespace("http"), &testEvent{Name: "test-2"}), ), - test: func(t *testing.T, evts []events.Event, err error) { + test: func(t *testing.T, result any, err error) { require.NoError(t, err) + require.IsType(t, []events.Event{}, result) + evts := result.([]events.Event) require.Len(t, evts, 1) require.Equal(t, &testEvent{Name: "test-2"}, evts[0].Data) }, @@ -58,8 +62,10 @@ func TestEvents(t *testing.T) { runtimetest.WithEvent(events.NewTraits().WithNamespace("http").WithName("foo"), &testEvent{Name: "test-1"}), runtimetest.WithEvent(events.NewTraits().WithNamespace("http").WithName("bar"), &testEvent{Name: "test-2"}), ), - test: func(t *testing.T, evts []events.Event, err error) { + test: func(t *testing.T, result any, err error) { require.NoError(t, err) + require.IsType(t, []events.Event{}, result) + evts := result.([]events.Event) require.Len(t, evts, 1) require.Equal(t, &testEvent{Name: "test-2"}, evts[0].Data) }, @@ -71,8 +77,10 @@ func TestEvents(t *testing.T) { runtimetest.WithEvent(events.NewTraits().WithNamespace("http").With("path", "/users"), &testEvent{Name: "test-1"}), runtimetest.WithEvent(events.NewTraits().WithNamespace("http").With("path", "/pets"), &testEvent{Name: "test-2"}), ), - test: func(t *testing.T, evts []events.Event, err error) { + test: func(t *testing.T, result any, err error) { require.NoError(t, err) + require.IsType(t, []events.Event{}, result) + evts := result.([]events.Event) require.Len(t, evts, 1) require.Equal(t, &testEvent{Name: "test-2"}, evts[0].Data) }, @@ -84,12 +92,27 @@ func TestEvents(t *testing.T) { runtimetest.WithEvent(events.NewTraits().WithNamespace("http").With("method", "GET"), &testEvent{Name: "test-1"}), runtimetest.WithEvent(events.NewTraits().WithNamespace("http").With("method", "POST"), &testEvent{Name: "test-2"}), ), - test: func(t *testing.T, evts []events.Event, err error) { + test: func(t *testing.T, result any, err error) { require.NoError(t, err) + require.IsType(t, []events.Event{}, result) + evts := result.([]events.Event) require.Len(t, evts, 1) require.Equal(t, &testEvent{Name: "test-2"}, evts[0].Data) }, }, + { + name: "get specific event", + code: "mokapi.getEvent(mokapi.getEvents({ method: 'GET' })[0].id)", + app: runtimetest.NewApp( + runtimetest.WithEvent(events.NewTraits().WithNamespace("http").With("method", "GET"), &testEvent{Name: "test-1"}), + ), + test: func(t *testing.T, result any, err error) { + require.NoError(t, err) + require.IsType(t, events.Event{}, result) + evt := result.(events.Event) + require.Equal(t, &testEvent{Name: "test-1"}, evt.Data) + }, + }, } for _, tc := range testcases { @@ -100,9 +123,8 @@ func TestEvents(t *testing.T) { context.Background(), mcp.RunInput{Code: tc.code}, ) - require.IsType(t, []events.Event{}, r.Result) - tc.test(t, r.Result.([]events.Event), err) + tc.test(t, r.Result, err) }) } } diff --git a/mcp/run_http.go b/mcp/run_http.go index 3616dbf8c..77644c4f3 100644 --- a/mcp/run_http.go +++ b/mcp/run_http.go @@ -75,7 +75,7 @@ type Response struct { } func (m *mokapi) getHttpApi(name string) any { - for _, api := range m.app.ListHttp() { + for _, api := range m.app.Http.List() { if api.Info.Name == name { result := &OpenAPI{ Name: name, diff --git a/mcp/run_kafka.go b/mcp/run_kafka.go index ca2474e89..511a27305 100644 --- a/mcp/run_kafka.go +++ b/mcp/run_kafka.go @@ -1,14 +1,14 @@ package mcp import ( + "errors" "fmt" - "mokapi/engine" - "mokapi/engine/common" "mokapi/kafka" + "mokapi/media" + "mokapi/providers/asyncapi3/kafka/store" "mokapi/runtime" "slices" "strings" - "time" ) type Kafka struct { @@ -17,7 +17,7 @@ type Kafka struct { Brokers []Broker `json:"brokers"` info *runtime.KafkaInfo - client *engine.KafkaClient + client *store.Client } type Broker struct { @@ -40,7 +40,7 @@ type Topic struct { Operations []KafkaOperation `json:"operations,omitempty"` info *runtime.KafkaInfo - client *engine.KafkaClient + client *store.Client } type KafkaPartition struct { @@ -77,11 +77,14 @@ type KafkaRecord struct { func (m *mokapi) getKafkaApi(name string) any { for _, api := range m.app.Kafka.List() { if api.Info.Name == name { + client := store.NewClient(api.Store, m.app.Monitor.Kafka) + client.ClientId = "mokapi-mcp" + result := &Kafka{ Name: name, Type: "kafka", info: api, - client: engine.NewKafkaClient(m.app), + client: client, } for it := api.Servers.Iter(); it.Next(); { b := it.Value() @@ -197,33 +200,30 @@ func (k *Kafka) GetTopic(name string) (Topic, error) { return t, nil } -func (t *Topic) Produce(partition int, value string, key string, headers map[string]string) error { - msg := common.KafkaMessage{ - Value: []byte(value), - Data: nil, - Headers: headers, - Partition: partition, - } - if key != "" { - msg.Key = []byte(key) +func (t *Topic) Produce(partition int, value any, key string, headers map[string]string) error { + var h []store.RecordHeader + for hk, hv := range headers { + h = append(h, store.RecordHeader{Name: hk, Value: hv}) } - _, err := t.client.Produce(&common.KafkaProduceArgs{ - Cluster: t.info.Info.Name, - Topic: t.Name, - Messages: []common.KafkaMessage{msg}, - Retry: common.KafkaProduceRetry{ - MaxRetryTime: 3 * time.Minute, - InitialRetryTime: 500 * time.Millisecond, - Retries: 10, - Factor: 2, + result, err := t.client.Write(t.Name, []store.Record{ + { + Key: key, + Value: value, + Partition: partition, + Headers: h, }, - ClientId: "mokapi-mcp", - }) + }, media.ContentType{}) + if err != nil { return err } + if len(result) > 0 && result[0].Error != "" { + return errors.New(result[0].Error) + } + + // update JS topic and partition topic := t.info.Store.Topic(t.Name) if topic == nil { return fmt.Errorf("topic '%s' not found", t.Name) @@ -237,6 +237,7 @@ func (t *Topic) Produce(partition int, value string, key string, headers map[str pt.Offset = p.Offset() } } + return nil } diff --git a/mcp/run_kafka_test.go b/mcp/run_kafka_test.go index 1ca471b24..f0b42d53c 100644 --- a/mcp/run_kafka_test.go +++ b/mcp/run_kafka_test.go @@ -330,7 +330,7 @@ lastMessage context.Background(), mcp.RunInput{ Code: `const t = mokapi.getApi('foo').getTopic('channel-1') -t.produce(0, '{"foo":"bar"}', 'foo'); +t.produce(0, {foo: 'bar'}, 'foo'); t`, }, ) @@ -341,6 +341,65 @@ t`, require.Equal(t, int64(1), topic.Partitions[0].Offset) }, }, + { + name: "produce into topic but invalid message", + app: func() *runtime.App { + msg := asyncapi3test.NewMessage( + asyncapi3test.WithMessageName("msg-name-1"), + asyncapi3test.WithMessageTitle("msg-title-1"), + asyncapi3test.WithMessageSummary("msg-summary-1"), + asyncapi3test.WithMessageDescription("msg-description-1"), + asyncapi3test.WithContentType("application/json"), + asyncapi3test.WithPayload( + schematest.New("object", + schematest.WithProperty("foo", schematest.New("string")), + ), + ), + ) + + ch := asyncapi3test.NewChannel( + asyncapi3test.WithChannelTitle("title-1"), + asyncapi3test.WithChannelSummary("channel-1 summary"), + asyncapi3test.WithChannelDescription("description"), + asyncapi3test.UseMessage("foo", &asyncapi3.MessageRef{Value: msg}), + ) + + return runtimetest.NewKafkaApp( + asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "", ""), + asyncapi3test.AddChannel("channel-1", ch), + asyncapi3test.WithOperation("publish", + asyncapi3test.WithOperationAction("send"), + asyncapi3test.WithOperationTitle("op-title-1"), + asyncapi3test.WithOperationSummary("op-summary-1"), + asyncapi3test.WithOperationDescription("op-description-1"), + asyncapi3test.WithOperationChannel(ch), + asyncapi3test.UseOperationMessage(msg), + ), + asyncapi3test.WithOperation("consume", + asyncapi3test.WithOperationAction("receive"), + asyncapi3test.WithOperationChannel(ch), + asyncapi3test.UseOperationMessage(msg), + ), + ), + ) + }(), + test: func(t *testing.T, s *mcp.Service) { + _, err := s.GetRunResponse( + context.Background(), + mcp.RunInput{ + Code: `const t = mokapi.getApi('foo').getTopic('channel-1') +t.produce(0, { foo: 123 }, 'foo'); +t`, + }, + ) + require.EqualError(t, err, `no matching message configuration found for the given value: {"foo":123} +hint: +encoding data to 'application/json' failed: error count 1: + - #/foo/type: invalid type, expected string but got integer +`) + }, + }, } for _, tc := range testcases { diff --git a/mcp/run_test.go b/mcp/run_test.go index 2a84478eb..1a9e94207 100644 --- a/mcp/run_test.go +++ b/mcp/run_test.go @@ -2,12 +2,21 @@ package mcp_test import ( "context" + "mokapi/config/dynamic" + "mokapi/config/dynamic/dynamictest" + "mokapi/config/static" "mokapi/mcp" + "mokapi/providers/openapi" "mokapi/providers/openapi/openapitest" "mokapi/runtime" + "mokapi/runtime/events" "mokapi/runtime/runtimetest" + "mokapi/runtime/search" + "mokapi/safe" "mokapi/schema/json/generator" + "net/http" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -92,3 +101,202 @@ To ensure you are using the correct global variables and methods: }) } } + +func TestService_Run_Search(t *testing.T) { + testcases := []struct { + name string + app *runtime.App + test func(t *testing.T, s *mcp.Service, app *runtime.App) + }{ + { + name: "search", + app: func() *runtime.App { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + app.Http.Add(&dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: openapitest.NewConfig("3.1.0", + openapitest.WithInfo("foo", "", ""), + openapitest.WithPath("/foo/payment/bar", + openapitest.WithOperation(http.MethodPost, + openapitest.WithOperationSummary("Payment summary"), + ), + ), + ), + }) + return app + }(), + test: func(t *testing.T, s *mcp.Service, app *runtime.App) { + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + waitSearchIndex(t, func() bool { + r, err := app.Search(search.Request{QueryText: "foo", Limit: 10}) + require.NoError(t, err) + return len(r.Results) > 0 + }) + r, err := s.GetRunResponse( + context.Background(), + mcp.RunInput{ + Code: `mokapi.search("method:POST AND path:*payments*")`, + }, + ) + require.NoError(t, err) + require.Equal(t, mcp.SearchResult{Items: []mcp.SearchResultItem{ + { + Type: "HTTP", + Domain: "foo", + Title: "/foo/payment/bar", + Fragments: []string{"POST"}, + Metadata: map[string]string{"method": "POST", "path": "/foo/payment/bar", "service": "foo", "type": "http"}, + Time: "", + }}, Total: 1}, r.Result) + + r, err = s.GetRunResponse( + context.Background(), + mcp.RunInput{Code: `const searchResponse = mokapi.search("method:POST AND path:*payments*"); +let op = null +if (searchResponse.items.length > 0) { + const match = searchResponse.items[0]; + const api = mokapi.getApi(match.metadata.service); + const opSummary = api.getOperations().find(x => x.path === match.metadata.path && x.method === match.metadata.method); + if (opSummary) { + op = api.getOperation(opSummary.id); + } +} +op`}, + ) + require.NoError(t, err) + require.IsType(t, &mcp.Operation{}, r.Result) + op := r.Result.(*mcp.Operation) + require.Equal(t, "/foo/payment/bar", op.Path) + }, + }, + { + name: "search for event", + app: func() *runtime.App { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + app.Http.Add(&dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: openapitest.NewConfig("3.1.0", + openapitest.WithInfo("Petstore", "", ""), + openapitest.WithPath("/pets", + openapitest.WithOperation(http.MethodPost, + openapitest.WithOperationSummary("Payment summary"), + ), + ), + ), + }) + err := app.Events.Push(&openapi.HttpLog{ + Request: &openapi.HttpRequestLog{ + Method: http.MethodPost, + Url: "/pets", + }, + Response: &openapi.HttpResponseLog{ + StatusCode: http.StatusNotFound, + }, + }, events.NewTraits().WithNamespace("http").WithName("Petstore")) + require.NoError(t, err) + err = app.Events.Push(&openapi.HttpLog{ + Request: &openapi.HttpRequestLog{ + Method: http.MethodDelete, + Url: "/pets", + }, + Response: &openapi.HttpResponseLog{ + StatusCode: http.StatusInternalServerError, + }, + }, events.NewTraits().WithNamespace("http").WithName("Petstore")) + require.NoError(t, err) + err = app.Events.Push(&openapi.HttpLog{ + Request: &openapi.HttpRequestLog{ + Method: http.MethodGet, + Url: "/pets", + }, + Response: &openapi.HttpResponseLog{ + StatusCode: http.StatusOK, + }, + }, events.NewTraits().WithNamespace("http").WithName("Petstore")) + require.NoError(t, err) + return app + }(), + test: func(t *testing.T, s *mcp.Service, app *runtime.App) { + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + waitSearchIndex(t, func() bool { + r, err := app.Search(search.Request{QueryText: "Petstore", Limit: 10}) + require.NoError(t, err) + return len(r.Results) > 0 + }) + r, err := s.GetRunResponse( + context.Background(), + mcp.RunInput{ + Code: `const errors = mokapi.search('+api:"Petstore" +response.statusCode:>=400 +type:event') +errors.items.map(x => mokapi.getEvent(x.metadata.id))`, + }, + ) + require.NoError(t, err) + require.IsType(t, []any{}, r.Result) + evts := r.Result.([]any) + require.Len(t, evts, 2) + evt := evts[0] + require.IsType(t, events.Event{}, evt) + d1 := evts[0].(events.Event).Data.(*openapi.HttpLog) + d2 := evts[1].(events.Event).Data.(*openapi.HttpLog) + var evt404 *openapi.HttpLog + var evt500 *openapi.HttpLog + if d1.Response.StatusCode == http.StatusNotFound { + evt404 = d1 + evt500 = d2 + } else if d2.Response.StatusCode == http.StatusNotFound { + evt404 = d2 + evt500 = d1 + } + + require.NotNil(t, evt404) + require.NotNil(t, evt500) + + require.Equal(t, http.MethodPost, evt404.Request.Method) + require.Equal(t, http.MethodDelete, evt500.Request.Method) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + generator.Seed(123456) + + s := mcp.NewService(tc.app) + tc.test(t, s, tc.app) + }) + } +} + +func waitSearchIndex(t *testing.T, check func() bool) { + deadline := time.Now().Add(2 * time.Second) + + for { + if check() { + break + } + if time.Now().After(deadline) { + t.Fatal("wait search index reached deadline") + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/mcp/send_http_request.go b/mcp/send_http_request.go index ec621a60c..5a8f57061 100644 --- a/mcp/send_http_request.go +++ b/mcp/send_http_request.go @@ -103,7 +103,7 @@ Returns the full response including status code, headers, and body.`, func (s *Service) SendHttpRequest(_ context.Context, in SendHttpRequestInput) (SendHttpRequestResponse, error) { result := SendHttpRequestResponse{Headers: make(map[string][]string)} - info := s.app.GetHttp(in.APIName) + info := s.app.Http.Get(in.APIName) if info == nil { return result, fmt.Errorf("API '%s' not found", in.APIName) } diff --git a/mqtt/connect.go b/mqtt/connect.go index ec96ae631..3f2f79811 100644 --- a/mqtt/connect.go +++ b/mqtt/connect.go @@ -15,12 +15,10 @@ type ConnectRequest struct { Message []byte Username string Password string + Properties Properties } -type ConnectHeader struct { -} - -func (r *ConnectRequest) Read(d *Decoder) { +func (r *ConnectRequest) Read(d *Decoder, _ *Header) { r.Protocol = d.ReadString() r.Version = d.ReadByte() @@ -33,6 +31,11 @@ func (r *ConnectRequest) Read(d *Decoder) { r.CleanSession = (b>>1)&0x1 > 0 r.KeepAlive = d.ReadInt16() + if r.Version == 5 { + r.Properties = Properties{} + r.Properties.Read(d) + } + r.ClientId = d.ReadString() if r.WillFlag { @@ -48,7 +51,7 @@ func (r *ConnectRequest) Read(d *Decoder) { } } -func (r *ConnectRequest) Write(e *Encoder) { +func (r *ConnectRequest) Write(e *Encoder, _ *Header) { e.writeString(r.Protocol) e.writeByte(r.Version) b := byte(0) @@ -61,7 +64,7 @@ func (r *ConnectRequest) Write(e *Encoder) { if r.WillRetain { b |= 0x1 << 5 } - b |= (r.WillQoS & 0x03) << 1 + b |= (r.WillQoS & 0x3) << 3 if r.WillFlag { b |= 0x1 << 2 } @@ -70,8 +73,19 @@ func (r *ConnectRequest) Write(e *Encoder) { } e.writeByte(b) e.writeInt16(r.KeepAlive) + + if r.Version == 5 { + r.Properties.Write(e) + } + e.writeString(r.ClientId) + if r.WillFlag { + e.writeString(r.Topic) + e.writeInt16(int16(len(r.Message))) + e.Write(r.Message) + } + if r.HasUsername { e.writeString(r.Username) } @@ -82,21 +96,30 @@ func (r *ConnectRequest) Write(e *Encoder) { type ConnectResponse struct { SessionPresent bool - ReturnCode Code + ReasonCode Code + Properties Properties } -func (r *ConnectResponse) Write(e *Encoder) { +func (r *ConnectResponse) Write(e *Encoder, _ *Header) { if r.SessionPresent { - e.writeByte(0x01) + e.writeByte(0x1) } else { e.writeByte(0x0) } - e.writeByte(r.ReturnCode.Code) + e.writeByte(r.ReasonCode.Code) + if e.IsV5() { + r.Properties.Write(e) + } } -func (r *ConnectResponse) Read(d *Decoder) { - r.SessionPresent = d.ReadByte() == 0x01 - r.ReturnCode = Code{ +func (r *ConnectResponse) Read(d *Decoder, _ *Header) { + r.SessionPresent = d.ReadByte() == 0x1 + r.ReasonCode = Code{ Code: d.ReadByte(), } + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } } diff --git a/mqtt/connect_test.go b/mqtt/connect_test.go index 2043ee0c5..841a6f7a8 100644 --- a/mqtt/connect_test.go +++ b/mqtt/connect_test.go @@ -3,11 +3,12 @@ package mqtt_test import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/try" "testing" + + "github.com/stretchr/testify/require" ) func TestConnect_ReadRequest(t *testing.T) { @@ -19,10 +20,10 @@ func TestConnect_ReadRequest(t *testing.T) { { name: "simple connect", in: []byte{ - 0x10, // flags + 0x10, // Packet type 0x10, // length 0x00, 0x04, // protocol length - 0x4d, 0x51, 0x54, 0x54, // protocol + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT 0x04, // version 0x02, // connect flags 0x00, 0x3c, // keep alive @@ -53,10 +54,10 @@ func TestConnect_ReadRequest(t *testing.T) { { name: "connect with topic and message", in: []byte{ - 0x10, // flags + 0x10, // Packet type 0x1A, // length 0x00, 0x04, // protocol length - 0x4d, 0x51, 0x54, 0x54, // protocol + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT 0x04, // version 0x0e, // connect flags 0x00, 0x3c, // keep alive @@ -90,6 +91,45 @@ func TestConnect_ReadRequest(t *testing.T) { require.Equal(t, []byte("bar"), msg.Message) }, }, + { + name: "connect v5", + in: []byte{ + 0x10, // Packet type + 0x18, // length + 0x00, 0x04, // protocol length + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT + 0x05, // version + 0x02, // connect flags + 0x00, 0x3c, // keep alive + 0x5, // properties length + 0x11, // Session Expiry Interval + 0x0, 0x0, 0x0, 0x0, // value + 0x00, 0x06, // client id length + 'm', 'o', 'k', 'a', 'p', 'i', // client id + }, + test: func(t *testing.T, r *mqtt.Message, err error) { + require.NoError(t, err) + + require.Equal(t, 24, r.Header.Size) + + require.IsType(t, &mqtt.ConnectRequest{}, r.Payload) + msg := r.Payload.(*mqtt.ConnectRequest) + + require.Equal(t, "MQTT", msg.Protocol) + require.Equal(t, byte(5), msg.Version) + + require.False(t, msg.HasUsername) + require.False(t, msg.HasPassword) + require.False(t, msg.WillRetain) + require.Equal(t, byte(0), msg.WillQoS) + require.False(t, msg.WillFlag) + require.True(t, msg.CleanSession) + require.Equal(t, int16(60), msg.KeepAlive) + require.Contains(t, msg.Properties, mqtt.SessionExpiryInterval) + require.Equal(t, int32(0), msg.Properties[mqtt.SessionExpiryInterval]) + require.Equal(t, "mokapi", msg.ClientId) + }, + }, } t.Parallel() @@ -99,12 +139,126 @@ func TestConnect_ReadRequest(t *testing.T) { t.Parallel() r := &mqtt.Message{} - err := r.Read(bytes.NewReader(tc.in)) + err := r.Read(bytes.NewReader(tc.in), &mqtt.ClientContext{}) tc.test(t, r, err) }) } } +func TestConnect_WriteRequest(t *testing.T) { + testcases := []struct { + name string + msg mqtt.Message + out []byte + }{ + { + name: "simple connect", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.CONNECT, + }, + Payload: &mqtt.ConnectRequest{ + Protocol: "MQTT", + Version: 4, + CleanSession: true, + KeepAlive: 60, + ClientId: "mqtt", + }, + }, + out: []byte{ + 0x10, // Packet type + 0x10, // length + 0x00, 0x04, // protocol length + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT + 0x04, // version + 0x02, // connect flags + 0x00, 0x3c, // keep alive + 0x00, 0x04, // client id length + 0x6d, 0x71, 0x74, 0x74, // client id + }, + }, + { + name: "connect with topic and message", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.CONNECT, + }, + Payload: &mqtt.ConnectRequest{ + Protocol: "MQTT", + Version: 4, + WillQoS: 1, + CleanSession: true, + WillFlag: true, + KeepAlive: 60, + ClientId: "mqtt", + Topic: "foo", + Message: []byte("bar"), + }, + }, + out: []byte{ + 0x10, // Packet type + 0x1A, // length + 0x00, 0x04, // protocol length + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT + 0x04, // version + 0x0e, // connect flags + 0x00, 0x3c, // keep alive + 0x00, 0x04, // client id length + 0x6d, 0x71, 0x74, 0x74, // client id + 0x00, 0x03, // topic length + 'f', 'o', 'o', // topic + 0x00, 0x03, // message length + 'b', 'a', 'r', // message + }, + }, + { + name: "connect v5", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.CONNECT, + }, + Payload: &mqtt.ConnectRequest{ + Protocol: "MQTT", + Version: 5, + CleanSession: true, + KeepAlive: 60, + ClientId: "mokapi", + Properties: mqtt.Properties{ + mqtt.SessionExpiryInterval: int32(0), + }, + }, + }, + out: []byte{ + 0x10, // Packet type + 0x18, // length + 0x00, 0x04, // protocol length + 0x4d, 0x51, 0x54, 0x54, // protocol name MQTT + 0x05, // version + 0x02, // connect flags + 0x00, 0x3c, // keep alive + 0x5, // properties length + 0x11, // Session Expiry Interval + 0x0, 0x0, 0x0, 0x0, // value + 0x00, 0x06, // client id length + 'm', 'o', 'k', 'a', 'p', 'i', // client id + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + err := tc.msg.Write(&b, &mqtt.ClientContext{}) + require.NoError(t, err) + require.Equal(t, tc.out, b.Bytes()) + }) + } +} + func TestConnect(t *testing.T) { testcases := []struct { name string @@ -120,7 +274,7 @@ func TestConnect(t *testing.T) { }, Payload: &mqtt.ConnectResponse{ SessionPresent: false, - ReturnCode: mqtt.Accepted, + ReasonCode: mqtt.Success, }, }) }), diff --git a/mqtt/context.go b/mqtt/context.go index 3da5d7a15..af9a86cf1 100644 --- a/mqtt/context.go +++ b/mqtt/context.go @@ -9,18 +9,14 @@ import ( const clientKey = "client" type ClientContext struct { - Addr string - ClientId string + Addr string + ClientId string + ProtocolVersion byte + ServerAddress string conn net.Conn } -type packet struct { - header *Header - payload buffer.Buffer - retries int -} - func ClientFromContext(ctx context.Context) *ClientContext { if ctx == nil { return nil @@ -30,14 +26,14 @@ func ClientFromContext(ctx context.Context) *ClientContext { } func NewClientContext(ctx context.Context, conn net.Conn) context.Context { - return context.WithValue(ctx, clientKey, &ClientContext{Addr: conn.RemoteAddr().String(), conn: conn}) + return context.WithValue(ctx, clientKey, &ClientContext{Addr: conn.RemoteAddr().String(), ServerAddress: conn.LocalAddr().String(), conn: conn}) } func (c *ClientContext) Send(r *Message) error { b := buffer.NewPageBuffer() - e := NewEncoder(b) - r.Payload.Write(e) + e := NewEncoder(b, c.ProtocolVersion) + r.Payload.Write(e, r.Header) r.Header.Size = b.Size() diff --git a/mqtt/decode.go b/mqtt/decode.go index c664db540..f0afa1f92 100644 --- a/mqtt/decode.go +++ b/mqtt/decode.go @@ -7,14 +7,19 @@ import ( ) type Decoder struct { - reader io.Reader - buffer [8]byte - err error - leftSize int + reader io.Reader + buffer [8]byte + err error + leftSize int + protocolVersion byte } -func NewDecoder(reader io.Reader, size int) *Decoder { - return &Decoder{reader: reader, leftSize: size} +func NewDecoder(reader io.Reader, size int, protocolVersion byte) *Decoder { + return &Decoder{reader: reader, leftSize: size, protocolVersion: protocolVersion} +} + +func (d *Decoder) IsV5() bool { + return d.protocolVersion == 5 } func (d *Decoder) ReadByte() byte { @@ -31,7 +36,7 @@ func (d *Decoder) ReadByte() byte { } func (d *Decoder) ReadString() string { - if n := d.ReadInt16(); n < 0 { + if n := d.ReadInt16(); n == 0 { return "" } else { b := make([]byte, n) @@ -63,6 +68,24 @@ func (d *Decoder) ReadInt16() int16 { } } +func (d *Decoder) ReadUInt16() uint16 { + if d.readFull(d.buffer[:2]) { + i := binary.BigEndian.Uint16(d.buffer[:2]) + return uint16(i) + } else { + return 0 + } +} + +func (d *Decoder) ReadInt32() int32 { + if d.readFull(d.buffer[:4]) { + i := binary.BigEndian.Uint16(d.buffer[:4]) + return int32(i) + } else { + return 0 + } +} + func (d *Decoder) readFull(b []byte) bool { if d.err != nil { return false @@ -103,3 +126,25 @@ func (d *Decoder) readRemainingLength() int { d.err = fmt.Errorf("malformed Remaining Length: exceeds 4 bytes") return 0 } + +func (d *Decoder) ReadVariableInt() int { + var multiplier = 1 + var value = 0 + + for { + b := d.ReadByte() + + value += int(b&127) * multiplier + + if multiplier > 128*128*128 { + panic("Malformed Variable Byte Integer") + } + multiplier *= 128 + + if b&128 == 0 { + break + } + } + + return value +} diff --git a/mqtt/disconnect.go b/mqtt/disconnect.go new file mode 100644 index 000000000..d7c3ab5d9 --- /dev/null +++ b/mqtt/disconnect.go @@ -0,0 +1,29 @@ +package mqtt + +type DisconnectReason uint8 + +const ( + DisconnectNormal DisconnectReason = iota + DisconnectWithWillMessage DisconnectReason = 4 +) + +type DisconnectRequest struct { + Reason DisconnectReason + Properties Properties +} + +func (r *DisconnectRequest) Read(d *Decoder, _ *Header) { + if d.leftSize > 0 { + r.Reason = DisconnectReason(d.ReadByte()) + } + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } +} + +func (r *DisconnectRequest) Write(e *Encoder, _ *Header) { + if r.Reason != 0 { + e.writeByte(uint8(r.Reason)) + } +} diff --git a/mqtt/disconnect_test.go b/mqtt/disconnect_test.go new file mode 100644 index 000000000..baa1cb840 --- /dev/null +++ b/mqtt/disconnect_test.go @@ -0,0 +1,65 @@ +package mqtt_test + +import ( + "bytes" + "mokapi/mqtt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDisconnect_ReadRequest(t *testing.T) { + testcases := []struct { + name string + in []byte + ctx *mqtt.ClientContext + test func(t *testing.T, r *mqtt.Message, err error) + }{ + { + name: "disconnect no reason", + in: []byte{ + 0xE0, // Protocol Type + 0x0, // length + }, + ctx: &mqtt.ClientContext{}, + test: func(t *testing.T, r *mqtt.Message, err error) { + require.NoError(t, err) + require.IsType(t, &mqtt.DisconnectRequest{}, r.Payload) + msg := r.Payload.(*mqtt.DisconnectRequest) + require.Equal(t, mqtt.DisconnectNormal, msg.Reason) + }, + }, + { + name: "disconnect no reason", + in: []byte{ + 0xE0, // Protocol Type + 0x8, // Remaining Length: 1 (Reason) + 1 (PropLen) + 3 (StrLen) + 3 (Data) = 8 + 0x80, // Reason: Unspecified error + 0x06, // Property length + 0x1F, // Property ID: Reason String + 0x0, 0x03, // String length + 'f', 'o', 'o', + }, + ctx: &mqtt.ClientContext{ProtocolVersion: 5}, + test: func(t *testing.T, r *mqtt.Message, err error) { + require.NoError(t, err) + require.IsType(t, &mqtt.DisconnectRequest{}, r.Payload) + msg := r.Payload.(*mqtt.DisconnectRequest) + require.Equal(t, mqtt.DisconnectReason(128), msg.Reason) + require.Equal(t, "foo", msg.Properties.ReasonString()) + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r := &mqtt.Message{} + err := r.Read(bytes.NewReader(tc.in), tc.ctx) + tc.test(t, r, err) + }) + } +} diff --git a/mqtt/encode.go b/mqtt/encode.go index 6bcf4a380..d382e2267 100644 --- a/mqtt/encode.go +++ b/mqtt/encode.go @@ -7,12 +7,17 @@ import ( ) type Encoder struct { - writer io.Writer - buffer [32]byte + writer io.Writer + buffer [32]byte + protocolVersion byte } -func NewEncoder(writer io.Writer) *Encoder { - return &Encoder{writer: writer} +func NewEncoder(writer io.Writer, protocolVersion byte) *Encoder { + return &Encoder{writer: writer, protocolVersion: protocolVersion} +} + +func (e *Encoder) IsV5() bool { + return e.protocolVersion == 5 } func (e *Encoder) writeByte(b byte) { @@ -31,6 +36,22 @@ func (e *Encoder) writeInt16(i int16) { } } +func (e *Encoder) writeUInt16(i uint16) { + binary.BigEndian.PutUint16(e.buffer[:2], i) + _, err := e.writer.Write(e.buffer[:2]) + if err != nil { + panic(err) + } +} + +func (e *Encoder) writeInt32(i int32) { + binary.BigEndian.PutUint32(e.buffer[:4], uint32(i)) + _, err := e.writer.Write(e.buffer[:4]) + if err != nil { + panic(err) + } +} + func (e *Encoder) Write(b []byte) { _, err := e.writer.Write(b) if err != nil { @@ -50,6 +71,23 @@ func (e *Encoder) writeString(s string) { } } +func (e *Encoder) WriteVariableInt(value int) { + for { + encodedByte := byte(value % 128) + value = value / 128 + + if value > 0 { + encodedByte |= 128 + } + + e.writeByte(encodedByte) + + if value <= 0 { + break + } + } +} + func encodeBool(b bool) byte { if b { return 1 diff --git a/mqtt/message.go b/mqtt/message.go index dabcd98b6..2f7281a23 100644 --- a/mqtt/message.go +++ b/mqtt/message.go @@ -14,8 +14,8 @@ type Message struct { } type Payload interface { - Write(e *Encoder) - Read(*Decoder) + Write(e *Encoder, h *Header) + Read(d *Decoder, h *Header) } type MessageOptions func(*Message) @@ -25,12 +25,15 @@ func (m *Message) WithContext(ctx context.Context) *Message { return m } -func (m *Message) Write(w io.Writer) error { +func (m *Message) Write(w io.Writer, ctx *ClientContext) error { b := buffer.NewPageBuffer() defer b.Unref() - e := NewEncoder(b) - m.Payload.Write(e) + e := NewEncoder(b, ctx.ProtocolVersion) + if m.Payload == nil { + return fmt.Errorf("mqtt: message has no payload") + } + m.Payload.Write(e, m.Header) m.Header.Size = b.Size() err := m.Header.Write(w) @@ -42,8 +45,8 @@ func (m *Message) Write(w io.Writer) error { return err } -func (m *Message) Read(reader io.Reader) error { - d := NewDecoder(reader, 5) +func (m *Message) Read(reader io.Reader, ctx *ClientContext) error { + d := NewDecoder(reader, 5, ctx.ProtocolVersion) m.Header = readHeader(d) if d.err != nil { return d.err @@ -67,11 +70,17 @@ func (m *Message) Read(reader io.Reader) error { m.Payload = &UnsubscribeRequest{} case UNSUBACK: m.Payload = &UnsubscribeResponse{} + case PINGREQ: + m.Payload = &PingRequest{} + case PINGRESP: + m.Payload = &PingResponse{} + case DISCONNECT: + m.Payload = &DisconnectRequest{} default: return fmt.Errorf("unknown MQTT protocol type %d", m.Header.Type) } - m.Payload.Read(d) + m.Payload.Read(d, m.Header) if d.err != nil { return d.err @@ -81,5 +90,10 @@ func (m *Message) Read(reader io.Reader) error { return fmt.Errorf("mqtt: remaining length %d is not zero", d.leftSize) } + if m.Header.Type == CONNECT { + c := m.Payload.(*ConnectRequest) + ctx.ProtocolVersion = c.Version + } + return nil } diff --git a/mqtt/mqtttest/client.go b/mqtt/mqtttest/client.go index 0c726b2d8..2015b8338 100644 --- a/mqtt/mqtttest/client.go +++ b/mqtt/mqtttest/client.go @@ -33,14 +33,14 @@ func (c *Client) Send(m *mqtt.Message) (*mqtt.Message, error) { return nil, err } - err := m.Write(c.conn) + err := m.Write(c.conn, &mqtt.ClientContext{}) if err != nil { return nil, err } - c.conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) + _ = c.conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) res := &mqtt.Message{} - err = res.Read(c.conn) + err = res.Read(c.conn, &mqtt.ClientContext{}) return res, err } @@ -49,7 +49,7 @@ func (c *Client) SendNoResponse(r *mqtt.Message) error { return err } - return r.Write(c.conn) + return r.Write(c.conn, &mqtt.ClientContext{}) } func (c *Client) Recv() (*mqtt.Message, error) { @@ -58,7 +58,7 @@ func (c *Client) Recv() (*mqtt.Message, error) { } res := &mqtt.Message{} - err := res.Read(c.conn) + err := res.Read(c.conn, &mqtt.ClientContext{}) return res, err } diff --git a/mqtt/ping.go b/mqtt/ping.go new file mode 100644 index 000000000..379d0d937 --- /dev/null +++ b/mqtt/ping.go @@ -0,0 +1,13 @@ +package mqtt + +type PingRequest struct{} + +type PingResponse struct{} + +func (r *PingRequest) Read(_ *Decoder, _ *Header) {} + +func (r *PingRequest) Write(_ *Encoder, _ *Header) {} + +func (r *PingResponse) Read(_ *Decoder, _ *Header) {} + +func (r *PingResponse) Write(_ *Encoder, _ *Header) {} diff --git a/mqtt/property.go b/mqtt/property.go new file mode 100644 index 000000000..db6a3ab97 --- /dev/null +++ b/mqtt/property.go @@ -0,0 +1,83 @@ +package mqtt + +import "bytes" + +const ( + PayloadFormatIndicator byte = 0x1 + ContentType byte = 0x3 + SessionExpiryInterval byte = 0x11 + ReasonString byte = 0x1F + UserProperty byte = 0x26 +) + +type Properties map[byte]any + +func (p Properties) Read(d *Decoder) { + propLen := d.ReadVariableInt() + if propLen == 0 { + return + } + + stopAt := d.leftSize - propLen + for d.leftSize > stopAt && d.leftSize > 0 { + propID := d.ReadByte() + switch propID { + case SessionExpiryInterval: + p[propID] = d.ReadInt32() + case ReasonString, ContentType: + p[propID] = d.ReadString() + case UserProperty: + if p[propID] == nil { + p[propID] = map[string]string{} + } + key := d.ReadString() + val := d.ReadString() + p[propID].(map[string]string)[key] = val + } + } +} + +func (p Properties) Write(e *Encoder) { + if len(p) == 0 { + e.WriteVariableInt(0) + return + } + + var b bytes.Buffer + propBuffer := NewEncoder(&b, e.protocolVersion) + for id, val := range p { + propBuffer.writeByte(id) + + switch v := val.(type) { + case string: + propBuffer.writeString(v) + case int32: + propBuffer.writeInt32(v) + } + } + + e.WriteVariableInt(b.Len()) + e.Write(b.Bytes()) +} + +func (p Properties) SessionExpiryInterval() int32 { + if p == nil { + return 0 + } + v, ok := p[SessionExpiryInterval] + if !ok { + return 0 + } + return v.(int32) +} + +func (p Properties) ReasonString() string { + if p == nil { + return "" + } + v, ok := p[ReasonString] + if !ok { + return "" + } + return v.(string) +} diff --git a/mqtt/protocol.go b/mqtt/protocol.go index 217867470..ce5d70e06 100644 --- a/mqtt/protocol.go +++ b/mqtt/protocol.go @@ -35,8 +35,8 @@ func readHeader(d *Decoder) *Header { b := d.ReadByte() h.Type = Type(b >> 4) h.Dup = (b>>3)&0x01 > 0 - h.QoS = (b >> 0x06) >> 1 - h.Retain = b&0x01 != 0 + h.QoS = (b >> 1) & 0x03 + h.Retain = (b & 0x01) > 0 h.Size = d.readRemainingLength() d.leftSize = h.Size @@ -61,13 +61,14 @@ func (h *Header) with(messageType Type, size int) *Header { } type Code struct { - Reason string - Code byte + Reason string `json:"reason"` + Code byte `json:"code"` } var ( - Accepted = Code{Code: 0x00, Reason: "accepted"} + Success = Code{Code: 0x00, Reason: "Success"} ErrUnsupportedProtocolVersion = Code{Code: 0x01, Reason: "unacceptable protocol version"} ErrIdentifierRejected = Code{Code: 0x02, Reason: "identifier rejected"} ErrUnspecifiedError = Code{Code: 0x80, Reason: "unspecified error"} + ErrTopicNameInvalid = Code{Code: byte(TopicNameInvalid), Reason: "topic name invalid"} ) diff --git a/mqtt/publish.go b/mqtt/publish.go index 800eb7fab..8b0ad7935 100644 --- a/mqtt/publish.go +++ b/mqtt/publish.go @@ -1,32 +1,69 @@ package mqtt +type PublishReason byte + +const ( + PublishSuccess PublishReason = 0 + TopicNameInvalid PublishReason = 144 + PayloadFormatInvalid PublishReason = 153 +) + type PublishRequest struct { - Topic string - MessageId int16 - Data []byte + Topic string + MessageId uint16 + Data []byte + Properties Properties } -func (r *PublishRequest) Read(d *Decoder) { +func (r *PublishRequest) Read(d *Decoder, h *Header) { r.Topic = d.ReadString() - r.MessageId = d.ReadInt16() + + if h.QoS > 0 { + r.MessageId = d.ReadUInt16() + } + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } + r.Data = make([]byte, d.leftSize) d.readFull(r.Data) } -func (r *PublishRequest) Write(e *Encoder) { +func (r *PublishRequest) Write(e *Encoder, h *Header) { e.writeString(r.Topic) - e.writeInt16(r.MessageId) + if h.QoS > 0 { + e.writeUInt16(r.MessageId) + } + + if e.IsV5() { + r.Properties.Write(e) + } + e.Write(r.Data) } type PublishResponse struct { - MessageId int16 + MessageId uint16 + ReasonCode PublishReason + Properties Properties } -func (r *PublishResponse) Read(d *Decoder) { - r.MessageId = d.ReadInt16() +func (r *PublishResponse) Read(d *Decoder, h *Header) { + r.MessageId = d.ReadUInt16() + r.ReasonCode = PublishReason(d.ReadByte()) + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } } -func (r *PublishResponse) Write(e *Encoder) { - e.writeInt16(r.MessageId) +func (r *PublishResponse) Write(e *Encoder, _ *Header) { + e.writeUInt16(r.MessageId) + e.writeByte(byte(r.ReasonCode)) + if e.IsV5() { + r.Properties.Write(e) + } } diff --git a/mqtt/publish_test.go b/mqtt/publish_test.go index c6e6c8398..ed8a296f0 100644 --- a/mqtt/publish_test.go +++ b/mqtt/publish_test.go @@ -3,11 +3,12 @@ package mqtt_test import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/try" "testing" + + "github.com/stretchr/testify/require" ) func TestPublish_ReadRequest(t *testing.T) { @@ -19,7 +20,28 @@ func TestPublish_ReadRequest(t *testing.T) { { name: "publish to foo", in: []byte{ - 0x30, // flags + 0x30, // Protocol Type + 0x8, // length + 0x00, 0x03, // topic length + 'f', 'o', 'o', // topic + 'b', 'a', 'r', // Payload + }, + test: func(t *testing.T, r *mqtt.Message, err error) { + require.NoError(t, err) + + require.Equal(t, 8, r.Header.Size) + + require.IsType(t, &mqtt.PublishRequest{}, r.Payload) + msg := r.Payload.(*mqtt.PublishRequest) + + require.Equal(t, "foo", msg.Topic) + require.Equal(t, "bar", string(msg.Data)) + }, + }, + { + name: "publish to foo with QoS", + in: []byte{ + 0x32, // Protocol Type 0xA, // length 0x00, 0x03, // topic length 'f', 'o', 'o', // Payload @@ -35,8 +57,8 @@ func TestPublish_ReadRequest(t *testing.T) { msg := r.Payload.(*mqtt.PublishRequest) require.Equal(t, "foo", msg.Topic) - require.Equal(t, int16(10), msg.MessageId) - require.Equal(t, []byte("bar"), msg.Data) + //require.Equal(t, int16(10), msg.MessageId) + require.Equal(t, "bar", string(msg.Data)) }, }, } @@ -48,12 +70,103 @@ func TestPublish_ReadRequest(t *testing.T) { t.Parallel() r := &mqtt.Message{} - err := r.Read(bytes.NewReader(tc.in)) + err := r.Read(bytes.NewReader(tc.in), &mqtt.ClientContext{}) tc.test(t, r, err) }) } } +func TestPublish_Write(t *testing.T) { + testcases := []struct { + name string + msg mqtt.Message + ctx *mqtt.ClientContext + out []byte + }{ + { + name: "request QoS=0", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PUBLISH, + }, + Payload: &mqtt.PublishRequest{ + Topic: "foo", + MessageId: uint16(123), + Data: []byte("bar"), + }, + }, + ctx: &mqtt.ClientContext{}, + out: []byte{ + 0x30, // Packet type + 0x08, // length + 0x0, 0x03, // topic length + 'f', 'o', 'o', + 'b', 'a', 'r', // data + }, + }, + { + name: "request QoS=1", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PUBLISH, + QoS: 1, + }, + Payload: &mqtt.PublishRequest{ + Topic: "foo", + MessageId: uint16(123), + Data: []byte("bar"), + }, + }, + ctx: &mqtt.ClientContext{}, + out: []byte{ + 0x32, // Packet type + 0x0a, // length + 0x0, 0x03, // topic length + 'f', 'o', 'o', + 0x0, 0x7b, // message id + 'b', 'a', 'r', // data + }, + }, + { + name: "request v5", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PUBLISH, + QoS: 1, + }, + Payload: &mqtt.PublishRequest{ + Topic: "foo", + MessageId: uint16(123), + Data: []byte("bar"), + }, + }, + ctx: &mqtt.ClientContext{ProtocolVersion: 5}, + out: []byte{ + 0x32, // Packet type + 0x0b, // length + 0x0, 0x03, // topic length + 'f', 'o', 'o', + 0x0, 0x7b, // message id + 0x0, // properties + 'b', 'a', 'r', // data + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + err := tc.msg.Write(&b, tc.ctx) + require.NoError(t, err) + require.Equal(t, tc.out, b.Bytes()) + }) + } +} + func TestPublish(t *testing.T) { testcases := []struct { name string @@ -78,6 +191,7 @@ func TestPublish(t *testing.T) { r := &mqtt.Message{ Header: &mqtt.Header{ Type: mqtt.PUBLISH, + QoS: 1, }, Payload: &mqtt.PublishRequest{ Topic: "foo", @@ -89,7 +203,7 @@ func TestPublish(t *testing.T) { require.NoError(t, err) require.Equal(t, mqtt.PUBACK, res.Header.Type) msg := res.Payload.(*mqtt.PublishResponse) - require.Equal(t, int16(10), msg.MessageId) + require.Equal(t, uint16(10), msg.MessageId) }, }, } diff --git a/mqtt/server.go b/mqtt/server.go index 7bf53e1e1..c81200a1f 100644 --- a/mqtt/server.go +++ b/mqtt/server.go @@ -2,14 +2,15 @@ package mqtt import ( "context" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "io" "mokapi/safe" "net" "runtime/debug" "sync" "syscall" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) var ErrServerClosed = errors.New("mqtt: Server closed") @@ -90,7 +91,7 @@ func (s *Server) serve(conn net.Conn, ctx context.Context) { client.conn = conn for { r := &Message{Context: ctx} - err := r.Read(conn) + err := r.Read(conn, client) if err != nil { switch { case err == io.EOF || errors.Is(err, net.ErrClosed) || errors.Is(err, syscall.ECONNRESET): @@ -103,6 +104,7 @@ func (s *Server) serve(conn net.Conn, ctx context.Context) { res := &messageWriter{ conn: conn, + ctx: client, } s.Handler.ServeMessage(res, r) @@ -161,8 +163,6 @@ func (s *Server) trackConn(conn net.Conn) context.Context { s.activeConn = make(map[net.Conn]context.Context) } - // delete conn struct and implement all in server - // NewClientContext(context, net.Conn) ctx := NewClientContext(context.Background(), conn) s.activeConn[conn] = ctx @@ -171,8 +171,9 @@ func (s *Server) trackConn(conn net.Conn) context.Context { type messageWriter struct { conn net.Conn + ctx *ClientContext } func (mw messageWriter) Write(msg *Message) error { - return msg.Write(mw.conn) + return msg.Write(mw.conn, mw.ctx) } diff --git a/mqtt/server_test.go b/mqtt/server_test.go new file mode 100644 index 000000000..85053f977 --- /dev/null +++ b/mqtt/server_test.go @@ -0,0 +1,69 @@ +package mqtt_test + +import ( + "errors" + "fmt" + "mokapi/mqtt" + "mokapi/mqtt/mqtttest" + "mokapi/try" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestServer(t *testing.T) { + testcases := []struct { + name string + handler func(t *testing.T) mqtt.HandlerFunc + test func(t *testing.T, c *mqtttest.Client) + }{ + { + name: "Ping", + handler: func(t *testing.T) mqtt.HandlerFunc { + return func(rw mqtt.MessageWriter, m *mqtt.Message) { + err := rw.Write(&mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PINGRESP, + }, + Payload: &mqtt.PingResponse{}, + }) + require.NoError(t, err) + } + }, + test: func(t *testing.T, c *mqtttest.Client) { + res, err := c.Send(&mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PINGREQ, + }, + Payload: &mqtt.PingRequest{}, + }) + require.NoError(t, err) + require.NotNil(t, res) + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + addr := fmt.Sprintf(":%d", try.GetFreePort()) + s := &mqtt.Server{ + Addr: addr, + Handler: tc.handler(t), + } + go func() { + err := s.ListenAndServe() + if err != nil && !errors.Is(err, mqtt.ErrServerClosed) { + panic(err) + } + }() + defer s.Close() + + c := mqtttest.NewClient(addr) + tc.test(t, c) + }) + } +} diff --git a/mqtt/subscribe.go b/mqtt/subscribe.go index b4371916c..ba2bfc483 100644 --- a/mqtt/subscribe.go +++ b/mqtt/subscribe.go @@ -1,8 +1,18 @@ package mqtt +type SubscriptionReason byte + +const ( + GrantedQoS0 SubscriptionReason = 0 + GrantedQoS1 SubscriptionReason = 1 + GrantedQoS2 SubscriptionReason = 2 + UnspecifiedError SubscriptionReason = 128 +) + type SubscribeRequest struct { - MessageId int16 - Topics []SubscribeTopic + MessageId uint16 + Topics []SubscribeTopic + Properties Properties } type SubscribeTopic struct { @@ -10,8 +20,13 @@ type SubscribeTopic struct { QoS byte } -func (r *SubscribeRequest) Read(d *Decoder) { - r.MessageId = d.ReadInt16() +func (r *SubscribeRequest) Read(d *Decoder, _ *Header) { + r.MessageId = d.ReadUInt16() + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } for d.leftSize > 0 { name := d.ReadString() @@ -23,8 +38,13 @@ func (r *SubscribeRequest) Read(d *Decoder) { } } -func (r *SubscribeRequest) Write(e *Encoder) { - e.writeInt16(r.MessageId) +func (r *SubscribeRequest) Write(e *Encoder, _ *Header) { + e.writeUInt16(r.MessageId) + + if e.IsV5() { + r.Properties.Write(e) + } + for _, t := range r.Topics { e.writeString(t.Name) e.writeByte(t.QoS) @@ -32,20 +52,33 @@ func (r *SubscribeRequest) Write(e *Encoder) { } type SubscribeResponse struct { - MessageId int16 - TopicQoS []byte + MessageId uint16 + ReasonCodes []SubscriptionReason + Properties Properties } -func (r *SubscribeResponse) Write(e *Encoder) { - e.writeInt16(r.MessageId) - for _, qos := range r.TopicQoS { - e.writeByte(qos) +func (r *SubscribeResponse) Write(e *Encoder, _ *Header) { + e.writeUInt16(r.MessageId) + + if e.IsV5() { + r.Properties.Write(e) + } + + for _, reason := range r.ReasonCodes { + e.writeByte(byte(reason)) } } -func (r *SubscribeResponse) Read(d *Decoder) { - r.MessageId = d.ReadInt16() +func (r *SubscribeResponse) Read(d *Decoder, _ *Header) { + r.MessageId = d.ReadUInt16() + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } + for d.leftSize > 0 { - r.TopicQoS = append(r.TopicQoS, d.ReadByte()) + code := d.ReadByte() + r.ReasonCodes = append(r.ReasonCodes, SubscriptionReason(code)) } } diff --git a/mqtt/subscribe_test.go b/mqtt/subscribe_test.go index ebac68cdb..44fb5c09f 100644 --- a/mqtt/subscribe_test.go +++ b/mqtt/subscribe_test.go @@ -3,11 +3,12 @@ package mqtt_test import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/try" "testing" + + "github.com/stretchr/testify/require" ) func TestSubscribe_ReadRequest(t *testing.T) { @@ -19,9 +20,9 @@ func TestSubscribe_ReadRequest(t *testing.T) { { name: "subscribe to foo", in: []byte{ - 0x82, // flags - 0x08, // length - 0x0, 0xA, // MessageId + 0x82, // Protocol Type + 0x08, // length + 0x0, 0x10, // Message Identifier 0x00, 0x03, // topic length 'f', 'o', 'o', // topic 0x1, // QoS @@ -34,6 +35,7 @@ func TestSubscribe_ReadRequest(t *testing.T) { require.IsType(t, &mqtt.SubscribeRequest{}, r.Payload) msg := r.Payload.(*mqtt.SubscribeRequest) + require.Equal(t, uint16(16), msg.MessageId) require.Len(t, msg.Topics, 1) require.Equal(t, "foo", msg.Topics[0].Name) require.Equal(t, byte(1), msg.Topics[0].QoS) @@ -48,12 +50,74 @@ func TestSubscribe_ReadRequest(t *testing.T) { t.Parallel() r := &mqtt.Message{} - err := r.Read(bytes.NewReader(tc.in)) + err := r.Read(bytes.NewReader(tc.in), &mqtt.ClientContext{}) tc.test(t, r, err) }) } } +func TestSubscribe_WriteResponse(t *testing.T) { + testcases := []struct { + name string + msg mqtt.Message + ctx *mqtt.ClientContext + out []byte + }{ + { + name: "simple subscribe", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.SUBACK, + }, + Payload: &mqtt.SubscribeResponse{ + MessageId: uint16(53902), + ReasonCodes: []mqtt.SubscriptionReason{mqtt.GrantedQoS2}, + }, + }, + ctx: &mqtt.ClientContext{}, + out: []byte{ + 0x90, // Packet type + 0x03, // length + 0xd2, 0x8e, // message id + 0x02, // reason + }, + }, + { + name: "subscribe v5", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.SUBACK, + }, + Payload: &mqtt.SubscribeResponse{ + MessageId: uint16(53902), + ReasonCodes: []mqtt.SubscriptionReason{mqtt.GrantedQoS2}, + }, + }, + ctx: &mqtt.ClientContext{ProtocolVersion: 5}, + out: []byte{ + 0x90, // Packet type + 0x04, // length + 0xd2, 0x8e, // message id + 0x00, // properties + 0x02, // reason + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + err := tc.msg.Write(&b, tc.ctx) + require.NoError(t, err) + require.Equal(t, tc.out, b.Bytes()) + }) + } +} + func TestSubscribe(t *testing.T) { testcases := []struct { name string @@ -68,9 +132,9 @@ func TestSubscribe(t *testing.T) { Type: mqtt.SUBACK, }, Payload: &mqtt.SubscribeResponse{ - MessageId: 10, - TopicQoS: []byte{ - byte(1), + MessageId: 1, + ReasonCodes: []mqtt.SubscriptionReason{ + mqtt.GrantedQoS1, }, }, }) @@ -83,7 +147,6 @@ func TestSubscribe(t *testing.T) { Type: mqtt.SUBSCRIBE, }, Payload: &mqtt.SubscribeRequest{ - MessageId: 10, Topics: []mqtt.SubscribeTopic{ { Name: "foo", @@ -96,9 +159,9 @@ func TestSubscribe(t *testing.T) { require.NoError(t, err) require.Equal(t, mqtt.SUBACK, res.Header.Type) msg := res.Payload.(*mqtt.SubscribeResponse) - require.Equal(t, int16(10), msg.MessageId) - require.Len(t, msg.TopicQoS, 1) - require.Equal(t, byte(1), msg.TopicQoS[0]) + require.Equal(t, uint16(1), msg.MessageId) + require.Len(t, msg.ReasonCodes, 1) + require.Equal(t, mqtt.GrantedQoS1, msg.ReasonCodes[0]) }, }, } diff --git a/mqtt/unsubscribe.go b/mqtt/unsubscribe.go index 069f694b1..c26b291fd 100644 --- a/mqtt/unsubscribe.go +++ b/mqtt/unsubscribe.go @@ -1,34 +1,70 @@ package mqtt +type UnsubscriptionReason byte + +const ( + UnsubscribeSuccess UnsubscriptionReason = 0 + NoSubscriptionExisted UnsubscriptionReason = 17 +) + type UnsubscribeRequest struct { - MessageId int16 - Topics []string + MessageId uint16 + Topics []string + Properties Properties } -func (r *UnsubscribeRequest) Read(d *Decoder) { - r.MessageId = d.ReadInt16() +func (r *UnsubscribeRequest) Read(d *Decoder, _ *Header) { + r.MessageId = d.ReadUInt16() + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } for d.leftSize > 0 { name := d.ReadString() r.Topics = append(r.Topics, name) } } -func (r *UnsubscribeRequest) Write(e *Encoder) { - e.writeInt16(r.MessageId) +func (r *UnsubscribeRequest) Write(e *Encoder, _ *Header) { + e.writeUInt16(r.MessageId) + + if e.IsV5() { + r.Properties.Write(e) + } for _, name := range r.Topics { e.writeString(name) } } type UnsubscribeResponse struct { - MessageId int16 + MessageId uint16 + ReasonCodes []UnsubscriptionReason + Properties Properties } -func (r *UnsubscribeResponse) Write(e *Encoder) { - e.writeInt16(r.MessageId) +func (r *UnsubscribeResponse) Write(e *Encoder, _ *Header) { + e.writeUInt16(r.MessageId) + + if e.IsV5() { + r.Properties.Write(e) + + for _, reason := range r.ReasonCodes { + e.writeByte(byte(reason)) + } + } } -func (r *UnsubscribeResponse) Read(d *Decoder) { - r.MessageId = d.ReadInt16() +func (r *UnsubscribeResponse) Read(d *Decoder, _ *Header) { + r.MessageId = d.ReadUInt16() + + if d.IsV5() { + r.Properties = Properties{} + r.Properties.Read(d) + } + + for d.leftSize > 0 { + code := d.ReadByte() + r.ReasonCodes = append(r.ReasonCodes, UnsubscriptionReason(code)) + } } diff --git a/mqtt/unsubscribe_test.go b/mqtt/unsubscribe_test.go index 7f63f2ce3..f7ac6c93a 100644 --- a/mqtt/unsubscribe_test.go +++ b/mqtt/unsubscribe_test.go @@ -3,11 +3,12 @@ package mqtt_test import ( "bytes" "fmt" - "github.com/stretchr/testify/require" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/try" "testing" + + "github.com/stretchr/testify/require" ) func TestUnsubscribe_ReadRequest(t *testing.T) { @@ -19,9 +20,9 @@ func TestUnsubscribe_ReadRequest(t *testing.T) { { name: "unsubscribe from foo", in: []byte{ - 0xA0, // flags - 0x07, // length - 0x0, 0xA, // MessageId + 0xA0, // Protocol Type + 0x07, // length + 0x00, 0x03, // message id 0x00, 0x03, // topic length 'f', 'o', 'o', // topic }, @@ -33,6 +34,7 @@ func TestUnsubscribe_ReadRequest(t *testing.T) { require.IsType(t, &mqtt.UnsubscribeRequest{}, r.Payload) msg := r.Payload.(*mqtt.UnsubscribeRequest) + require.Equal(t, uint16(3), msg.MessageId) require.Len(t, msg.Topics, 1) require.Equal(t, "foo", msg.Topics[0]) }, @@ -46,12 +48,74 @@ func TestUnsubscribe_ReadRequest(t *testing.T) { t.Parallel() r := &mqtt.Message{} - err := r.Read(bytes.NewReader(tc.in)) + err := r.Read(bytes.NewReader(tc.in), &mqtt.ClientContext{}) tc.test(t, r, err) }) } } +func TestUnsubscribe_WriteResponse(t *testing.T) { + testcases := []struct { + name string + msg mqtt.Message + ctx *mqtt.ClientContext + out []byte + }{ + { + name: "simple unsubscribe", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.UNSUBACK, + }, + Payload: &mqtt.UnsubscribeResponse{ + MessageId: uint16(12), + ReasonCodes: []mqtt.UnsubscriptionReason{mqtt.UnsubscribeSuccess}, + }, + }, + ctx: &mqtt.ClientContext{}, + out: []byte{ + 0xB0, // Packet type + 0x02, // length + 0x00, 0x0C, // message id + // reason available in v5 + }, + }, + { + name: "subscribe v5", + msg: mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.UNSUBACK, + }, + Payload: &mqtt.UnsubscribeResponse{ + MessageId: uint16(12), + ReasonCodes: []mqtt.UnsubscriptionReason{mqtt.UnsubscribeSuccess}, + }, + }, + ctx: &mqtt.ClientContext{ProtocolVersion: 5}, + out: []byte{ + 0xB0, // Packet type + 0x04, // length + 0x00, 0x0C, // message id + 0x00, // properties + 0x00, // reason available in v5 + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var b bytes.Buffer + err := tc.msg.Write(&b, tc.ctx) + require.NoError(t, err) + require.Equal(t, tc.out, b.Bytes()) + }) + } +} + func TestUnsubscribe(t *testing.T) { testcases := []struct { name string @@ -66,7 +130,10 @@ func TestUnsubscribe(t *testing.T) { Type: mqtt.UNSUBACK, }, Payload: &mqtt.UnsubscribeResponse{ - MessageId: 10, + MessageId: 7, + ReasonCodes: []mqtt.UnsubscriptionReason{ + mqtt.UnsubscribeSuccess, + }, }, }) }), @@ -78,15 +145,16 @@ func TestUnsubscribe(t *testing.T) { Type: mqtt.UNSUBSCRIBE, }, Payload: &mqtt.UnsubscribeRequest{ - MessageId: 10, - Topics: []string{"foo"}, + Topics: []string{"foo"}, }, } res, err := c.Send(r) require.NoError(t, err) require.Equal(t, mqtt.UNSUBACK, res.Header.Type) msg := res.Payload.(*mqtt.UnsubscribeResponse) - require.Equal(t, int16(10), msg.MessageId) + require.Equal(t, uint16(7), msg.MessageId) + // only used from version 5 onwards + require.Len(t, msg.ReasonCodes, 0) }, }, } diff --git a/pkg/cmd/mokapi/flags/mcp.go b/pkg/cmd/mokapi/flags/mcp.go new file mode 100644 index 000000000..0c3ab0dae --- /dev/null +++ b/pkg/cmd/mokapi/flags/mcp.go @@ -0,0 +1,54 @@ +package flags + +import "mokapi/pkg/cli" + +func RegisterMcpFlags(cmd *cli.Command) { + cmd.Flags().Bool("mcp-server-enabled", false, mcpServerEnabled) + cmd.Flags().Int("mcp-server-port", 8080, mcpServerPort) + cmd.Flags().String("mcp-server-path", "/mcp", mcpServerPath) +} + +var mcpServerEnabled = cli.FlagDoc{ + Short: "Enable the MCP server", + Long: `Enables the MCP (Model Context Protocol) server. +When enabled, Mokapi exposes an MCP-compatible endpoint that allows external tools and AI clients to interact with mocked APIs, events, and configuration through a standardized protocol.`, + Examples: []cli.Example{ + { + Codes: []cli.Code{ + {Title: "CLI", Source: "--mcp-server-enabled"}, + {Title: "Env", Source: "MOKAPI_MCP_SERVER=true"}, + {Title: "File", Source: "mcp:\n server:\n enabled: true"}, + }, + }, + }, +} + +var mcpServerPort = cli.FlagDoc{ + Short: "Port for the MCP server", + Long: `Specifies the TCP port on which the MCP server listens. +The MCP server provides access to Mokapi resources via the Model Context Protocol and can be consumed by compatible clients and tools.`, + Examples: []cli.Example{ + { + Codes: []cli.Code{ + {Title: "CLI", Source: "--mcp-server-port 5000"}, + {Title: "Env", Source: "MOKAPI_MCP_PORT=5000"}, + {Title: "File", Source: "mcp:\n server:\n port: 5000"}, + }, + }, + }, +} + +var mcpServerPath = cli.FlagDoc{ + Short: "Path for the MCP server endpoint", + Long: `Defines the HTTP path under which the MCP server is exposed. +This is useful when Mokapi is running behind a reverse proxy or when the MCP endpoint should be served under a specific path.`, + Examples: []cli.Example{ + { + Codes: []cli.Code{ + {Title: "CLI", Source: "--mcp-server-path /foo/mcp"}, + {Title: "Env", Source: "MOKAPI_MCP_PATH=/foo/mcp"}, + {Title: "File", Source: "mcp:\n server:\n path: /foo/mcp"}, + }, + }, + }, +} diff --git a/pkg/cmd/mokapi/mokapi.go b/pkg/cmd/mokapi/mokapi.go index fc392d6af..551d9d12d 100644 --- a/pkg/cmd/mokapi/mokapi.go +++ b/pkg/cmd/mokapi/mokapi.go @@ -3,7 +3,6 @@ package mokapi import ( "context" "fmt" - stdlog "log" "mokapi/api" "mokapi/config/dynamic" "mokapi/config/dynamic/asyncApi" @@ -64,6 +63,7 @@ func NewCmdMokapi() *cli.Command { flags.RegisterNpmProvider(cmd) flags.RegisterApiFlags(cmd) + flags.RegisterMcpFlags(cmd) flags.RegisterHealthFlags(cmd) flags.RegisterTlsFlags(cmd) flags.RegisterEventStoreFlags(cmd) @@ -107,8 +107,6 @@ func runRoot(cmd *cli.Command, cfg *static.Config) error { fmt.Printf(logo, version.BuildVersion, strings.Repeat(" ", 17-len(versionString))) - configureLogging(cfg) - s, err := createServer(cfg) if err != nil { log.WithField("error", err).Error("error creating server") @@ -133,6 +131,7 @@ func createServer(cfg *static.Config) (*server.Server, error) { watcher := server.NewConfigWatcher(cfg) app := runtime.New(cfg, watcher) + app.EnableLogHook() scriptEngine := engine.New(watcher, app, cfg, true) app.Engine = scriptEngine @@ -143,10 +142,12 @@ func createServer(cfg *static.Config) (*server.Server, error) { http := server.NewHttpManager(scriptEngine, certStore, app) apiHandler := api.New(app, cfg.Api) - if u, err := api.BuildUrl(cfg.Api); err == nil { - err = http.AddInternalService("api", u, apiHandler) - if err != nil { - return nil, err + if urls, err := api.BuildUrl(cfg.Api); err == nil { + for _, u := range urls { + err = http.AddInternalService("api", u, apiHandler) + if err != nil { + return nil, err + } } } else { return nil, err @@ -199,23 +200,6 @@ func createServer(cfg *static.Config) (*server.Server, error) { return server.NewServer(pool, app, watcher, kafka, http, mailManager, ldap, scriptEngine), nil } -func configureLogging(cfg *static.Config) { - stdlog.SetFlags(stdlog.Lshortfile | stdlog.LstdFlags) - - level, err := log.ParseLevel(cfg.Log.Level) - if err != nil { - log.WithField("logLevel", cfg.Log.Level).Errorf("error parsing log level: %v", err.Error()) - } - log.SetLevel(level) - - if strings.ToLower(cfg.Log.Format) == "json" { - log.SetFormatter(&log.JSONFormatter{}) - } else { - formatter := &log.TextFormatter{DisableColors: false, FullTimestamp: true, DisableSorting: true} - log.SetFormatter(formatter) - } -} - func init() { registerDynamicTypes() } diff --git a/providers/asyncapi3/asyncapi3test/channel.go b/providers/asyncapi3/asyncapi3test/channel.go index 194f46e3b..a850e9a16 100644 --- a/providers/asyncapi3/asyncapi3test/channel.go +++ b/providers/asyncapi3/asyncapi3test/channel.go @@ -43,6 +43,18 @@ func WithKafkaChannelBinding(bindings asyncapi3.TopicBindings) ChannelOptions { } } +func WithChannelAddress(address string) ChannelOptions { + return func(c *asyncapi3.Channel) { + c.Address = address + } +} + +func WithChannelName(name string) ChannelOptions { + return func(c *asyncapi3.Channel) { + c.Name = name + } +} + func WithChannelTitle(title string) ChannelOptions { return func(c *asyncapi3.Channel) { c.Title = title @@ -77,3 +89,14 @@ func WithChannelTag(name, description string) ChannelOptions { }) } } + +func WithParameter(name string, param *asyncapi3.Parameter) ChannelOptions { + return func(c *asyncapi3.Channel) { + if c.Parameters == nil { + c.Parameters = map[string]*asyncapi3.ParameterRef{} + } + c.Parameters[name] = &asyncapi3.ParameterRef{ + Value: param, + } + } +} diff --git a/providers/asyncapi3/channel.go b/providers/asyncapi3/channel.go index ca0f69f30..bb19f4a29 100644 --- a/providers/asyncapi3/channel.go +++ b/providers/asyncapi3/channel.go @@ -1,7 +1,9 @@ package asyncapi3 import ( + "fmt" "mokapi/config/dynamic" + "regexp" "gopkg.in/yaml.v3" ) @@ -116,3 +118,68 @@ func (c *Channel) GetName() string { } return c.Title } + +func (c *Channel) IsChannelAvailable(protocol string) bool { + if len(c.Servers) == 0 { + return true + } + + for _, v := range c.Servers { + if v.Value == nil { + continue + } + if protocol == v.Value.Protocol { + return true + } + } + return false +} + +func (c *Channel) ResolveAddress() string { + if c.Address != "" { + return c.Address + } + return c.Name +} + +// IsNameValid use if channel contains parameters +func (c *Channel) IsNameValid(topic string) error { + address := c.ResolveAddress() + + // Find all {param} names + re := regexp.MustCompile(`\{([^}]+)\}`) + + // Replace {param} with regex group + pattern := "^" + re.ReplaceAllString(address, `([^/]+)`) + "$" + re = regexp.MustCompile(pattern) + + match := re.FindStringSubmatch(topic) + if match == nil { + return fmt.Errorf("topic name does not match channel address expression") + } + return nil +} + +func (c *Channel) ExtractParams(topicName string) (map[string]string, error) { + // Find all {param} names + re := regexp.MustCompile(`\{([^}]+)\}`) + names := re.FindAllStringSubmatch(c.ResolveAddress(), -1) + + // Replace {param} with regex group + pattern := "^" + re.ReplaceAllString(c.ResolveAddress(), `([^/]+)`) + "$" + re = regexp.MustCompile(pattern) + + match := re.FindStringSubmatch(topicName) + if match == nil { + // path parameters are always required + return nil, fmt.Errorf("topic name does not match channel address expression") + } + + // Build result map + params := map[string]string{} + for i, name := range names { + params[name[1]] = match[i+1] + } + + return params, nil +} diff --git a/providers/asyncapi3/config.go b/providers/asyncapi3/config.go index 575aa9597..efb1dbb2f 100644 --- a/providers/asyncapi3/config.go +++ b/providers/asyncapi3/config.go @@ -116,3 +116,16 @@ func (c *Config) HasKafkaServer() bool { } return false } + +func (c *Config) HasMqttServer() bool { + if c == nil { + return false + } + for it := c.Servers.Iter(); it.Next(); { + server := it.Value() + if server.Value.Protocol == "mqtt" { + return true + } + } + return false +} diff --git a/providers/asyncapi3/kafka/store/client.go b/providers/asyncapi3/kafka/store/client.go index b41131b98..92d10afae 100644 --- a/providers/asyncapi3/kafka/store/client.go +++ b/providers/asyncapi3/kafka/store/client.go @@ -23,6 +23,7 @@ import ( var TopicNotFound = errors.New("topic not found") var PartitionNotFound = errors.New("partition not found") +var OffsetOutOfRange = errors.New("offset out of range") type Record struct { Offset int64 `json:"offset"` @@ -212,6 +213,73 @@ func (c *Client) Read(topic string, partition int, offset int64, ct *media.Conte return records, nil } +func (c *Client) Offset(topic string, partition int, offset int64, ct *media.ContentType) (Record, error) { + t := c.store.Topic(topic) + if t == nil { + return Record{}, TopicNotFound + } + p := t.Partition(partition) + if p == nil { + return Record{}, PartitionNotFound + } + + if offset < 0 { + offset = p.Head + } + + s := p.GetSegment(offset) + if s == nil { + return Record{}, OffsetOutOfRange + } + + r := s.Record(offset) + if r == nil { + return Record{}, OffsetOutOfRange + } + + var getValue func(value []byte) (any, error) + switch { + case ct.Key() == "application/vnd.mokapi.kafka.binary+json": + getValue = func(value []byte) (any, error) { + return base64.StdEncoding.EncodeToString(value), nil + } + case ct.Key() == "application/json": + getValue = func(value []byte) (any, error) { + var val any + err := json.Unmarshal(value, &val) + if err != nil { + return nil, fmt.Errorf("parse record value as JSON failed: %v", err) + } + return val, nil + } + default: + getValue = func(value []byte) (any, error) { + return string(value), nil + } + } + + key := string(kafka.Read(r.Key)) + val, err := getValue(kafka.Read(r.Value)) + if err != nil { + return Record{}, err + } + + rec := Record{ + Offset: r.Offset, + Partition: p.Index, + Key: key, + Value: val, + } + + for _, h := range r.Headers { + rec.Headers = append(rec.Headers, RecordHeader{ + Name: h.Key, + Value: string(h.Value), + }) + } + return rec, nil +} + func (c *Client) getPartition(t *Topic, id int) (*Partition, error) { if id < 0 { r := rand.New(rand.NewSource(time.Now().Unix())) diff --git a/providers/asyncapi3/kafka/store/log.go b/providers/asyncapi3/kafka/store/log.go index cca696fab..35d2a7cfd 100644 --- a/providers/asyncapi3/kafka/store/log.go +++ b/providers/asyncapi3/kafka/store/log.go @@ -39,6 +39,47 @@ func (l *KafkaMessageLog) Title() string { } } +type headerIndex struct { + Key string `json:"key"` + Value string `json:"value"` +} + +func (l *KafkaMessageLog) IndexFields() map[string]any { + m := map[string]any{ + "clientId": l.ClientId, + "offset": l.Offset, + "messageId": l.MessageId, + "partition": l.Partition, + } + + if l.Key.Value != "" { + m["key"] = l.Key.Value + } else { + m["key"] = string(l.Key.Binary) + } + + if l.Message.Value != "" { + m["message"] = l.Message.Value + } else { + m["message"] = string(l.Message.Binary) + } + + var headers []headerIndex + for k, h := range l.Headers { + val := h.Value + if val == "" { + val = string(h.Binary) + } + headers = append(headers, headerIndex{ + Key: k, + Value: val, + }) + } + m["headers"] = headers + + return m +} + func newKafkaLog(record *kafka.Record) *KafkaMessageLog { return &KafkaMessageLog{ Key: LogValue{Binary: kafka.Read(record.Key)}, diff --git a/providers/asyncapi3/kafka/store/metadata_test.go b/providers/asyncapi3/kafka/store/metadata_test.go index d0ba51f87..ad3237056 100644 --- a/providers/asyncapi3/kafka/store/metadata_test.go +++ b/providers/asyncapi3/kafka/store/metadata_test.go @@ -2,6 +2,8 @@ package store_test import ( "fmt" + "mokapi/config/dynamic" + "mokapi/config/dynamic/dynamictest" "mokapi/engine/enginetest" "mokapi/kafka" "mokapi/kafka/kafkatest" @@ -162,11 +164,14 @@ func TestMetadata(t *testing.T) { { "2.2.0 assigning channels to servers", func(t *testing.T, s *store.Store) { - s.Update(asyncapi3test.NewConfig( + cfg := asyncapi3test.NewConfig( asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), asyncapi3test.WithServer("bar", "kafka", "127.0.0.1:9093"), - asyncapi3test.WithChannel("foo", asyncapi3test.AssignToServer("foo")), - )) + asyncapi3test.WithChannel("foo", asyncapi3test.AssignToServer("#/servers/foo")), + ) + err := cfg.Parse(&dynamic.Config{Data: cfg}, &dynamictest.Reader{}) + require.NoError(t, err) + s.Update(cfg) rr := kafkatest.NewRecorder() r := kafkatest.NewRequest("kafkatest", 4, &metaData.Request{}) r.Host = "127.0.0.1:9092" diff --git a/providers/asyncapi3/kafka/store/store.go b/providers/asyncapi3/kafka/store/store.go index 445bd4e66..11aa1475d 100644 --- a/providers/asyncapi3/kafka/store/store.go +++ b/providers/asyncapi3/kafka/store/store.go @@ -172,6 +172,9 @@ func (s *Store) Update(c *asyncapi3.Config) { if ch.Value == nil { continue } + if !ch.Value.IsChannelAvailable("kafka") { + continue + } if ch.Value.Address != "" { n = ch.Value.Address @@ -472,3 +475,11 @@ func (s *Store) Clients() []kafka.ClientContext { } return result } + +func (s *Store) Client(clientId string) (kafka.ClientContext, bool) { + c, ok := s.clients[clientId] + if !ok { + return kafka.ClientContext{}, false + } + return *c, ok +} diff --git a/providers/asyncapi3/kafka/store/validation.go b/providers/asyncapi3/kafka/store/validation.go index 59b67449f..624faf899 100644 --- a/providers/asyncapi3/kafka/store/validation.go +++ b/providers/asyncapi3/kafka/store/validation.go @@ -8,7 +8,6 @@ import ( "mokapi/kafka" "mokapi/media" "mokapi/providers/asyncapi3" - openapi "mokapi/providers/openapi/schema" avro "mokapi/schema/avro/schema" "mokapi/schema/encoding" "mokapi/schema/json/parser" @@ -69,7 +68,7 @@ func newMessageValidator(messageId string, msg *asyncapi3.Message, channel *asyn var msgParser encoding.Parser if msg.Payload != nil && channel.Bindings.Kafka.ValueSchemaValidation { var err error - msgParser, err = getParser(msg.Payload, msg.ContentType) + msgParser, err = msg.Payload.GetParser(msg.ContentType) if err != nil { log.Errorf("unsupported payload type: %T", msg.Payload.Value) } @@ -336,22 +335,3 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str } return result, nil } - -func getParser(ref *asyncapi3.SchemaRef, contentType string) (encoding.Parser, error) { - switch s := ref.Value.(type) { - case *schema.Schema: - return &parser.Parser{Schema: s, ConvertToSortedMap: true}, nil - case *openapi.Schema: - mt := media.ParseContentType(contentType) - if mt.IsXml() { - return openapi.NewXmlParser(s), nil - } - return &parser.Parser{Schema: openapi.ConvertToJsonSchema(s), ConvertToSortedMap: true}, nil - case *asyncapi3.AvroRef: - return &avro.Parser{Schema: s.Schema}, nil - case *asyncapi3.MultiSchemaFormat: - return getParser(s.Schema, contentType) - default: - return nil, fmt.Errorf("unsupported payload type: %T", s) - } -} diff --git a/providers/asyncapi3/mqtt/store/client.go b/providers/asyncapi3/mqtt/store/client.go index 725919df2..6fe288169 100644 --- a/providers/asyncapi3/mqtt/store/client.go +++ b/providers/asyncapi3/mqtt/store/client.go @@ -5,15 +5,29 @@ import ( "mokapi/mqtt" "sync" "time" + + log "github.com/sirupsen/logrus" +) + +type ClientState uint8 + +const ( + ClientConnected ClientState = iota + ClientDisconnected ) type Client struct { - Id string - Clean bool - Subscription map[string]Subscription + Id string + Clean bool + Subscription map[string]Subscription + SessionExpiryInterval int32 + WillMessage *Message + KeepAlive int16 + LastSeen time.Time + State ClientState ctx *mqtt.ClientContext - messageId int16 + messageId uint16 inflight []*InflightMessage m sync.Mutex } @@ -25,7 +39,7 @@ type Subscription struct { } type InflightMessage struct { - MessageId int16 + MessageId uint16 Message *Message QoS byte Retries int @@ -34,15 +48,16 @@ type InflightMessage struct { func (c *Client) publish(msg *Message) { for _, sub := range c.Subscription { - if sub.Name == msg.Topic { + if topicMatches(sub.Name, msg.Topic) { effectiveQoS := min(msg.QoS, sub.QoS) - id := c.nextMessageId() + id := uint16(0) if effectiveQoS > 0 { + id = c.nextMessageId() c.appendInflight(id, msg) } - c.ctx.Send(&mqtt.Message{ + err := c.ctx.Send(&mqtt.Message{ Header: &mqtt.Header{ Type: mqtt.PUBLISH, QoS: effectiveQoS, @@ -54,6 +69,9 @@ func (c *Client) publish(msg *Message) { Data: msg.Data, }, }) + if err != nil { + log.Errorf("mqtt: failed to publish msg %d: %v", id, err) + } } } } @@ -98,21 +116,38 @@ func (c *Client) ResendInflight(duration time.Duration) { } } -func (c *Client) appendInflight(id int16, msg *Message) { +func (c *Client) appendInflight(id uint16, msg *Message) { c.m.Lock() defer c.m.Unlock() c.inflight = append(c.inflight, &InflightMessage{ + QoS: msg.QoS, MessageId: id, Message: msg, SendAt: time.Now(), }) } -func (c *Client) nextMessageId() int16 { +func (c *Client) nextMessageId() uint16 { c.m.Lock() defer c.m.Unlock() c.messageId++ return c.messageId } + +func (c *Client) Addr() string { + return c.ctx.Addr +} + +func (c *Client) ServerAddress() string { + return c.ctx.ServerAddress +} + +func (c *Client) ProtocolVersion() byte { + return c.ctx.ProtocolVersion +} + +func (c *Client) Alive() { + c.LastSeen = time.Now() +} diff --git a/providers/asyncapi3/mqtt/store/connect.go b/providers/asyncapi3/mqtt/store/connect.go index 44da1d2dd..6b4b08670 100644 --- a/providers/asyncapi3/mqtt/store/connect.go +++ b/providers/asyncapi3/mqtt/store/connect.go @@ -1,26 +1,50 @@ package store import ( - log "github.com/sirupsen/logrus" "mokapi/mqtt" + "time" + + log "github.com/sirupsen/logrus" ) func (s *Store) connect(rw mqtt.MessageWriter, connect *mqtt.ConnectRequest, ctx *mqtt.ClientContext) { + reqLog := &ConnectRequest{ + Version: connect.Version, + CleanSession: connect.CleanSession, + KeepAlive: connect.KeepAlive, + Message: nil, + Username: connect.Username, + Password: connect.Password, + } + if connect.Topic != "" { + reqLog.Message = &PublishMessage{ + QoS: connect.WillQoS, + Retain: connect.WillRetain, + Topic: connect.Topic, + Message: string(connect.Message), + } + } + if ctx != nil { ctx.ClientId = connect.ClientId } if len(connect.ClientId) == 0 || len(connect.ClientId) > 23 { - rw.Write(&mqtt.Message{ + err := rw.Write(&mqtt.Message{ Header: &mqtt.Header{ Type: mqtt.CONNACK, }, Payload: &mqtt.ConnectResponse{ - SessionPresent: false, - ReturnCode: mqtt.ErrIdentifierRejected, + ReasonCode: mqtt.ErrIdentifierRejected, }, }) + if err != nil { + log.Errorf("mqtt: failed to write connect response: %v", err) + } + s.logRequest(reqLog, ConnectResponse{ + ReasonCode: mqtt.ErrIdentifierRejected, + }, ctx) return } @@ -28,7 +52,8 @@ func (s *Store) connect(rw mqtt.MessageWriter, connect *mqtt.ConnectRequest, ctx if connect.CleanSession { delete(s.clients, connect.ClientId) } - if c, ok := s.clients[connect.ClientId]; ok { + c, ok := s.clients[connect.ClientId] + if ok { sessionPresent = true c.ctx = ctx go c.ResendInflight(0) @@ -36,49 +61,68 @@ func (s *Store) connect(rw mqtt.MessageWriter, connect *mqtt.ConnectRequest, ctx if s.clients == nil { s.clients = map[string]*Client{} } - s.clients[connect.ClientId] = &Client{Id: connect.ClientId, ctx: ctx} + c = &Client{ + Id: connect.ClientId, + ctx: ctx, + SessionExpiryInterval: connect.Properties.SessionExpiryInterval(), + State: ClientConnected, + } + s.clients[connect.ClientId] = c } + c.KeepAlive = connect.KeepAlive + c.LastSeen = time.Now() if connect.Topic != "" { - s.m.Lock() + if _, ok := s.Topics[connect.Topic]; ok { - if t, ok := s.Topics[connect.Topic]; ok { - m := &Message{ - Data: connect.Message, - QoS: connect.WillQoS, - } - for _, c := range s.clients { - c.publish(m) - } - if connect.WillRetain { - t.Retained = m + if connect.WillFlag { + c.WillMessage = &Message{ + Data: connect.Message, + QoS: connect.WillQoS, + Retain: connect.WillRetain, + } } - s.m.Unlock() } else { log.Infof("mqtt broker: invalid topic %v", connect.Topic) - rw.Write(&mqtt.Message{ + err := rw.Write(&mqtt.Message{ Header: &mqtt.Header{ Type: mqtt.CONNACK, }, Payload: &mqtt.ConnectResponse{ SessionPresent: sessionPresent, - ReturnCode: mqtt.ErrUnspecifiedError, + ReasonCode: mqtt.ErrTopicNameInvalid, }, }) - s.m.Unlock() + if err != nil { + log.Errorf("mqtt: failed to write connect response: %v", err) + } + s.logRequest(reqLog, ConnectResponse{ + ReasonCode: mqtt.ErrTopicNameInvalid, + }, ctx) return } } - rw.Write(&mqtt.Message{ + err := rw.Write(&mqtt.Message{ Header: &mqtt.Header{ Type: mqtt.CONNACK, }, Payload: &mqtt.ConnectResponse{ SessionPresent: sessionPresent, - ReturnCode: mqtt.Accepted, + ReasonCode: mqtt.Success, + Properties: mqtt.Properties{ + mqtt.SessionExpiryInterval: c.SessionExpiryInterval, + }, }, }) + if err != nil { + log.Errorf("mqtt: failed to write connect response: %v", err) + } + + s.logRequest(reqLog, ConnectResponse{ + SessionPresent: sessionPresent, + ReasonCode: mqtt.Success, + }, ctx) s.startQoS() } diff --git a/providers/asyncapi3/mqtt/store/connect_test.go b/providers/asyncapi3/mqtt/store/connect_test.go index 63d4d3563..697ce5a16 100644 --- a/providers/asyncapi3/mqtt/store/connect_test.go +++ b/providers/asyncapi3/mqtt/store/connect_test.go @@ -1,14 +1,17 @@ package store_test import ( - "github.com/stretchr/testify/require" "mokapi/engine/enginetest" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/providers/asyncapi3/asyncapi3test" "mokapi/providers/asyncapi3/mqtt/store" + "mokapi/runtime/events/eventstest" + "mokapi/runtime/monitor" "strings" "testing" + + "github.com/stretchr/testify/require" ) func TestConnect(t *testing.T) { @@ -24,7 +27,7 @@ func TestConnect(t *testing.T) { Payload: &mqtt.ConnectRequest{}, }) res := rr.Message.Payload.(*mqtt.ConnectResponse) - require.Equal(t, mqtt.ErrIdentifierRejected, res.ReturnCode) + require.Equal(t, mqtt.ErrIdentifierRejected, res.ReasonCode) }, }, { @@ -37,7 +40,7 @@ func TestConnect(t *testing.T) { }, }) res := rr.Message.Payload.(*mqtt.ConnectResponse) - require.Equal(t, mqtt.ErrIdentifierRejected, res.ReturnCode) + require.Equal(t, mqtt.ErrIdentifierRejected, res.ReasonCode) }, }, { @@ -109,7 +112,7 @@ func TestConnect(t *testing.T) { }, }) res := rr.Message.Payload.(*mqtt.ConnectResponse) - require.Equal(t, mqtt.ErrUnspecifiedError, res.ReturnCode) + require.Equal(t, mqtt.ErrTopicNameInvalid, res.ReasonCode) }, }, { @@ -127,7 +130,7 @@ func TestConnect(t *testing.T) { }, }) res := rr.Message.Payload.(*mqtt.ConnectResponse) - require.Equal(t, mqtt.Accepted, res.ReturnCode) + require.Equal(t, mqtt.Success, res.ReasonCode) }, }, } @@ -138,7 +141,7 @@ func TestConnect(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine()) + s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewMqtt()) defer s.Close() tc.test(t, s) diff --git a/providers/asyncapi3/mqtt/store/disconnect.go b/providers/asyncapi3/mqtt/store/disconnect.go new file mode 100644 index 000000000..350489df3 --- /dev/null +++ b/providers/asyncapi3/mqtt/store/disconnect.go @@ -0,0 +1,19 @@ +package store + +import "mokapi/mqtt" + +func (s *Store) disconnect(_ mqtt.MessageWriter, disconnect *mqtt.DisconnectRequest, ctx *mqtt.ClientContext) { + client, ok := s.clients[ctx.ClientId] + if !ok { + panic("client not found") + } + + if disconnect.Reason == mqtt.DisconnectWithWillMessage { + t := s.Topics[client.WillMessage.Topic] + t.Retained = client.WillMessage + for _, c := range s.clients { + c.publish(client.WillMessage) + } + } + s.logRequest(&DisconnectRequest{Reason: disconnect.Reason}, nil, ctx) +} diff --git a/providers/asyncapi3/mqtt/store/log.go b/providers/asyncapi3/mqtt/store/log.go new file mode 100644 index 000000000..361fc4eb5 --- /dev/null +++ b/providers/asyncapi3/mqtt/store/log.go @@ -0,0 +1,113 @@ +package store + +import ( + "mokapi/mqtt" + "mokapi/runtime/events" +) + +type LogMessage struct { + Topic string `json:"topic"` + Message LogValue `json:"message"` + MessageId string `json:"messageId"` + Api string `json:"api"` + ClientId string `json:"clientId"` + ScriptFile string `json:"script"` +} + +type LogValue struct { + Value string `json:"value"` + Binary []byte `json:"binary"` +} + +func (l *LogMessage) Title() string { + return l.Topic +} + +type RequestLogEvent struct { + Api string `json:"api"` + Type mqtt.Type `json:"type"` + Request RequestLog `json:"request"` + Response any `json:"response"` +} + +func (r *RequestLogEvent) Title() string { + return r.Request.Title() +} + +type RequestLog interface { + Title() string +} + +type ConnectRequest struct { + Version byte `json:"version"` + CleanSession bool `json:"cleanSession"` + KeepAlive int16 `json:"keepAlive"` + Message *PublishMessage `json:"message,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` +} + +func (r *ConnectRequest) Title() string { + return "Connect" +} + +type PublishMessage struct { + QoS byte `json:"qos"` + Retain bool `json:"retain"` + Topic string `json:"topic"` + Message string `json:"value"` +} + +type ConnectResponse struct { + SessionPresent bool `json:"sessionPresent"` + ReasonCode mqtt.Code `json:"reasonCode"` +} + +type SubscribeRequest struct { + MessageId uint16 `json:"messageId"` + Topics []SubscribeTopic `json:"topics"` +} + +func (r *SubscribeRequest) Title() string { + return "Subscribe" +} + +type SubscribeTopic struct { + Name string `json:"name"` + QoS byte `json:"qos"` +} + +type SubscribeResponse struct { + ReasonCodes []mqtt.SubscriptionReason `json:"reasonCodes"` +} + +type DisconnectRequest struct { + Reason mqtt.DisconnectReason `json:"reason"` +} + +func (r *DisconnectRequest) Title() string { + return "Disconnect" +} + +func (s *Store) logRequest(req RequestLog, res any, ctx *mqtt.ClientContext) { + log := &RequestLogEvent{Api: s.cfg.Info.Name, Request: req} + + switch req.(type) { + case *ConnectRequest: + log.Type = mqtt.CONNECT + case *SubscribeRequest: + log.Type = mqtt.SUBSCRIBE + case *DisconnectRequest: + log.Type = mqtt.DISCONNECT + } + + log.Response = res + t := events.NewTraits(). + WithNamespace("mqtt"). + WithName(s.cfg.Info.Name). + With("type", "request") + if ctx != nil { + t.With("clientId", ctx.ClientId) + } + _ = s.eh.Push(log, t) +} diff --git a/providers/asyncapi3/mqtt/store/ping.go b/providers/asyncapi3/mqtt/store/ping.go new file mode 100644 index 000000000..09ae9bf43 --- /dev/null +++ b/providers/asyncapi3/mqtt/store/ping.go @@ -0,0 +1,20 @@ +package store + +import ( + "mokapi/mqtt" +) + +func (s *Store) ping(rw mqtt.MessageWriter, ping *mqtt.PingRequest, ctx *mqtt.ClientContext) { + client, ok := s.clients[ctx.ClientId] + if !ok { + panic("client not found") + } + client.Alive() + + _ = rw.Write(&mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PINGRESP, + }, + Payload: &mqtt.PingResponse{}, + }) +} diff --git a/providers/asyncapi3/mqtt/store/publish.go b/providers/asyncapi3/mqtt/store/publish.go index cc7b0d4a6..e2c021e9c 100644 --- a/providers/asyncapi3/mqtt/store/publish.go +++ b/providers/asyncapi3/mqtt/store/publish.go @@ -1,8 +1,20 @@ package store -import "mokapi/mqtt" +import ( + "mokapi/mqtt" + "mokapi/runtime/events" + "time" + + log "github.com/sirupsen/logrus" +) + +func (s *Store) publish(rw mqtt.MessageWriter, publish *mqtt.PublishRequest, qos byte, retain bool, ctx *mqtt.ClientContext) { + client, ok := s.clients[ctx.ClientId] + if !ok { + panic("client not found") + } + client.Alive() -func (s *Store) publish(rw mqtt.MessageWriter, publish *mqtt.PublishRequest, qos byte, retain bool) { msg := &Message{ Topic: publish.Topic, Data: publish.Data, @@ -10,24 +22,125 @@ func (s *Store) publish(rw mqtt.MessageWriter, publish *mqtt.PublishRequest, qos Retain: retain, } - if topic, ok := s.Topics[msg.Topic]; ok { - if retain { - topic.Retained = msg + topic, ok := s.getTopic(msg.Topic) + if !ok { + log.Infof("mqtt: topic not specified %s", msg.Topic) + if qos != 0 { + puback(rw, &mqtt.PublishResponse{ + MessageId: publish.MessageId, + ReasonCode: mqtt.TopicNameInvalid, + }) } + return } - rw.Write(&mqtt.Message{ - Header: &mqtt.Header{ - Type: mqtt.PUBACK, - }, - Payload: &mqtt.PublishResponse{ + messageId, err := topic.validate(msg.Data) + if err != nil { + log.Errorf("mqtt: topic validation error '%s': %s", msg.Topic, err) + puback(rw, &mqtt.PublishResponse{ + MessageId: publish.MessageId, + ReasonCode: mqtt.PayloadFormatInvalid, + }) + return + } + + if retain { + topic.Retained = msg + } + + if qos == 1 { + puback(rw, &mqtt.PublishResponse{ MessageId: publish.MessageId, - }, - }) + }) + } go func() { for _, client := range s.clients { client.publish(msg) } }() + + s.logMessage(messageId, topic, publish, ctx) +} + +func puback(rw mqtt.MessageWriter, payload *mqtt.PublishResponse) error { + return rw.Write(&mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.PUBACK, + }, + Payload: payload, + }) +} + +func (s *Store) getTopic(topic string) (*Topic, bool) { + if t, ok := s.Topics[topic]; ok { + return t, ok + } + + for _, ch := range s.cfg.Channels { + if ch == nil || ch.Value == nil { + continue + } + if !ch.Value.IsChannelAvailable("mqtt") { + continue + } + if len(ch.Value.Parameters) == 0 { + continue + } + + err := ch.Value.IsNameValid(topic) + if err != nil { + continue + } + + if s.Topics == nil { + s.Topics = make(map[string]*Topic) + } + + t := &Topic{Name: topic, cfg: ch.Value} + s.Topics[topic] = t + return t, true + } + + return nil, false +} + +func (s *Store) logMessage(messageId string, topic *Topic, publish *mqtt.PublishRequest, ctx *mqtt.ClientContext) { + topicName := topic.cfg.ResolveAddress() + labels := []string{s.cfg.Info.Name, topicName} + + s.monitor.Messages.WithLabel(labels...).Add(1) + s.monitor.LastMessage.WithLabel(labels...).Set(float64(time.Now().Unix())) + + traits := events. + NewTraits(). + WithNamespace("mqtt"). + WithName(s.cfg.Info.Name). + With("topic", topicName). + With("type", "message"). + With("clientId", ctx.ClientId) + if len(topic.cfg.Parameters) > 0 { + params, err := topic.cfg.ExtractParams(topic.Name) + if err != nil { + log.Errorf("mqtt: failed to log message: %s", err) + } + for k, v := range params { + traits.With(k, v) + } + } + + client, _ := s.clients[ctx.ClientId] + err := s.eh.Push(&LogMessage{ + Topic: topic.Name, + MessageId: messageId, + Message: LogValue{ + Value: string(publish.Data), + Binary: publish.Data, + }, + Api: s.cfg.Info.Name, + ClientId: client.Id, + }, traits) + if err != nil { + log.Errorf("mqtt: failed to log message: %s", err) + } } diff --git a/providers/asyncapi3/mqtt/store/publish_test.go b/providers/asyncapi3/mqtt/store/publish_test.go index 6eb232a72..3bb3d57a5 100644 --- a/providers/asyncapi3/mqtt/store/publish_test.go +++ b/providers/asyncapi3/mqtt/store/publish_test.go @@ -2,25 +2,32 @@ package store_test import ( "context" - "github.com/stretchr/testify/require" "mokapi/engine/enginetest" "mokapi/mqtt" "mokapi/mqtt/mqtttest" + "mokapi/providers/asyncapi3" "mokapi/providers/asyncapi3/asyncapi3test" "mokapi/providers/asyncapi3/mqtt/store" + "mokapi/runtime/events" + "mokapi/runtime/events/eventstest" + "mokapi/runtime/metrics" + "mokapi/runtime/monitor" + "mokapi/schema/json/schema/schematest" "net" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestPublish(t *testing.T) { testcases := []struct { name string - test func(t *testing.T, s *store.Store) + test func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) }{ { - name: "publish no consumers", - test: func(t *testing.T, s *store.Store) { + name: "publish QoS=0", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { publisher := newClient("publisher", s) defer publisher.close() @@ -32,19 +39,129 @@ func TestPublish(t *testing.T) { Retain: false, }, Payload: &mqtt.PublishRequest{ - MessageId: 11, + Topic: "/foo/bar", + Data: []byte("hello world"), + }, + Context: publisher.ctx, + }) + require.Nil(t, rr.Message) + }, + }, + { + name: "publish QoS=1 topic not specified", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { + publisher := newClient("publisher", s) + defer publisher.close() + + publisher.connect() + + rr := publisher.send(&mqtt.Message{ + Header: &mqtt.Header{ + QoS: 1, + Retain: false, + }, + Payload: &mqtt.PublishRequest{ + MessageId: uint16(123), + Topic: "/foo/bar", + Data: []byte("hello world"), + }, + Context: publisher.ctx, + }) + require.NotNil(t, rr.Message) + res := rr.Message.Payload.(*mqtt.PublishResponse) + require.Equal(t, uint16(123), res.MessageId) + require.Equal(t, mqtt.TopicNameInvalid, res.ReasonCode) + }, + }, + { + name: "publish QoS=1 topic specified", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("test-server", "", ""), + asyncapi3test.WithChannel("/foo/bar", + asyncapi3test.WithMessage("msg-name"), + ), + )) + + publisher := newClient("publisher", s) + defer publisher.close() + + publisher.connect() + + rr := publisher.send(&mqtt.Message{ + Header: &mqtt.Header{ + QoS: 1, + Retain: false, + }, + Payload: &mqtt.PublishRequest{ + MessageId: uint16(123), + Topic: "/foo/bar", + Data: []byte("hello world"), + }, + Context: publisher.ctx, + }) + require.NotNil(t, rr.Message) + res := rr.Message.Payload.(*mqtt.PublishResponse) + require.Equal(t, uint16(123), res.MessageId) + require.Equal(t, mqtt.PublishSuccess, res.ReasonCode) + + evts := eh.GetEvents(events.NewTraits().WithNamespace("mqtt").With("type", "message")) + require.Len(t, evts, 1) + d := evts[0].Data.(*store.LogMessage) + require.Equal(t, "/foo/bar", d.Topic) + require.Equal(t, "publisher", d.ClientId) + require.Equal(t, "msg-name", d.MessageId) + require.Equal(t, "hello world", d.Message.Value) + require.Equal(t, "publisher", d.ClientId) + + require.Equal(t, float64(1), m.Messages.Sum(metrics.NewQuery())) + require.Equal(t, float64(1), m.Messages.WithLabel("test-server", "/foo/bar").Value()) + require.Greater(t, m.LastMessage.WithLabel("test-server", "/foo/bar").Value(), float64(1)) + }, + }, + { + name: "publish QoS=1 topic specified message not valid", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("test-server", "", ""), + asyncapi3test.WithChannel("/foo/bar", + asyncapi3test.WithMessage("bar", asyncapi3test.WithPayload(schematest.New("integer"))), + ), + )) + + publisher := newClient("publisher", s) + defer publisher.close() + + publisher.connect() + + rr := publisher.send(&mqtt.Message{ + Header: &mqtt.Header{ + QoS: 1, + Retain: false, + }, + Payload: &mqtt.PublishRequest{ + MessageId: uint16(123), Topic: "/foo/bar", Data: []byte("hello world"), }, Context: publisher.ctx, }) + require.NotNil(t, rr.Message) res := rr.Message.Payload.(*mqtt.PublishResponse) - require.Equal(t, int16(11), res.MessageId) + require.Equal(t, uint16(123), res.MessageId) + require.Equal(t, mqtt.PayloadFormatInvalid, res.ReasonCode) }, }, { - name: "publish with one consumer QoS=0", - test: func(t *testing.T, s *store.Store) { + name: "publish with one consumer QoS=1", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("test-server", "", ""), + asyncapi3test.WithChannel("/foo/bar", + asyncapi3test.WithMessage("bar", asyncapi3test.WithPayload(schematest.New("string"))), + ), + )) + publisher := newClient("publisher", s) defer publisher.close() consumer := newClient("consumer", s) @@ -60,6 +177,7 @@ func TestPublish(t *testing.T) { Topics: []mqtt.SubscribeTopic{ { Name: "/foo/bar", + QoS: 1, }, }, }, @@ -69,7 +187,7 @@ func TestPublish(t *testing.T) { // publish publisher.send(&mqtt.Message{ Header: &mqtt.Header{ - QoS: 0, + QoS: 1, Retain: false, }, Payload: &mqtt.PublishRequest{ @@ -80,20 +198,20 @@ func TestPublish(t *testing.T) { Context: publisher.ctx, }) - consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) + _ = consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) res := &mqtt.Message{} - err := res.Read(consumer.conn) + err := res.Read(consumer.conn, consumer.clientCtx) require.NoError(t, err) require.NotNil(t, res) pub := res.Payload.(*mqtt.PublishRequest) - require.Equal(t, int16(1), pub.MessageId) + require.Equal(t, uint16(1), pub.MessageId) require.Equal(t, "/foo/bar", pub.Topic) require.Equal(t, []byte("hello world"), pub.Data) }, }, { - name: "consumer subscribes after published retain message QoS=0", - test: func(t *testing.T, s *store.Store) { + name: "consumer subscribes after published retain message QoS=1", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { s.Update(asyncapi3test.NewConfig(asyncapi3test.WithChannel("/foo/bar"))) publisher := newClient("publisher", s) @@ -107,7 +225,7 @@ func TestPublish(t *testing.T) { // publish publisher.send(&mqtt.Message{ Header: &mqtt.Header{ - QoS: 0, + QoS: 1, Retain: true, }, Payload: &mqtt.PublishRequest{ @@ -126,6 +244,7 @@ func TestPublish(t *testing.T) { Topics: []mqtt.SubscribeTopic{ { Name: "/foo/bar", + QoS: 1, }, }, }, @@ -134,70 +253,18 @@ func TestPublish(t *testing.T) { consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) res := &mqtt.Message{} - err := res.Read(consumer.conn) + err := res.Read(consumer.conn, consumer.clientCtx) require.NoError(t, err) require.NotNil(t, res) pub := res.Payload.(*mqtt.PublishRequest) - require.Equal(t, int16(1), pub.MessageId) + require.Equal(t, uint16(1), pub.MessageId) require.Equal(t, "/foo/bar", pub.Topic) require.Equal(t, []byte("hello world"), pub.Data) }, }, - { - name: "consumer subscribes but is offline when publishing QoS=0", - test: func(t *testing.T, s *store.Store) { - s.Update(asyncapi3test.NewConfig(asyncapi3test.WithChannel("/foo/bar"))) - - publisher := newClient("publisher", s) - defer publisher.close() - consumer := newClient("consumer", s) - defer consumer.close() - - publisher.connect() - consumer.connect() - - // subscribe - consumer.send(&mqtt.Message{ - Payload: &mqtt.SubscribeRequest{ - MessageId: 1, - Topics: []mqtt.SubscribeTopic{ - { - Name: "/foo/bar", - }, - }, - }, - Context: consumer.ctx, - }) - - consumer.close() - - // publish - publisher.send(&mqtt.Message{ - Header: &mqtt.Header{ - QoS: 0, - }, - Payload: &mqtt.PublishRequest{ - MessageId: 11, - Topic: "/foo/bar", - Data: []byte("hello world"), - }, - Context: publisher.ctx, - }) - - time.Sleep(500 * time.Millisecond) - - consumer = newClient("consumer", s) - consumer.connect() - - consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) - res := &mqtt.Message{} - err := res.Read(consumer.conn) - require.Error(t, err) - }, - }, { name: "consumer subscribes but is offline when publishing QoS=1", - test: func(t *testing.T, s *store.Store) { + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { s.RetryInterval = 500 * time.Millisecond s.Update(asyncapi3test.NewConfig(asyncapi3test.WithChannel("/foo/bar"))) @@ -216,7 +283,7 @@ func TestPublish(t *testing.T) { Topics: []mqtt.SubscribeTopic{ { Name: "/foo/bar", - QoS: byte(1), + QoS: 1, }, }, }, @@ -248,22 +315,22 @@ func TestPublish(t *testing.T) { consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) res := &mqtt.Message{} - err := res.Read(consumer.conn) + err := res.Read(consumer.conn, consumer.clientCtx) require.NoError(t, err) require.NotNil(t, res) pub := res.Payload.(*mqtt.PublishRequest) - require.Equal(t, int16(1), pub.MessageId) + require.Equal(t, uint16(1), pub.MessageId) require.Equal(t, "/foo/bar", pub.Topic) require.Equal(t, []byte("hello world"), pub.Data) // broker should send the message again because no ACK was sent consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) res = &mqtt.Message{} - err = res.Read(consumer.conn) + err = res.Read(consumer.conn, consumer.clientCtx) require.NoError(t, err) require.NotNil(t, res) pub = res.Payload.(*mqtt.PublishRequest) - require.Equal(t, int16(1), pub.MessageId) + require.Equal(t, uint16(1), pub.MessageId) require.Equal(t, "/foo/bar", pub.Topic) require.Equal(t, []byte("hello world"), pub.Data) @@ -277,7 +344,60 @@ func TestPublish(t *testing.T) { // no further message should be received consumer.conn.SetReadDeadline(time.Now().Add(1 * time.Second)) res = &mqtt.Message{} - err = res.Read(consumer.conn) + err = res.Read(consumer.conn, consumer.clientCtx) + }, + }, + { + name: "publish topic with parameter", + test: func(t *testing.T, s *store.Store, eh events.Handler, m *monitor.Mqtt) { + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithInfo("test-server", "", ""), + asyncapi3test.WithChannel("sensors/{sensorId}/data", + asyncapi3test.WithParameter( + "sensorId", &asyncapi3.Parameter{}, + ), + )), + ) + + publisher := newClient("publisher", s) + defer publisher.close() + + publisher.connect() + + rr := publisher.send(&mqtt.Message{ + Header: &mqtt.Header{ + QoS: 1, + Retain: false, + }, + Payload: &mqtt.PublishRequest{ + MessageId: uint16(123), + Topic: "sensors/1234z/data", + Data: []byte("hello world"), + }, + Context: publisher.ctx, + }) + require.NotNil(t, rr.Message) + res := rr.Message.Payload.(*mqtt.PublishResponse) + require.Equal(t, uint16(123), res.MessageId) + require.Equal(t, mqtt.PublishSuccess, res.ReasonCode) + + evts := eh.GetEvents(events. + NewTraits(). + WithNamespace("mqtt"). + WithName("test-server"). + With("topic", "sensors/{sensorId}/data"). + With("type", "message"), + ) + require.Len(t, evts, 1) + d := evts[0].Data.(*store.LogMessage) + require.Equal(t, "sensors/1234z/data", d.Topic) + require.Equal(t, "publisher", d.ClientId) + require.Equal(t, "hello world", d.Message.Value) + require.Equal(t, "namespace=mqtt, name=test-server, clientId=publisher, sensorId=1234z, topic=sensors/{sensorId}/data, type=message", evts[0].Traits.String()) + + require.Equal(t, float64(1), m.Messages.Sum(metrics.NewQuery())) + require.Equal(t, float64(1), m.Messages.WithLabel("test-server", "sensors/{sensorId}/data").Value()) + require.Greater(t, m.LastMessage.WithLabel("test-server", "sensors/{sensorId}/data").Value(), float64(1)) }, }, } @@ -288,10 +408,17 @@ func TestPublish(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine()) + eh := &eventstest.Handler{} + m := monitor.NewMqtt() + s := store.New( + asyncapi3test.NewConfig(asyncapi3test.WithInfo("test-server", "", "")), + enginetest.NewEngine(), + eh, + m, + ) defer s.Close() - tc.test(t, s) + tc.test(t, s, eh, m) }) } } diff --git a/providers/asyncapi3/mqtt/store/store.go b/providers/asyncapi3/mqtt/store/store.go index 730517d2d..343c40d08 100644 --- a/providers/asyncapi3/mqtt/store/store.go +++ b/providers/asyncapi3/mqtt/store/store.go @@ -4,6 +4,9 @@ import ( engine "mokapi/engine/common" "mokapi/mqtt" "mokapi/providers/asyncapi3" + "mokapi/runtime/events" + "mokapi/runtime/monitor" + "mokapi/version" "sync" "time" ) @@ -16,53 +19,82 @@ type Store struct { startedQoS bool m sync.RWMutex close chan bool + eh events.Handler + cfg *asyncapi3.Config + monitor *monitor.Mqtt + + stopClientCleaner chan bool } -func New(cfg *asyncapi3.Config, emitter engine.EventEmitter) *Store { +func New(cfg *asyncapi3.Config, emitter engine.EventEmitter, eh events.Handler, m *monitor.Mqtt) *Store { s := &Store{ RetryInterval: 10 * time.Second, + Topics: make(map[string]*Topic), close: make(chan bool, 1), + cfg: cfg, + eh: eh, + monitor: m, } - for _, ch := range cfg.Channels { - if s.Topics == nil { - s.Topics = make(map[string]*Topic) - } + s.Update(cfg) - if ch != nil && ch.Value != nil { - s.Topics[ch.Value.Name] = &Topic{ - Name: ch.Value.Name, - } - } - } + s.addSysTopic("$SYS/broker/version", "Mokapi "+version.BuildVersion) + s.addSysTopic("$SYS/broker/uptime", time.Now().Format(time.RFC3339)) + + s.startClientSessionCleaner() return s } func (s *Store) Update(cfg *asyncapi3.Config) { - for _, ch := range cfg.Channels { + s.cfg = cfg + + for address, ch := range cfg.Channels { + if ch == nil || ch.Value == nil { + continue + } + if !ch.Value.IsChannelAvailable("mqtt") { + continue + } if s.Topics == nil { s.Topics = make(map[string]*Topic) } - s.Topics[ch.Value.Name] = &Topic{Name: ch.Value.Name} + + if ch.Value.Address != "" { + address = ch.Value.Address + } + + if len(ch.Value.Parameters) == 0 { + s.Topics[address] = &Topic{Name: address, cfg: ch.Value} + } } } func (s *Store) ServeMessage(rw mqtt.MessageWriter, req *mqtt.Message) { + s.m.Lock() + defer s.m.Unlock() + ctx := mqtt.ClientFromContext(req.Context) switch msg := req.Payload.(type) { case *mqtt.ConnectRequest: s.connect(rw, msg, ctx) + case *mqtt.DisconnectRequest: + s.disconnect(rw, msg, ctx) case *mqtt.SubscribeRequest: s.subscribe(rw, msg, ctx) case *mqtt.PublishRequest: - s.publish(rw, msg, req.Header.QoS, req.Header.Retain) + s.publish(rw, msg, req.Header.QoS, req.Header.Retain, ctx) + case *mqtt.UnsubscribeRequest: + s.unsubscribe(rw, msg, ctx) + case *mqtt.PingRequest: + s.ping(rw, msg, ctx) } } func (s *Store) Close() { s.close <- true + close(s.stopClientCleaner) } func (s *Store) startQoS() { @@ -70,13 +102,6 @@ func (s *Store) startQoS() { return } - s.m.Lock() - defer s.m.Unlock() - - if s.startedQoS { - return - } - ticker := time.NewTicker(s.RetryInterval) go func() { @@ -95,3 +120,40 @@ func (s *Store) startQoS() { s.startedQoS = true } + +func (s *Store) Clients() []*Client { + var result []*Client + for _, c := range s.clients { + result = append(result, c) + } + return result +} + +func (s *Store) startClientSessionCleaner() { + ticker := time.NewTicker(30 * time.Second) + s.stopClientCleaner = make(chan bool) + + go func() { + for { + select { + case <-s.stopClientCleaner: + ticker.Stop() + return + case <-ticker.C: + s.checkClientSession() + } + } + }() +} + +func (s *Store) checkClientSession() { + s.m.Lock() + defer s.m.Unlock() + + now := time.Now() + for _, client := range s.clients { + if now.After(client.LastSeen.Add(time.Duration(client.KeepAlive) * time.Second)) { + client.State = ClientConnected + } + } +} diff --git a/providers/asyncapi3/mqtt/store/subscribe.go b/providers/asyncapi3/mqtt/store/subscribe.go index 0c2e21362..9c9983ad8 100644 --- a/providers/asyncapi3/mqtt/store/subscribe.go +++ b/providers/asyncapi3/mqtt/store/subscribe.go @@ -1,6 +1,9 @@ package store -import "mokapi/mqtt" +import ( + "mokapi/mqtt" + "strings" +) func (s *Store) subscribe(rw mqtt.MessageWriter, subscribe *mqtt.SubscribeRequest, ctx *mqtt.ClientContext) { client, ok := s.clients[ctx.ClientId] @@ -14,7 +17,7 @@ func (s *Store) subscribe(rw mqtt.MessageWriter, subscribe *mqtt.SubscribeReques for _, topic := range subscribe.Topics { client.Subscribe(topic.Name, topic.QoS) - res.TopicQoS = append(res.TopicQoS, topic.QoS) + res.ReasonCodes = append(res.ReasonCodes, mqtt.SubscriptionReason(topic.QoS)) go func() { for _, msg := range s.getRetainedMessages(topic.Name) { @@ -29,18 +32,95 @@ func (s *Store) subscribe(rw mqtt.MessageWriter, subscribe *mqtt.SubscribeReques }, Payload: res, }) + + reqLog := &SubscribeRequest{ + MessageId: subscribe.MessageId, + } + for _, topic := range subscribe.Topics { + reqLog.Topics = append(reqLog.Topics, SubscribeTopic{ + Name: topic.Name, + QoS: topic.QoS, + }) + } + resLog := &SubscribeResponse{ReasonCodes: res.ReasonCodes} + + s.logRequest(reqLog, resLog, ctx) +} + +func (s *Store) unsubscribe(rw mqtt.MessageWriter, req *mqtt.UnsubscribeRequest, ctx *mqtt.ClientContext) { + client, ok := s.clients[ctx.ClientId] + if !ok { + panic("client not found") + } + + res := &mqtt.UnsubscribeResponse{ + MessageId: req.MessageId, + } + + for _, topic := range req.Topics { + if client.Subscription == nil { + res.ReasonCodes = append(res.ReasonCodes, mqtt.NoSubscriptionExisted) + } else { + _, ok = client.Subscription[topic] + if !ok { + res.ReasonCodes = append(res.ReasonCodes, mqtt.NoSubscriptionExisted) + } else { + delete(client.Subscription, topic) + res.ReasonCodes = append(res.ReasonCodes, mqtt.UnsubscribeSuccess) + } + } + } + + rw.Write(&mqtt.Message{ + Header: &mqtt.Header{ + Type: mqtt.UNSUBACK, + }, + Payload: res, + }) } func (s *Store) getRetainedMessages(name string) []*Message { var retained []*Message for _, topic := range s.Topics { - if topic.Name == name { - if topic.Retained != nil { - { - retained = append(retained, topic.Retained) - } - } + if topic.Retained == nil || !topicMatches(name, topic.Name) { + continue } + + retained = append(retained, topic.Retained) } return retained } + +func topicMatches(filter string, topic string) bool { + // special case for system topics + if len(topic) > 0 && topic[0] == '$' { + if len(filter) > 0 && filter[0] != '$' { + return false + } + } + + fLevels := strings.Split(filter, "/") + tLevels := strings.Split(topic, "/") + + for i := 0; i < len(fLevels); i++ { + f := fLevels[i] + + // multi level matches all + if f == "#" { + return true + } + + if i >= len(tLevels) { + return false + } + + // single level wildcard or exact match + if f == "+" || f == tLevels[i] { + continue + } + + return false + } + + return len(fLevels) == len(tLevels) +} diff --git a/providers/asyncapi3/mqtt/store/subscribe_test.go b/providers/asyncapi3/mqtt/store/subscribe_test.go index 108920d51..a41907c0d 100644 --- a/providers/asyncapi3/mqtt/store/subscribe_test.go +++ b/providers/asyncapi3/mqtt/store/subscribe_test.go @@ -1,13 +1,16 @@ package store_test import ( - "github.com/stretchr/testify/require" "mokapi/engine/enginetest" "mokapi/mqtt" "mokapi/mqtt/mqtttest" "mokapi/providers/asyncapi3/asyncapi3test" "mokapi/providers/asyncapi3/mqtt/store" + "mokapi/runtime/events/eventstest" + "mokapi/runtime/monitor" "testing" + + "github.com/stretchr/testify/require" ) func TestSubscribe(t *testing.T) { @@ -40,9 +43,9 @@ func TestSubscribe(t *testing.T) { Context: ctx, }) res := rr.Message.Payload.(*mqtt.SubscribeResponse) - require.Equal(t, int16(11), res.MessageId) - require.Len(t, res.TopicQoS, 1) - require.Equal(t, byte(0), res.TopicQoS[0]) + require.Equal(t, uint16(11), res.MessageId) + require.Len(t, res.ReasonCodes, 1) + require.Equal(t, mqtt.GrantedQoS0, res.ReasonCodes[0]) }, }, { @@ -71,6 +74,73 @@ func TestSubscribe(t *testing.T) { t.Error("Test failed, panic was expected") }, }, + { + name: "unsubscribe", + test: func(t *testing.T, s *store.Store) { + rr := mqtttest.NewRecorder() + ctx, conn := mqtttest.NewTestClientContext() + defer conn.Close() + client := mqtt.ClientFromContext(ctx) + client.ClientId = "foo" + + s.ServeMessage(rr, &mqtt.Message{ + Payload: &mqtt.ConnectRequest{ + ClientId: client.ClientId, + }, + }) + + s.ServeMessage(rr, &mqtt.Message{ + Payload: &mqtt.SubscribeRequest{ + MessageId: 11, + Topics: []mqtt.SubscribeTopic{ + {Name: "foo"}, + }, + }, + Context: ctx, + }) + + s.ServeMessage(rr, &mqtt.Message{ + Payload: &mqtt.UnsubscribeRequest{ + MessageId: 11, + Topics: []string{"foo"}, + }, + Context: ctx, + }) + require.Equal(t, mqtt.UNSUBACK, rr.Message.Header.Type) + res := rr.Message.Payload.(*mqtt.UnsubscribeResponse) + require.Equal(t, uint16(11), res.MessageId) + require.Len(t, res.ReasonCodes, 1) + require.Equal(t, mqtt.UnsubscribeSuccess, res.ReasonCodes[0]) + }, + }, + { + name: "unsubscribe but not existing", + test: func(t *testing.T, s *store.Store) { + rr := mqtttest.NewRecorder() + ctx, conn := mqtttest.NewTestClientContext() + defer conn.Close() + client := mqtt.ClientFromContext(ctx) + client.ClientId = "foo" + + s.ServeMessage(rr, &mqtt.Message{ + Payload: &mqtt.ConnectRequest{ + ClientId: client.ClientId, + }, + }) + + s.ServeMessage(rr, &mqtt.Message{ + Payload: &mqtt.UnsubscribeRequest{ + MessageId: 11, + Topics: []string{"foo"}, + }, + Context: ctx, + }) + res := rr.Message.Payload.(*mqtt.UnsubscribeResponse) + require.Equal(t, uint16(11), res.MessageId) + require.Len(t, res.ReasonCodes, 1) + require.Equal(t, mqtt.NoSubscriptionExisted, res.ReasonCodes[0]) + }, + }, } t.Parallel() @@ -79,7 +149,7 @@ func TestSubscribe(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine()) + s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewMqtt()) defer s.Close() tc.test(t, s) diff --git a/providers/asyncapi3/mqtt/store/topic.go b/providers/asyncapi3/mqtt/store/topic.go index 7a70b0079..7c30dce45 100644 --- a/providers/asyncapi3/mqtt/store/topic.go +++ b/providers/asyncapi3/mqtt/store/topic.go @@ -1,7 +1,10 @@ package store import ( + "fmt" + "mokapi/media" "mokapi/providers/asyncapi3" + "mokapi/schema/encoding" ) type Message struct { @@ -17,3 +20,55 @@ type Topic struct { cfg *asyncapi3.Channel } + +func (t *Topic) validate(value []byte) (messageId string, err error) { + if t.cfg == nil { + return + } + + for id, msg := range t.cfg.Messages { + if msg.Value == nil { + continue + } + messageId = id + payload := msg.Value.Payload + if payload == nil || payload.Value == nil { + continue + } + + var p encoding.Parser + p, err = payload.GetParser(msg.Value.ContentType) + if err != nil { + return + } + + _, err = encoding.Decode(value, encoding.WithContentType(media.ParseContentType(msg.Value.ContentType)), encoding.WithParser(p)) + if err == nil { + return + } + } + return +} + +func (s *Store) addSysTopic(name string, val string) { + t := &Topic{ + Name: name, + Retained: &Message{ + Topic: name, + Data: []byte(val), + Retain: true, + }, + } + if s.Topics == nil { + s.Topics = map[string]*Topic{} + } + s.Topics[name] = t +} + +func (s *Store) updateSysTopic(name string, val string) { + t, ok := s.Topics[name] + if !ok { + panic(fmt.Sprintf("mqtt: sys topic '%s' not found", name)) + } + t.Retained.Data = []byte(val) +} diff --git a/providers/asyncapi3/schema.go b/providers/asyncapi3/schema.go index f66c3e568..58c08db22 100644 --- a/providers/asyncapi3/schema.go +++ b/providers/asyncapi3/schema.go @@ -9,6 +9,7 @@ import ( openapi "mokapi/providers/openapi/schema" avro "mokapi/schema/avro/schema" "mokapi/schema/encoding" + "mokapi/schema/json/parser" jsonSchema "mokapi/schema/json/schema" "reflect" @@ -190,8 +191,12 @@ func (m *MultiSchemaFormat) UnmarshalJSON(b []byte) error { if delim, ok := token.(json.Delim); ok && delim == '}' { break } + s, ok := token.(string) + if !ok { + return fmt.Errorf("unexpected token %s; expected string", token) + } - switch token.(string) { + switch s { case "schemaFormat": token, err = d.Token() if err != nil { @@ -399,3 +404,26 @@ func (r *AvroRef) UnmarshalYAML(node *yaml.Node) error { func (r *AvroRef) UnmarshalJSON(b []byte) error { return r.Reference.UnmarshalJson(b, &r.Schema) } + +func (r *SchemaRef) GetParser(contentType string) (encoding.Parser, error) { + if r == nil || r.Value == nil { + return &encoding.NullParser{}, nil + } + + switch s := r.Value.(type) { + case *jsonSchema.Schema: + return &parser.Parser{Schema: s, ConvertToSortedMap: true}, nil + case *openapi.Schema: + mt := media.ParseContentType(contentType) + if mt.IsXml() { + return openapi.NewXmlParser(s), nil + } + return &parser.Parser{Schema: openapi.ConvertToJsonSchema(s), ConvertToSortedMap: true}, nil + case *AvroRef: + return &avro.Parser{Schema: s.Schema}, nil + case *MultiSchemaFormat: + return s.Schema.GetParser(contentType) + default: + return nil, fmt.Errorf("unsupported payload type: %T", s) + } +} diff --git a/providers/directory/log.go b/providers/directory/log.go index fb8476699..3e1ad1293 100644 --- a/providers/directory/log.go +++ b/providers/directory/log.go @@ -44,7 +44,7 @@ func NewSearchLogEvent(r *ldap.SearchRequest, eh events.Handler, traits events.T Duration: 0, Actions: nil, } - _ = eh.Push(event, traits.WithNamespace("ldap")) + _ = eh.Push(event, traits.WithNamespace("ldap").With("operation", "search")) return event } @@ -127,13 +127,13 @@ func NewBindLogEvent(req *ldap.BindRequest, res *ldap.BindResponse, eh events.Ha Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "bind")) return l } func NewUnbindEvent(eh events.Handler, traits events.Traits) *UnbindLog { l := &UnbindLog{Request: &UnbindRequest{Operation: "Unbind"}} - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "unbind")) return l } @@ -160,7 +160,7 @@ func NewAddLogEvent(req *ldap.AddRequest, res *ldap.AddResponse, eh events.Handl Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "add")) return l } @@ -209,7 +209,7 @@ func NewModifyLogEvent(req *ldap.ModifyRequest, res *ldap.ModifyResponse, eh eve Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "modify")) return l } @@ -240,7 +240,7 @@ func NewDeleteLogEvent(req *ldap.DeleteRequest, res *ldap.DeleteResponse, eh eve Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "delete")) return l } @@ -277,7 +277,7 @@ func NewModifyDNLogEvent(req *ldap.ModifyDNRequest, res *ldap.ModifyDNResponse, Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "modifydn")) return l } @@ -311,9 +311,10 @@ func NewCompareLogEvent(req *ldap.CompareRequest, res *ldap.CompareResponse, eh Duration: 0, Actions: nil, } - _ = eh.Push(l, traits.WithNamespace("ldap")) + _ = eh.Push(l, traits.WithNamespace("ldap").With("operation", "compare")) return l + } func (l *BindLog) Title() string { @@ -328,6 +329,16 @@ func (l *SearchLog) Title() string { return l.Request.Filter } +func (l *SearchLog) IndexFields() map[string]any { + m := map[string]any{ + "operation": l.Request.Operation, + "request": l.Request, + "response": l.Response, + "metadata.baseDN": l.Request.BaseDN, + } + return m +} + func (l *CompareLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Value) } diff --git a/providers/directory/search_test.go b/providers/directory/search_test.go index 807fd4881..069515c82 100644 --- a/providers/directory/search_test.go +++ b/providers/directory/search_test.go @@ -10,6 +10,7 @@ import ( "mokapi/ldap" "mokapi/ldap/ldaptest" "mokapi/providers/directory" + "mokapi/runtime/events" "mokapi/runtime/events/eventstest" "mokapi/try" "strings" @@ -481,7 +482,7 @@ func TestSearch(t *testing.T) { name string input string reader dynamic.Reader - test func(t *testing.T, h ldap.Handler, err error) + test func(t *testing.T, h ldap.Handler, eh events.Handler, err error) }{ { name: "presence", @@ -489,7 +490,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\nmail: user@foo.com")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -500,6 +501,10 @@ func TestSearch(t *testing.T) { res := rr.Message.(*ldap.SearchResponse) require.Len(t, res.Results, 1) + + evts := eh.GetEvents(events.NewTraits()) + require.Len(t, evts, 1) + require.Equal(t, "search", evts[0].Traits.Get("operation")) }, }, { @@ -508,7 +513,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\ncn: Smith")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -527,7 +532,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\ndescription: Software Developers")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -546,7 +551,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\ndescription: Software Developers")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -565,7 +570,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\nuserAccountControl: 512")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -584,7 +589,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=user\n\ndn: cn=group\nmember: cn=user")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -603,7 +608,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: cn=uSEr\n\ndn: cn=group\nmember: cn=UseR")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -622,7 +627,7 @@ func TestSearch(t *testing.T) { reader: &dynamictest.Reader{Data: map[string]*dynamic.Config{ "file:/users.ldif": {Raw: []byte("dn: uid=ff, cn=user\n\ndn: uid=cc,cn=group\nmember: uid=ff,cn=user")}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -652,7 +657,7 @@ dn: id=user3,ou=Accounting,dc=example,dc=com foo: bar `)}, }}, - test: func(t *testing.T, h ldap.Handler, err error) { + test: func(t *testing.T, h ldap.Handler, eh events.Handler, err error) { require.NoError(t, err) rr := ldaptest.NewRecorder() @@ -677,10 +682,12 @@ foo: bar var c *directory.Config err := json.Unmarshal([]byte(tc.input), &c) if err != nil { - tc.test(t, nil, err) + tc.test(t, nil, nil, err) } else { err = c.Parse(&dynamic.Config{Data: c, Info: dynamic.ConfigInfo{Url: try.MustUrl("file:/foo.yml")}}, tc.reader) - tc.test(t, directory.NewHandler(c, enginetest.NewEngine(), &eventstest.Handler{}), err) + eh := &eventstest.Handler{} + h := directory.NewHandler(c, enginetest.NewEngine(), eh) + tc.test(t, h, eh, err) } }) } diff --git a/providers/mail/log.go b/providers/mail/log.go index df25b6e2f..b636523f4 100644 --- a/providers/mail/log.go +++ b/providers/mail/log.go @@ -45,3 +45,13 @@ func NewLogEvent(msg *smtp.Message, ctx *smtp.ClientContext, eh events.Handler, func (l *Log) Title() string { return fmt.Sprintf("%s", l.Subject) } + +func (l *Log) IndexFields() map[string]any { + m := map[string]any{ + "from": l.From, + "to": l.To, + "messageId": l.MessageId, + "subject": l.Subject, + } + return m +} diff --git a/providers/openapi/components_test.go b/providers/openapi/components_test.go index 4a8c9d932..22883bcdc 100644 --- a/providers/openapi/components_test.go +++ b/providers/openapi/components_test.go @@ -12,6 +12,8 @@ import ( "net/url" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -193,11 +195,11 @@ func TestComponents_UnmarshalYAML(t *testing.T) { func TestComponents_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "schema ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -233,11 +235,12 @@ func TestComponents_Parse(t *testing.T) { }, { name: "schema ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -249,12 +252,15 @@ func TestComponents_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse content '' failed: parse schema failed: resolve reference '/foo.yml#/components/schemas/foo' failed: TESTING ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse content '' failed: parse schema failed: resolve reference '/foo.yml#/components/schemas/foo' failed: TESTING ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "response ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -287,11 +293,12 @@ func TestComponents_Parse(t *testing.T) { }, { name: "response ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponseRef(http.StatusOK, @@ -301,12 +308,15 @@ func TestComponents_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: resolve reference '/foo.yml#/components/responses/foo' failed: TESTING ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: resolve reference '/foo.yml#/components/responses/foo' failed: TESTING ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "requestBody ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -335,11 +345,12 @@ func TestComponents_Parse(t *testing.T) { }, { name: "requestBody ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithRequestBodyRef("foo.yml#/components/requestBodies/foo"), @@ -347,12 +358,15 @@ func TestComponents_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse request body failed: resolve reference '/foo.yml#/components/requestBodies/foo' failed: TESTING ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse request body failed: resolve reference '/foo.yml#/components/requestBodies/foo' failed: TESTING ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "parameter ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -380,7 +394,7 @@ func TestComponents_Parse(t *testing.T) { }, { name: "parameter ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) @@ -395,7 +409,7 @@ func TestComponents_Parse(t *testing.T) { }, { name: "example ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -430,11 +444,12 @@ func TestComponents_Parse(t *testing.T) { }, { name: "example ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -446,12 +461,15 @@ func TestComponents_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml#/components/parameters/foo' failed: TESTING ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml#/components/parameters/foo' failed: TESTING ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "header ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{ Info: dynamic.ConfigInfo{Url: u}, @@ -483,11 +501,12 @@ func TestComponents_Parse(t *testing.T) { }, { name: "header ref error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TESTING ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -497,17 +516,18 @@ func TestComponents_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse header 'foo' failed: resolve reference '/foo.yml#/components/headers/foo' failed: TESTING ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse header 'foo' failed: resolve reference '/foo.yml#/components/headers/foo' failed: TESTING ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, } - t.Parallel() for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/providers/openapi/config.go b/providers/openapi/config.go index 7dfc9f248..9a7f53f28 100644 --- a/providers/openapi/config.go +++ b/providers/openapi/config.go @@ -42,6 +42,14 @@ func init() { } } +type Status byte + +const ( + StatusValid Status = iota + StatusInvalid + StatusValidationError +) + type Config struct { OpenApi version.Version `yaml:"openapi" json:"openapi"` Info Info `yaml:"info" json:"info"` @@ -61,6 +69,10 @@ type Config struct { Tags []*Tag `yaml:"tags,omitempty" json:"tags,omitempty"` } +type Error struct { + Message string `yaml:"message" json:"message"` +} + type ExternalDocs struct { Description string `yaml:"description" json:"description"` Url string `yaml:"url" json:"url"` @@ -92,9 +104,14 @@ func (c *Config) Validate() (err error) { err = errors.Join(err, errors.New("an openapi title is required")) } - for name := range c.Paths { + for name, path := range c.Paths { if !strings.HasPrefix(name, "/") { - err = errors.Join(err, fmt.Errorf("should only have path names that start with `/`: '%s'", name)) + if path.Value == nil { + err = errors.Join(err, fmt.Errorf("should only have path names that start with `/`: '%s'", name)) + } else { + path.Value.Status = StatusValidationError + path.Value.Errors = append(path.Value.Errors, Error{Message: "path should start with `/`"}) + } } } @@ -172,3 +189,40 @@ func (c *Config) UnmarshalJSON(b []byte) error { *c = Config(a) return nil } + +func (s Status) String() string { + switch s { + case StatusValid: + return "valid" + case StatusInvalid: + return "invalid" + case StatusValidationError: + return "validation error" + default: + return "unknown" + } +} + +func getName(cfg *dynamic.Config) string { + if c, ok := cfg.Data.(*Config); ok { + return c.Info.Name + } + return "" +} + +func (c *Config) GetStatus() Status { + for _, pt := range c.Paths { + if pt.Value == nil { + continue + } + if pt.Value.Status != StatusValid { + return pt.Value.Status + } + for _, op := range pt.Value.Operations() { + if op.Status != StatusValid { + return op.Status + } + } + } + return StatusValid +} diff --git a/providers/openapi/config_test.go b/providers/openapi/config_test.go index 03788b1dc..690e4cf0a 100644 --- a/providers/openapi/config_test.go +++ b/providers/openapi/config_test.go @@ -29,7 +29,7 @@ func TestConfig_Validate(t *testing.T) { testdata := []struct { name string data string - test func(t *testing.T, err error) + test func(t *testing.T, c *openapi.Config, err error) }{ { name: "simple valid config 3", @@ -38,7 +38,7 @@ openapi: 3 info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.NoError(t, err) }, }, @@ -49,7 +49,7 @@ openapi: 3.0 info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.NoError(t, err) }, }, @@ -60,7 +60,7 @@ openapi: 3.1 info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.NoError(t, err) }, }, @@ -71,7 +71,7 @@ openapi: 2 info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.EqualError(t, err, "not supported version: 2.0.0") }, }, @@ -81,7 +81,7 @@ info: openapi: 2.0 info: `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.EqualError(t, err, "not supported version: 2.0.0\nan openapi title is required") }, }, @@ -91,7 +91,7 @@ info: openapi: 3.0.3 info: `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.EqualError(t, err, "an openapi title is required") }, }, @@ -102,7 +102,7 @@ openapi: 3.f info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.NoError(t, err) }, }, @@ -113,7 +113,7 @@ openapi: "" info: title: foo `, - test: func(t *testing.T, err error) { + test: func(t *testing.T, _ *openapi.Config, err error) { require.EqualError(t, err, "no OpenApi version defined") }, }, @@ -126,8 +126,10 @@ info: paths: foo: {} `, - test: func(t *testing.T, err error) { - require.EqualError(t, err, "should only have path names that start with `/`: 'foo'") + test: func(t *testing.T, c *openapi.Config, err error) { + require.NoError(t, err) + require.Equal(t, openapi.StatusValidationError, c.Paths["foo"].Value.Status) + require.Equal(t, "path should start with `/`", c.Paths["foo"].Value.Errors[0].Message) }, }, } @@ -141,7 +143,7 @@ paths: c := &openapi.Config{} err := yaml.Unmarshal([]byte(tc.data), c) require.NoError(t, err) - tc.test(t, c.Validate()) + tc.test(t, c, c.Validate()) }) } } diff --git a/providers/openapi/example_test.go b/providers/openapi/example_test.go index ab9f6d492..dbb427cf4 100644 --- a/providers/openapi/example_test.go +++ b/providers/openapi/example_test.go @@ -11,6 +11,8 @@ import ( "net/url" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -146,11 +148,11 @@ bar: { summary: bar summary, value: { bar: baz }, description: bar description } func TestExample_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "Example is nil", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -168,7 +170,7 @@ func TestExample_Parse(t *testing.T) { }, { name: "Example", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -186,11 +188,12 @@ func TestExample_Parse(t *testing.T) { }, { name: "error by resolving example ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -202,12 +205,15 @@ func TestExample_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "resolve external value", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { calledReader := false reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { require.Equal(t, "https://foo.bar", u.String()) @@ -235,12 +241,10 @@ func TestExample_Parse(t *testing.T) { }, } - t.Parallel() for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/providers/openapi/handler.go b/providers/openapi/handler.go index 91e7bf989..5cee41d0d 100644 --- a/providers/openapi/handler.go +++ b/providers/openapi/handler.go @@ -67,6 +67,24 @@ func (h *responseHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { }() } + if op.Status == StatusInvalid { + msg := "operation status invalid" + if len(op.Errors) > 1 { + for _, e := range op.Errors { + msg = fmt.Sprintf("%s\n", e.Message) + } + } else if len(op.Errors) > 0 { + msg = op.Errors[0].Message + } + writeError( + rw, + r, + fmt.Errorf("operation could not be executed due to parsing errors: %s", msg), + h.config.Info.Name, + ) + return + } + status, res, err := op.getFirstSuccessResponse() if err != nil { writeError( @@ -256,7 +274,11 @@ func (h *operationHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) *H } op, errOpResolve := findOperation(r.Method, requestPath, h.config.Paths) if op != nil { - params := append(op.Path.Parameters, op.Parameters...) + var params Parameters + if op.Path != nil { + params = op.Path.Parameters + } + params = append(params, op.Parameters...) route := op.Path.Path if len(route) > 1 { // there is no official specification for trailing slash. For ease of use, mokapi considers it equivalent @@ -276,14 +298,21 @@ func (h *operationHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) *H m.RequestCounter.WithLabel(h.config.Info.Name, op.Path.Path, r.Method).Add(1) } + traits := events.NewTraits().WithNamespace("http").WithName(h.config.Info.Name).With("path", op.Path.Path).With("method", r.Method) if ctx, err := NewLogEventContext( r, op.Deprecated, - h.eh, - events.NewTraits().WithName(h.config.Info.Name).With("path", op.Path.Path).With("method", r.Method), + traits, ); err != nil { log.Errorf("unable to log http event: %v", err) } else { + defer func() { + l, _ := LogEventFromContext(ctx) + err = h.eh.Push(l, traits) + if err != nil { + log.Errorf("unable to log http event: %v", err) + } + }() r = r.WithContext(ctx) } diff --git a/providers/openapi/handler_test.go b/providers/openapi/handler_test.go index ae4b86161..fdadf7de6 100644 --- a/providers/openapi/handler_test.go +++ b/providers/openapi/handler_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "mokapi/config/dynamic" + "mokapi/config/dynamic/dynamictest" "mokapi/engine/common" "mokapi/engine/enginetest" "mokapi/providers/openapi" @@ -18,6 +19,7 @@ import ( "strings" "testing" + log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" ) @@ -735,6 +737,31 @@ func TestResolveEndpoint(t *testing.T) { require.Equal(t, `"findByStatus"`, string(b)) }, }, + { + name: "parse error", + test: func(t *testing.T, h http.HandlerFunc, c *openapi.Config) { + c.Servers[0].Url = "http://localhost/root" + op := openapitest.NewOperation( + openapitest.WithResponse( + http.StatusOK, + openapitest.WithContent( + "application/json", + openapitest.WithSchemaRef("#/invalid"), + ), + ), + ) + openapitest.AppendPath("/foo", c, openapitest.UseOperation("get", op)) + err := c.Parse(&dynamic.Config{Data: c}, &dynamictest.Reader{}) + require.NoError(t, err) + + r := httptest.NewRequest("get", "http://localhost/root/foo", nil) + r = r.WithContext(context.WithValue(r.Context(), "servicePath", "/root")) + rr := httptest.NewRecorder() + h(rr, r) + require.Equal(t, 500, rr.Code) + require.Equal(t, "operation could not be executed due to parsing errors: parse response '200' failed: parse content 'application/json' failed: parse schema failed: resolve reference '#/invalid' failed: path element 'invalid' not found\n", rr.Body.String()) + }, + }, } for _, data := range testdata { @@ -1167,7 +1194,7 @@ func TestHandler_Event(t *testing.T) { tc.test(t, func(rw http.ResponseWriter, r *http.Request) { h := openapi.NewHandler(config, eh, sm) - ctx, err := openapi.NewLogEventContext(r, false, sm, events.NewTraits()) + ctx, err := openapi.NewLogEventContext(r, false, events.NewTraits()) require.NoError(t, err) r = r.WithContext(ctx) httpErr := h.ServeHTTP(rw, r) @@ -1535,6 +1562,7 @@ func TestHandler_Parameter(t *testing.T) { { name: "path parameter missing", test: func(t *testing.T, h http.HandlerFunc, c *openapi.Config) { + log.SetLevel(log.WarnLevel) hook := test.NewGlobal() op := openapitest.NewOperation( diff --git a/providers/openapi/header_test.go b/providers/openapi/header_test.go index 79f662d1c..0aff11ee4 100644 --- a/providers/openapi/header_test.go +++ b/providers/openapi/header_test.go @@ -12,6 +12,8 @@ import ( "net/url" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -233,11 +235,11 @@ func TestHeader_UnmarshalYAML(t *testing.T) { func TestHeader_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "reference is nil", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -255,7 +257,7 @@ func TestHeader_Parse(t *testing.T) { }, { name: "header", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -275,11 +277,12 @@ func TestHeader_Parse(t *testing.T) { }, { name: "error by resolving example ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -288,17 +291,18 @@ func TestHeader_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse header 'foo' failed: resolve reference '/foo' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse header 'foo' failed: resolve reference '/foo' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, } - t.Parallel() for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/providers/openapi/log.go b/providers/openapi/log.go index febfb5d04..4acc07955 100644 --- a/providers/openapi/log.go +++ b/providers/openapi/log.go @@ -3,7 +3,6 @@ package openapi import ( "context" "encoding/json" - "fmt" "mokapi/engine/common" "mokapi/lib" "mokapi/runtime/events" @@ -48,7 +47,7 @@ type HttpParameter struct { Raw *string `json:"raw"` } -func NewLogEventContext(r *http.Request, deprecated bool, eh events.Handler, traits events.Traits) (context.Context, error) { +func NewLogEventContext(r *http.Request, deprecated bool, traits events.Traits) (context.Context, error) { l := &HttpLog{ Request: &HttpRequestLog{ Method: r.Method, @@ -97,10 +96,6 @@ func NewLogEventContext(r *http.Request, deprecated bool, eh events.Handler, tra } - err := eh.Push(l, traits.WithNamespace("http")) - if err != nil { - return nil, err - } ctx := context.WithValue(r.Context(), logKey, l) return ctx, nil @@ -112,7 +107,21 @@ func LogEventFromContext(ctx context.Context) (*HttpLog, bool) { } func (l *HttpLog) Title() string { - return fmt.Sprintf("%s %s", l.Request.Method, l.Request.Url) + return l.Request.Url +} + +func (l *HttpLog) IndexFields() map[string]any { + m := map[string]any{ + "request": l.Request, + "response": l.Response, + } + if l.Request != nil { + m["method"] = l.Request.Method + } + if l.Response != nil { + m["statusCode"] = l.Response.StatusCode + } + return m } func (l *HttpRequestLog) setParams(name string, params map[string]RequestParameterValue) { diff --git a/providers/openapi/log_test.go b/providers/openapi/log_test.go index 5f69a69ad..66701c3d3 100644 --- a/providers/openapi/log_test.go +++ b/providers/openapi/log_test.go @@ -4,7 +4,6 @@ import ( "context" "mokapi/providers/openapi" "mokapi/runtime/events" - "mokapi/runtime/events/eventstest" "net/http/httptest" "testing" @@ -24,7 +23,7 @@ func TestLog(t *testing.T) { r = r.WithContext(openapi.NewContext(context.Background(), &openapi.RequestParameters{})) - ctx, err := openapi.NewLogEventContext(r, false, &eventstest.Handler{}, events.NewTraits()) + ctx, err := openapi.NewLogEventContext(r, false, events.NewTraits()) require.NoError(t, err) require.NotNil(t, ctx) @@ -52,7 +51,7 @@ func TestLog(t *testing.T) { } r = r.WithContext(openapi.NewContext(context.Background(), params)) - ctx, err := openapi.NewLogEventContext(r, false, &eventstest.Handler{}, events.NewTraits()) + ctx, err := openapi.NewLogEventContext(r, false, events.NewTraits()) require.NoError(t, err) require.NotNil(t, ctx) @@ -80,7 +79,7 @@ func TestLog(t *testing.T) { } r = r.WithContext(openapi.NewContext(context.Background(), params)) - ctx, err := openapi.NewLogEventContext(r, false, &eventstest.Handler{}, events.NewTraits()) + ctx, err := openapi.NewLogEventContext(r, false, events.NewTraits()) require.NoError(t, err) require.NotNil(t, ctx) diff --git a/providers/openapi/media_type_test.go b/providers/openapi/media_type_test.go index 6770e4771..408ae855e 100644 --- a/providers/openapi/media_type_test.go +++ b/providers/openapi/media_type_test.go @@ -12,6 +12,8 @@ import ( "net/url" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -101,11 +103,11 @@ examples: func TestMediaType_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "no error", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -123,7 +125,7 @@ func TestMediaType_Parse(t *testing.T) { }, { name: "MediaType is nil", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -141,11 +143,12 @@ func TestMediaType_Parse(t *testing.T) { }, { name: "error by resolving schema ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -158,16 +161,20 @@ func TestMediaType_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse content 'application/json' failed: parse schema failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse content 'application/json' failed: parse schema failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "error by resolving example ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, @@ -179,17 +186,18 @@ func TestMediaType_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse content 'application/json' failed: parse example 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, } - t.Parallel() for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/providers/openapi/openapitest/operation.go b/providers/openapi/openapitest/operation.go index 5e11eaa6c..ef6a5f5fb 100644 --- a/providers/openapi/openapitest/operation.go +++ b/providers/openapi/openapitest/operation.go @@ -199,3 +199,10 @@ func WithTagName(name string) OperationOptions { o.Tags = append(o.Tags, name) } } + +func WithOperationErrors(err ...openapi.Error) OperationOptions { + return func(o *openapi.Operation) { + o.Status = openapi.StatusInvalid + o.Errors = append(o.Errors, err...) + } +} diff --git a/providers/openapi/openapitest/path.go b/providers/openapi/openapitest/path.go index b266dd730..006b012c9 100644 --- a/providers/openapi/openapitest/path.go +++ b/providers/openapi/openapitest/path.go @@ -87,3 +87,10 @@ func WithPathParamRef(ref string) PathOptions { }) } } + +func WithPathErrors(err ...openapi.Error) PathOptions { + return func(p *openapi.Path) { + p.Status = openapi.StatusInvalid + p.Errors = err + } +} diff --git a/providers/openapi/operation.go b/providers/openapi/operation.go index 6ecac2aa1..9ad3d0450 100644 --- a/providers/openapi/operation.go +++ b/providers/openapi/operation.go @@ -47,7 +47,9 @@ type Operation struct { Security []SecurityRequirement `yaml:"security" json:"security"` - Path *Path `yaml:"-" json:"-"` + Path *Path `yaml:"-" json:"-"` + Status Status `yaml:"-" json:"-"` + Errors []Error `yaml:"-" json:"-"` } func (o *Operation) getFirstSuccessResponse() (int, *Response, error) { @@ -84,13 +86,11 @@ func (o *Operation) getResponse(statusCode int) *Response { return o.Responses.GetResponse(statusCode) } -func (o *Operation) Parse(p *Path, config *dynamic.Config, reader dynamic.Reader) error { +func (o *Operation) Parse(config *dynamic.Config, reader dynamic.Reader) error { if o == nil { return nil } - o.Path = p - if err := o.Parameters.Parse(config, reader); err != nil { return err } diff --git a/providers/openapi/operation_test.go b/providers/openapi/operation_test.go index 7a2312c92..733c92708 100644 --- a/providers/openapi/operation_test.go +++ b/providers/openapi/operation_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -201,11 +203,11 @@ func TestOperation_UnmarshalYAML(t *testing.T) { func TestOperation_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "get", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -222,22 +224,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "get error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "post", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -254,22 +260,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "post error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodPost, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'POST' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "POST", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodPost).Status) + require.NoError(t, err) }, }, { name: "put", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -286,22 +296,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "put error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodPut, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'PUT' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "PUT", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodPut).Status) + require.NoError(t, err) }, }, { name: "patch", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -318,22 +332,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "patch error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodPatch, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'PATCH' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "PATCH", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodPatch).Status) + require.NoError(t, err) }, }, { name: "delete", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -350,22 +368,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "delete error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodDelete, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'DELETE' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "DELETE", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodDelete).Status) + require.NoError(t, err) }, }, { name: "head", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -382,22 +404,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "head error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodHead, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'HEAD' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "HEAD", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodHead).Status) + require.NoError(t, err) }, }, { name: "options", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -414,22 +440,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "options error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodOptions, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'OPTIONS' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "OPTIONS", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodOptions).Status) + require.NoError(t, err) }, }, { name: "trace", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -446,22 +476,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "trace error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodTrace, openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'TRACE' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "TRACE", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodTrace).Status) + require.NoError(t, err) }, }, { name: "query", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -478,22 +512,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "query error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation("QUERY", openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'QUERY' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "QUERY", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation("QUERY").Status) + require.NoError(t, err) }, }, { name: "custom LINK", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -510,22 +548,26 @@ func TestOperation_Parse(t *testing.T) { }, { name: "custom LINK error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation("LINK", openapitest.WithOperationParamRef("foo.yml"))), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'LINK' failed: parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "LINK", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse parameter index '0' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation("LINK").Status) + require.NoError(t, err) }, }, { name: "request body", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -544,7 +586,7 @@ func TestOperation_Parse(t *testing.T) { }, { name: "request body reference", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(u *url.URL, _ any) (*dynamic.Config, error) { cfg := &dynamic.Config{Info: dynamic.ConfigInfo{Url: u}, Data: openapitest.NewConfig("3.0", @@ -568,11 +610,12 @@ func TestOperation_Parse(t *testing.T) { }, { name: "request body reference error", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.UseOperation(http.MethodTrace, &openapi.Operation{ RequestBody: &openapi.RequestBodyRef{Reference: dynamic.Reference[*openapi.RequestBodyRef]{Ref: "foo.yml#/components/requestBodies/foo"}}, @@ -580,17 +623,18 @@ func TestOperation_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'TRACE' failed: parse request body failed: resolve reference '/foo.yml#/components/requestBodies/foo' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "TRACE", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse request body failed: resolve reference '/foo.yml#/components/requestBodies/foo' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodTrace).Status) + require.NoError(t, err) }, }, } - t.Parallel() for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/providers/openapi/path.go b/providers/openapi/path.go index c1ceaaf8a..c5a64a705 100644 --- a/providers/openapi/path.go +++ b/providers/openapi/path.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -60,7 +61,9 @@ type Path struct { // but cannot be removed there Parameters Parameters - Path string `yaml:"-" json:"-"` + Path string `yaml:"-" json:"-"` + Status Status `yaml:"-" json:"-"` + Errors []Error `yaml:"-" json:"-"` } func (r *PathRef) UnmarshalJSON(b []byte) error { @@ -156,23 +159,28 @@ func (p PathItems) Resolve(token string) (interface{}, error) { } func (p PathItems) Parse(config *dynamic.Config, reader dynamic.Reader) error { + if p == nil { + return nil + } + for name, e := range p { - if err := e.Parse(name, config, reader); err != nil { + if e == nil { + continue + } + if err := e.Parse(config, reader); err != nil { return fmt.Errorf("parse path '%v' failed: %w", name, err) } + if e.Value != nil { + e.Value.Path = name + } } return nil } -func (r *PathRef) Parse(name string, config *dynamic.Config, reader dynamic.Reader) error { +func (r *PathRef) Parse(config *dynamic.Config, reader dynamic.Reader) error { if r == nil { return nil } - defer func() { - if r.Value != nil { - r.Value.Path = name - } - }() if len(r.Ref) > 0 { resolved, err := r.Resolve(config, reader) @@ -197,36 +205,35 @@ func (p *Path) Parse(config *dynamic.Config, reader dynamic.Reader) error { } } - if err := p.Get.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'GET' failed: %w", err) - } - if err := p.Post.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'POST' failed: %w", err) - } - if err := p.Put.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'PUT' failed: %w", err) - } - if err := p.Patch.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'PATCH' failed: %w", err) - } - if err := p.Delete.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'DELETE' failed: %w", err) - } - if err := p.Head.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'HEAD' failed: %w", err) - } - if err := p.Options.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'OPTIONS' failed: %w", err) - } - if err := p.Trace.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'TRACE' failed: %w", err) - } - if err := p.Query.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation 'QUERY' failed: %w", err) + for method, op := range p.Operations() { + err := op.Parse(config, reader) + if err != nil { + op.Status = StatusInvalid + method = strings.ToUpper(method) + op.Errors = append(op.Errors, Error{Message: err.Error()}) + log. + WithField("api", getName(config)). + WithField("method", method). + WithField("path", p.Path). + WithField("namespace", "http"). + Error(err) + } else { + op.Path = p + } } + for name, op := range p.AdditionalOperations { - if err := op.Parse(p, config, reader); err != nil { - return fmt.Errorf("parse operation '%s' failed: %w", name, err) + if err := op.Parse(config, reader); err != nil { + op.Status = StatusInvalid + name = strings.ToUpper(name) + log. + WithField("api", getName(config)). + WithField("method", name). + WithField("path", p.Path). + WithField("namespace", "http"). + Error(err) + } else { + op.Path = p } } diff --git a/providers/openapi/response_test.go b/providers/openapi/response_test.go index 34e609fd4..266966fc6 100644 --- a/providers/openapi/response_test.go +++ b/providers/openapi/response_test.go @@ -15,6 +15,8 @@ import ( "strconv" "testing" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" ) @@ -300,11 +302,11 @@ func TestResponse_GetContent(t *testing.T) { func TestResponse_Parse(t *testing.T) { testcases := []struct { name string - test func(t *testing.T) + test func(t *testing.T, log *test.Hook) }{ { name: "no refs", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -321,7 +323,7 @@ func TestResponse_Parse(t *testing.T) { }, { name: "responses is nil", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -336,7 +338,7 @@ func TestResponse_Parse(t *testing.T) { }, { name: "ResponseRef is nil", - test: func(t *testing.T) { + test: func(t *testing.T, _ *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, nil }) @@ -353,11 +355,12 @@ func TestResponse_Parse(t *testing.T) { }, { name: "error by resolving response ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponseRef(http.StatusOK, "foo.yml"), @@ -365,16 +368,20 @@ func TestResponse_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, { name: "error by resolving header ref", - test: func(t *testing.T) { + test: func(t *testing.T, log *test.Hook) { reader := dynamictest.ReaderFunc(func(_ *url.URL, _ any) (*dynamic.Config, error) { return nil, fmt.Errorf("TEST ERROR") }) config := openapitest.NewConfig("3.0", + openapitest.WithInfo("HTTP API", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK, openapitest.WithResponseHeaderRef("foo", "foo.yml")), @@ -382,17 +389,19 @@ func TestResponse_Parse(t *testing.T) { ), ) err := config.Parse(&dynamic.Config{Info: dynamic.ConfigInfo{Url: &url.URL{}}, Data: config}, reader) - require.EqualError(t, err, "parse path '/foo' failed: parse operation 'GET' failed: parse response '200' failed: parse header 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR") + require.Equal(t, logrus.Fields{"method": "GET", "api": "HTTP API", "namespace": "http", "path": "/foo"}, log.LastEntry().Data) + require.Equal(t, "parse response '200' failed: parse header 'foo' failed: resolve reference '/foo.yml' failed: TEST ERROR", log.LastEntry().Message) + require.Equal(t, openapi.StatusInvalid, config.Paths["/foo"].Value.Operation(http.MethodGet).Status) + require.NoError(t, err) }, }, } - t.Parallel() for _, tc := range testcases { tc := tc t.Run(tc.name, func(t *testing.T) { - t.Parallel() - tc.test(t) + hook := test.NewGlobal() + tc.test(t, hook) }) } } diff --git a/runtime/events/events.go b/runtime/events/events.go index 60a293067..af3b5cd3f 100644 --- a/runtime/events/events.go +++ b/runtime/events/events.go @@ -40,9 +40,11 @@ func (m *StoreManager) Push(data EventData, traits Traits) error { score := 0 var bestStore *store for _, s := range m.stores { - if s.traits.Match(traits) && len(s.traits) > score { - bestStore = s - score = len(s.traits) + if s.traits.Match(traits) { + if bestStore == nil || len(s.traits) > score { + bestStore = s + score = len(s.traits) + } } } diff --git a/runtime/events/events_test.go b/runtime/events/events_test.go index 1aad07302..2aa6e49dd 100644 --- a/runtime/events/events_test.go +++ b/runtime/events/events_test.go @@ -43,6 +43,14 @@ func TestPush(t *testing.T) { require.NoError(t, err) }, }, + { + "store with empty traits", + func(t *testing.T, sm *events.StoreManager) { + sm.SetStore(10, events.NewTraits()) + err := sm.Push(nil, events.NewTraits().With("level", "debug")) + require.NoError(t, err) + }, + }, { "store matches", func(t *testing.T, sm *events.StoreManager) { diff --git a/runtime/events/eventstest/event.go b/runtime/events/eventstest/event.go index 7b6563265..0f8f9faba 100644 --- a/runtime/events/eventstest/event.go +++ b/runtime/events/eventstest/event.go @@ -11,6 +11,12 @@ func (e *Event) Title() string { return e.Name } +func (e *Event) IndexFields() map[string]any { + return map[string]any{ + "name": e.Name, + } +} + type Handler struct { Events []events.Event } @@ -23,7 +29,7 @@ func (h *Handler) Push(data events.EventData, traits events.Traits) error { func (h *Handler) GetEvents(traits events.Traits) []events.Event { var result []events.Event for _, e := range h.Events { - if e.Traits.Match(traits) { + if traits.Match(e.Traits) { result = append(result, e) } } diff --git a/runtime/events/index.go b/runtime/events/index.go index 13de8e724..e987dd8cc 100644 --- a/runtime/events/index.go +++ b/runtime/events/index.go @@ -4,16 +4,12 @@ import ( "fmt" "mokapi/runtime/search" "reflect" + "strings" "time" ) -type eventIndex struct { - Type string `json:"type"` - Discriminator string `json:"discriminator"` - Api string `json:"api"` - Title string `json:"_title" index:"false" store:"true"` - Event *Event `json:"event"` - Time string `json:"_time"` +type IndexFieldsProvider interface { + IndexFields() map[string]any } func (m *StoreManager) addToIndex(event *Event) { @@ -21,15 +17,24 @@ func (m *StoreManager) addToIndex(event *Event) { return } - data := eventIndex{ - Type: "event", - Discriminator: fmt.Sprintf("event_%s", event.Traits.String()), - Api: getApiFromEvent(event), - Event: event, - Time: event.Time.Format(time.RFC3339), + data := map[string]any{ + "api": event.Traits.GetName(), + "type": "event", + "discriminator": fmt.Sprintf("event_%s", event.Traits.String()), + "domain": getDomainFromEvent(event), + "id": event.Id, + "traits": event.Traits, + "_time": event.Time.Format(time.RFC3339), } + if event.Data != nil { - data.Title = event.Data.Title() + data["_title"] = event.Data.Title() + + if p, ok := event.Data.(IndexFieldsProvider); ok { + for k, v := range p.IndexFields() { + data[k] = v + } + } } m.index.Add(event.Id, data) @@ -39,27 +44,43 @@ func GetSearchResult(fields map[string]string, _ []string) (search.ResultItem, e result := search.ResultItem{ Type: "Event", Title: fields["_title"], - Domain: fields["event.data.api"], + Domain: fields["domain"], Time: fields["_time"], } - if fields["event.traits.namespace"] == "kafka" { - result.Domain = fmt.Sprintf("%s - %s", fields["event.data.api"], fields["event.traits.topic"]) - } result.Params = map[string]string{ - "namespace": fields["event.traits.namespace"], - "id": fields["event.id"], + "type": "event", + "id": fields["id"], + } + for k, v := range fields { + if strings.HasPrefix(k, "traits.") { + result.Params[k] = v + } else if strings.HasPrefix(k, "metadata.") { + k = strings.Replace(k, "metadata.", "", 1) + result.Params[k] = v + } } return result, nil } -func getApiFromEvent(event *Event) string { +func getDomainFromEvent(event *Event) string { + if d, ok := event.Data.(DomainData); ok { + return d.Domain() + } + return getDataField(event, "Api") +} + +type DomainData interface { + Domain() string +} + +func getDataField(event *Event, field string) string { if event.Data == nil { return "" } - f := reflect.ValueOf(event.Data).Elem().FieldByName("Api") + f := reflect.ValueOf(event.Data).Elem().FieldByName(field) if f.IsValid() { return f.String() } - return "" + return event.Traits.GetName() } diff --git a/runtime/events/index_test.go b/runtime/events/index_test.go index df46cc77d..6bb4adc1c 100644 --- a/runtime/events/index_test.go +++ b/runtime/events/index_test.go @@ -45,7 +45,7 @@ func TestIndex_Http(t *testing.T) { require.Equal(t, "My API", r.Results[0].Domain) require.Equal(t, "foo", r.Results[0].Title) require.Equal(t, []string{"foo"}, r.Results[0].Fragments) - require.Equal(t, "test", r.Results[0].Params["namespace"]) + require.Equal(t, "test", r.Results[0].Params["traits.namespace"]) require.NotEmpty(t, r.Results[0].Params["id"]) }, }, @@ -73,7 +73,7 @@ func TestIndex_Http(t *testing.T) { require.Equal(t, "My API", r.Results[0].Domain) require.Equal(t, "foo", r.Results[0].Title) require.Equal(t, []string{"event"}, r.Results[0].Fragments) - require.Equal(t, "test", r.Results[0].Params["namespace"]) + require.Equal(t, "test", r.Results[0].Params["traits.namespace"]) require.NotEmpty(t, r.Results[0].Params["id"]) }, }, @@ -125,7 +125,7 @@ func TestIndex_Http(t *testing.T) { require.Equal(t, "My API", r.Results[0].Domain) require.Equal(t, "bar", r.Results[0].Title) require.Equal(t, []string{"event"}, r.Results[0].Fragments) - require.Equal(t, "test", r.Results[0].Params["namespace"]) + require.Equal(t, "test", r.Results[0].Params["traits.namespace"]) require.NotEmpty(t, r.Results[0].Params["id"]) }, }, diff --git a/runtime/events/traits.go b/runtime/events/traits.go index 1949acfe5..eca471565 100644 --- a/runtime/events/traits.go +++ b/runtime/events/traits.go @@ -2,6 +2,7 @@ package events import ( "fmt" + "slices" "strings" ) @@ -52,14 +53,19 @@ func (t Traits) String() string { } sb.WriteString("name=" + n) } - for k, v := range t { + var names []string + for k, _ := range t { if k == namespace || k == name { continue } + names = append(names, k) + } + slices.SortFunc(names, strings.Compare) + for _, n := range names { if sb.Len() > 0 { sb.WriteString(", ") } - sb.WriteString(fmt.Sprintf("%v=%v", k, v)) + sb.WriteString(fmt.Sprintf("%v=%v", n, t[n])) } return sb.String() } diff --git a/runtime/index.go b/runtime/index.go index d7188ed00..248f02594 100644 --- a/runtime/index.go +++ b/runtime/index.go @@ -2,6 +2,7 @@ package runtime import ( "context" + "fmt" "mokapi/config/static" "mokapi/runtime/events" "mokapi/runtime/search" @@ -10,6 +11,8 @@ import ( "path/filepath" "regexp" "slices" + "sync" + "time" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" @@ -33,17 +36,21 @@ var fieldsNotIncludedInAll = []string{"api"} var SupportedFacets = []string{"type"} type SearchIndex struct { - cfg static.Search - idx bleve.Index - ready chan struct{} - queue chan func() + cfg static.Search + idx bleve.Index + ready chan struct{} + queue chan indexOp + initWG sync.WaitGroup // tracks initial items } func newSearchIndex(cfg static.Search) *SearchIndex { s := &SearchIndex{cfg: cfg} if cfg.Enabled { s.ready = make(chan struct{}) - s.queue = make(chan func(), 1000) + s.queue = make(chan indexOp, 1000) + if cfg.NumIndexWorker == 0 { + s.cfg.NumIndexWorker = 1 + } } return s } @@ -63,6 +70,10 @@ func (s *SearchIndex) start(pool *safe.Pool) { docMapping.AddFieldMappingsAt("_time", disableIndex) docMapping.AddFieldMappingsAt("discriminator", disableIndex) + metaMapping := bleve.NewDocumentMapping() + metaMapping.AddFieldMappingsAt("*", disableIndex) + docMapping.AddSubDocumentMapping("meta", metaMapping) + apiField := bleve.NewTextFieldMapping() apiField.Analyzer = "mokapi_analyzer" apiField.IncludeInAll = false // Exclude from default search @@ -75,6 +86,8 @@ func (s *SearchIndex) start(pool *safe.Pool) { defaultField.IncludeTermVectors = true docMapping.AddFieldMappingsAt("*", defaultField) + AddMappings(docMapping) + mapping := bleve.NewIndexMapping() mapping.DefaultMapping = docMapping mapping.DefaultAnalyzer = "mokapi_analyzer" @@ -111,57 +124,92 @@ func (s *SearchIndex) start(pool *safe.Pool) { return } -initialization: - for { - select { - case op := <-s.queue: - op() - default: - close(s.ready) - break initialization - } + for i := 0; i < 1; i++ { + pool.Go(s.runWorker) } - pool.Go(func(ctx context.Context) { - for { - select { - case op, ok := <-s.queue: - if !ok { - return - } - op() - case <-ctx.Done(): - close(s.queue) - - if !s.cfg.InMemory { - indexPath := getSearchIndexPath(s.cfg) - if indexPath != "" { - _ = os.RemoveAll(indexPath) - } - } + s.initWG.Wait() + // ensure last batch is flushed + time.Sleep(500 * time.Millisecond) + close(s.ready) +} - return - } - } - }) +type opType int + +const ( + opAdd opType = iota + opDelete +) + +type indexOp struct { + id string + data any + typ opType } func (s *SearchIndex) Add(id string, data any) { if !s.cfg.Enabled { return } - s.queue <- func() { - s.add(id, data) + + s.initWG.Add(1) + s.queue <- indexOp{ + id: id, + data: data, + typ: opAdd, } } -func (s *SearchIndex) add(id string, data any) { - if s.idx == nil { - return +func (s *SearchIndex) runWorker(ctx context.Context) { + batch := s.idx.NewBatch() + batchSize := 0 + + const maxBatchSize = 100 + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + + flush := func() { + if batchSize == 0 { + return + } + + if err := s.idx.Batch(batch); err != nil { + println("batch failed:", err.Error()) + } + + s.initWG.Add(-batchSize) + + batch = s.idx.NewBatch() + batchSize = 0 } - err := s.idx.Index(id, data) - if err != nil { - log.Errorf("add '%s' to search index failed: %v", id, err) + + for { + select { + case op, ok := <-s.queue: + if !ok { + return + } + + if op.typ == opAdd { + err := batch.Index(op.id, op.data) + if err != nil { + log.Errorf("add '%s' to search index failed: %v", op.id, err) + } + } else if op.typ == opDelete { + batch.Delete(op.id) + } + batchSize++ + + if batchSize >= maxBatchSize { + flush() + } + + case <-ticker.C: + flush() + + case <-ctx.Done(): + return + } } } @@ -169,8 +217,10 @@ func (s *SearchIndex) Delete(id string) { if !s.cfg.Enabled { return } - s.queue <- func() { - _ = s.idx.Delete(id) + s.initWG.Add(1) + s.queue <- indexOp{ + id: id, + typ: opDelete, } } @@ -193,10 +243,19 @@ func (s *SearchIndex) Search(r search.Request) (search.Result, error) { clauses = append(clauses, q) } - for k, v := range params { - term := bleve.NewMatchPhraseQuery(v) - term.SetField(k) - clauses = append(clauses, term) + for _, p := range params { + term := bleve.NewMatchPhraseQuery(p.value) + term.SetField(p.key) + bq := bleve.NewBooleanQuery() + switch p.operator { + case "+": + bq.AddMust(term) + case "-": + bq.AddMustNot(term) + default: + bq.AddShould(term) + } + clauses = append(clauses, bq) } qFacetsValues := make([]query.Query, len(clauses)) @@ -284,25 +343,39 @@ func (s *SearchIndex) Search(r search.Request) (search.Result, error) { func getSearchFields(doc index.Document) map[string]string { m := make(map[string]string) doc.VisitFields(func(field index.Field) { - m[field.Name()] = string(field.Value()) + var value string + switch f := field.(type) { + case index.NumericField: + v, _ := f.Number() + value = fmt.Sprintf("%v", v) + default: + value = string(field.Value()) + } + m[field.Name()] = value }) return m } -func parseQuery(query string) (string, map[string]string) { - re := regexp.MustCompile(`([\w.]+):("[^"]+"|\S+)`) +type param struct { + key string + value string + operator string +} + +func parseQuery(query string) (string, []param) { + re := regexp.MustCompile(`([+-]?)([\w.]+):("[^"]+"|\S+)`) - params := make(map[string]string) matches := re.FindAllStringSubmatch(query, -1) + var params []param s := query for _, m := range matches { - key := m[1] + key := m[2] if !slices.Contains(fieldsNotIncludedInAll, key) { continue } - value := strings.Trim(m[2], `"`) - params[key] = value + value := strings.Trim(m[3], `"`) + params = append(params, param{key: key, value: value, operator: m[1]}) s = strings.Replace(s, m[0], "", 1) } diff --git a/runtime/index_test.go b/runtime/index_test.go index eadffe8d2..5c20831da 100644 --- a/runtime/index_test.go +++ b/runtime/index_test.go @@ -26,7 +26,7 @@ func TestIndex(t *testing.T) { app.Start(pool) defer pool.Stop() - trait := events.NewTraits().WithNamespace("test") + trait := events.NewTraits().WithNamespace("test").WithName("Swagger Petstore") app.Events.SetStore(10, trait) err := app.Events.Push(&eventstest.Event{ Name: "foo", @@ -34,16 +34,16 @@ func TestIndex(t *testing.T) { }, trait) require.NoError(t, err) - petstore := openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server. You can find out more about Swagger at https://swagger.io"), + petStore := openapitest.NewConfig("3.0", openapitest.WithInfo("Swagger Petstore", "1.0", "This is a sample server Petstore server. You can find out more about Swagger at https://swagger.io"), openapitest.WithPath("/pet", openapitest.WithOperation("put", openapitest.WithOperationInfo("Update an existing pet.", "Update an existing pet by Id.", "updatePet", false), ), ), ) - app.AddHttp(&dynamic.Config{ + app.Http.Add(&dynamic.Config{ Info: dynamic.ConfigInfo{Url: try.MustUrl("petstore.yaml"), Provider: "NPM"}, - Data: petstore, + Data: petStore, }) result, err := app.Search(search.Request{ diff --git a/runtime/logs.go b/runtime/logs.go new file mode 100644 index 000000000..fe021daca --- /dev/null +++ b/runtime/logs.go @@ -0,0 +1,92 @@ +package runtime + +import ( + "fmt" + "mokapi/config/static" + "mokapi/runtime/events" + "strings" + + stdlog "log" + + log "github.com/sirupsen/logrus" +) + +type LogHook struct { + sm events.Handler + enabled bool +} + +type LogData struct { + Message string `json:"message"` + Level string `json:"level"` +} + +func NewLogHook(sm events.Handler) *LogHook { + return &LogHook{sm: sm, enabled: true} +} + +func (h *LogHook) Levels() []log.Level { + return log.AllLevels +} + +func (h *LogHook) Fire(entry *log.Entry) error { + if !h.enabled { + return nil + } + + data := &LogData{ + Message: entry.Message, + Level: entry.Level.String(), + } + + traits := extractTraits(entry) + + return h.sm.Push(data, traits) +} + +func (h *LogHook) Disable() { + h.enabled = false +} + +func (d *LogData) Title() string { + msg := d.Message + if len(d.Message) > 50 { + msg = msg[:47] + "..." + } + return fmt.Sprintf("%v: %v", d.Level, msg) +} + +func extractTraits(entry *log.Entry) events.Traits { + traits := events.NewTraits() + if ns, ok := entry.Data["namespace"].(string); ok { + traits.WithNamespace(ns) + } + if api, ok := entry.Data["api"]; ok { + traits.WithName(api.(string)) + } + traits.With("level", entry.Level.String()) + traits.With("type", "log") + return traits +} + +func configureLogging(cfg *static.Config) { + stdlog.SetFlags(stdlog.Lshortfile | stdlog.LstdFlags) + + level := log.InfoLevel + if cfg.Log.Level != "" { + var err error + level, err = log.ParseLevel(cfg.Log.Level) + if err != nil { + log.Errorf("error parsing log level: %v", err.Error()) + + } + } + log.SetLevel(level) + + if strings.ToLower(cfg.Log.Format) == "json" { + log.SetFormatter(&log.JSONFormatter{}) + } else { + formatter := &log.TextFormatter{DisableColors: false, FullTimestamp: true, DisableSorting: true} + log.SetFormatter(formatter) + } +} diff --git a/runtime/metrics/counter.go b/runtime/metrics/counter.go index 79279131c..e1dd692fe 100644 --- a/runtime/metrics/counter.go +++ b/runtime/metrics/counter.go @@ -126,13 +126,15 @@ func (m *CounterMap) Value(query *Query) float64 { return v } -func (m *CounterMap) Sum() float64 { +func (m *CounterMap) Sum(query *Query) float64 { m.m.Lock() defer m.m.Unlock() var v float64 for _, c := range m.counters { - v += c.Value() + if c.Info().Match(query) { + v += c.Value() + } } return v } @@ -154,3 +156,15 @@ func (m *CounterMap) Reset() { c.Reset() } } + +func (m *CounterMap) FindOne(query *Query) (*Counter, bool) { + m.m.Lock() + defer m.m.Unlock() + + for _, c := range m.counters { + if c.Info().Match(query) { + return c, true + } + } + return nil, false +} diff --git a/runtime/metrics/gauge.go b/runtime/metrics/gauge.go index 64c6b26bd..9d28fe1a4 100644 --- a/runtime/metrics/gauge.go +++ b/runtime/metrics/gauge.go @@ -120,11 +120,11 @@ func (m *GaugeMap) Value(query *Query) float64 { return v } -func (m *GaugeMap) FindAll(query *Query) []Metric { +func (m *GaugeMap) FindAll(query *Query) []*Gauge { m.m.Lock() defer m.m.Unlock() - result := make([]Metric, 0) + result := make([]*Gauge, 0) for _, c := range m.gauges { if c.Info().Match(query) { result = append(result, c) @@ -133,6 +133,18 @@ func (m *GaugeMap) FindAll(query *Query) []Metric { return result } +func (m *GaugeMap) FindOne(query *Query) (*Gauge, bool) { + m.m.Lock() + defer m.m.Unlock() + + for _, c := range m.gauges { + if c.Info().Match(query) { + return c, true + } + } + return nil, false +} + func (m *GaugeMap) Collect(ch chan<- Metric) { m.m.Lock() defer m.m.Unlock() @@ -170,3 +182,19 @@ func (m *GaugeMap) MarshalJSON() ([]byte, error) { } return json.Marshal(result) } + +func (m *GaugeMap) Max(query *Query) float64 { + m.m.Lock() + defer m.m.Unlock() + + var maxValue float64 + for _, c := range m.gauges { + if c.Info().Match(query) { + v := c.Value() + if v > maxValue { + maxValue = v + } + } + } + return maxValue +} diff --git a/runtime/metrics/metrics.go b/runtime/metrics/metrics.go index 199fc1045..a899e8c80 100644 --- a/runtime/metrics/metrics.go +++ b/runtime/metrics/metrics.go @@ -43,6 +43,14 @@ func (i *Info) getLabel(name string) (*Label, bool) { return nil, false } +func (i *Info) GetLabel(name string) string { + l, ok := i.getLabel(name) + if !ok { + return "" + } + return l.Value +} + func (i *Info) FQName() string { if len(i.Namespace) == 0 { return i.Name diff --git a/runtime/metrics/metrics_test.go b/runtime/metrics/metrics_test.go index 899924edd..c717e7c03 100644 --- a/runtime/metrics/metrics_test.go +++ b/runtime/metrics/metrics_test.go @@ -2,8 +2,9 @@ package metrics import ( "encoding/json" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) func TestCounter_Add(t *testing.T) { @@ -48,7 +49,7 @@ func TestNewCounterMap(t *testing.T) { f: func(t *testing.T) { c := NewCounterMap(WithName("foo"), WithLabelNames("foo")) c.WithLabel("bar").Add(1) - require.Equal(t, 1.0, c.Sum()) + require.Equal(t, 1.0, c.Sum(NewQuery())) }, }, { diff --git a/runtime/monitor/http_test.go b/runtime/monitor/http_test.go index 6e4e15851..3b300e281 100644 --- a/runtime/monitor/http_test.go +++ b/runtime/monitor/http_test.go @@ -11,13 +11,13 @@ import ( func TestHttp_Metrics_Request_Total(t *testing.T) { h := NewHttp() h.RequestCounter.WithLabel("service_a", "endpoint_a", "post").Add(1) - require.Equal(t, float64(1), h.RequestCounter.Sum()) + require.Equal(t, float64(1), h.RequestCounter.Sum(metrics.NewQuery())) } func TestHttp_Metrics_Request_Errors_Total(t *testing.T) { h := NewHttp() h.RequestErrorCounter.WithLabel("service_a", "endpoint_a", "put").Add(1) - require.Equal(t, float64(1), h.RequestErrorCounter.Sum()) + require.Equal(t, float64(1), h.RequestErrorCounter.Sum(metrics.NewQuery())) } func TestHttp_Metrics_LastRequest(t *testing.T) { diff --git a/runtime/monitor/kafka_test.go b/runtime/monitor/kafka_test.go index b86172638..4c9bb4e86 100644 --- a/runtime/monitor/kafka_test.go +++ b/runtime/monitor/kafka_test.go @@ -2,15 +2,16 @@ package monitor import ( "context" - "github.com/stretchr/testify/require" "mokapi/runtime/metrics" "testing" + + "github.com/stretchr/testify/require" ) func TestKafka_Metrics_Messages(t *testing.T) { k := NewKafka() k.Messages.WithLabel("service_a", "topic_a").Add(1) - require.Equal(t, float64(1), k.Messages.Sum()) + require.Equal(t, float64(1), k.Messages.Sum(metrics.NewQuery())) } func TestKafka_LastMessage(t *testing.T) { diff --git a/runtime/monitor/ldap_test.go b/runtime/monitor/ldap_test.go index e8300f462..1baa0f3d5 100644 --- a/runtime/monitor/ldap_test.go +++ b/runtime/monitor/ldap_test.go @@ -18,7 +18,7 @@ func TestLdapLabels(t *testing.T) { func TestLdap_Metrics_Bind(t *testing.T) { l := NewLdap() l.RequestCounter.WithLabel("service_a", "bind").Add(1) - require.Equal(t, float64(1), l.RequestCounter.Sum()) + require.Equal(t, float64(1), l.RequestCounter.Sum(metrics.NewQuery())) } func TestLdap_Search(t *testing.T) { diff --git a/runtime/monitor/mail_test.go b/runtime/monitor/mail_test.go index 94aa2f900..820c72dae 100644 --- a/runtime/monitor/mail_test.go +++ b/runtime/monitor/mail_test.go @@ -2,14 +2,16 @@ package monitor import ( "context" - "github.com/stretchr/testify/require" + "mokapi/runtime/metrics" "testing" + + "github.com/stretchr/testify/require" ) func TestSmtp_Metrics_Mails(t *testing.T) { s := NewMail() s.Mails.WithLabel("service_a", "sender_a").Add(1) - require.Equal(t, float64(1), s.Mails.Sum()) + require.Equal(t, float64(1), s.Mails.Sum(metrics.NewQuery())) } func TestSmtpContext(t *testing.T) { diff --git a/runtime/monitor/mqtt.go b/runtime/monitor/mqtt.go index 091d9c062..da7c38cdc 100644 --- a/runtime/monitor/mqtt.go +++ b/runtime/monitor/mqtt.go @@ -10,7 +10,6 @@ var mqttKey = contextKey("mqtt") type Mqtt struct { Messages *metrics.CounterMap LastMessage *metrics.GaugeMap - Lags *metrics.GaugeMap } func NewMqtt() *Mqtt { @@ -20,25 +19,20 @@ func NewMqtt() *Mqtt { lastMessage := metrics.NewGaugeMap( metrics.WithFQName("mqtt", "message_timestamp"), metrics.WithLabelNames("service", "topic")) - lag := metrics.NewGaugeMap( - metrics.WithFQName("mqtt", "consumer_group_lag"), - metrics.WithLabelNames("service", "group", "topic", "partition")) return &Mqtt{ Messages: messages, LastMessage: lastMessage, - Lags: lag, } } func (k *Mqtt) Metrics() []metrics.Metric { - return []metrics.Metric{k.Messages, k.LastMessage, k.Lags} + return []metrics.Metric{k.Messages, k.LastMessage} } func (k *Mqtt) Reset() { k.Messages.Reset() k.LastMessage.Reset() - k.Lags.Reset() } func NewMqttContext(ctx context.Context, mqtt *Mqtt) context.Context { diff --git a/runtime/monitor/mqtt_test.go b/runtime/monitor/mqtt_test.go index 9b8360515..68da29808 100644 --- a/runtime/monitor/mqtt_test.go +++ b/runtime/monitor/mqtt_test.go @@ -2,15 +2,16 @@ package monitor import ( "context" - "github.com/stretchr/testify/require" "mokapi/runtime/metrics" "testing" + + "github.com/stretchr/testify/require" ) func TestMqtt_Metrics_Messages(t *testing.T) { m := NewMqtt() m.Messages.WithLabel("service_a", "topic_a").Add(1) - require.Equal(t, float64(1), m.Messages.Sum()) + require.Equal(t, float64(1), m.Messages.Sum(metrics.NewQuery())) } func TestMqtt_LastMessage(t *testing.T) { @@ -19,12 +20,6 @@ func TestMqtt_LastMessage(t *testing.T) { require.Equal(t, float64(10), m.LastMessage.Value(metrics.NewQuery(metrics.ByLabel("service", "service_a")))) } -func TestMqtt_Metrics_Lags(t *testing.T) { - m := NewMqtt() - m.Lags.WithLabel("service_a", "group_a", "topic_a", "10").Set(10) - require.Equal(t, float64(10), m.Lags.Value(metrics.NewQuery(metrics.ByLabel("service", "service_a")))) -} - func TestMqttContext(t *testing.T) { ctx := context.Background() h := New() diff --git a/runtime/runtime.go b/runtime/runtime.go index 29f9c70c4..f728c0fb5 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -10,12 +10,14 @@ import ( "mokapi/safe" "mokapi/version" "sync" + + log "github.com/sirupsen/logrus" ) type App struct { Version string BuildTime string - http *HttpStore + Http *HttpStore Ldap *LdapStore Kafka *KafkaStore Mqtt *MqttStore @@ -30,6 +32,7 @@ type App struct { searchIndex *SearchIndex reader dynamic.Reader + hook *LogHook Configs map[string]*dynamic.Config } @@ -39,12 +42,15 @@ func New(cfg *static.Config, reader dynamic.Reader) *App { index := newSearchIndex(cfg.Api.Search) em := events.NewStoreManager(index) + configureLogging(cfg) + em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits()) em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("http")) em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("kafka")) em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("ldap")) em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("mail")) em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("job")) + em.SetStore(int(cfg.Event.Store["default"].Size), events.NewTraits().WithNamespace("logs")) app := &App{ Version: version.BuildVersion, @@ -52,9 +58,9 @@ func New(cfg *static.Config, reader dynamic.Reader) *App { Monitor: m, Events: em, Configs: map[string]*dynamic.Config{}, - http: &HttpStore{cfg: cfg, index: index, events: em, reader: reader}, + Http: &HttpStore{cfg: cfg, index: index, events: em, reader: reader}, Kafka: &KafkaStore{monitor: m, cfg: cfg, index: index, events: em, reader: reader}, - Mqtt: &MqttStore{monitor: m, cfg: cfg, sm: em}, + Mqtt: &MqttStore{monitor: m, cfg: cfg, index: index, sm: em, events: em, reader: reader}, Ldap: &LdapStore{cfg: cfg, events: em, index: index}, Mail: &MailStore{cfg: cfg, sm: em, index: index}, cfg: cfg, @@ -69,9 +75,19 @@ func (a *App) Start(p *safe.Pool) { go a.searchIndex.start(p) } +func (a *App) Stop() { + if a.hook != nil { + a.hook.Disable() + } +} + +func (a *App) EnableLogHook() { + a.hook = NewLogHook(a.Events) + log.AddHook(a.hook) +} + func (a *App) UpdateConfig(e dynamic.ConfigEvent) { a.m.Lock() - defer a.m.Unlock() if e.Event == dynamic.Delete { delete(a.Configs, e.Config.Info.Key()) @@ -81,6 +97,7 @@ func (a *App) UpdateConfig(e dynamic.ConfigEvent) { a.Configs[r.Info.Key()] = r } } + a.m.Unlock() if a.cfg.Api.Search.Enabled { a.removeConfigFromIndex(e.Config) @@ -105,22 +122,6 @@ func (a *App) FindConfig(key string) *dynamic.Config { return nil } -func (a *App) AddHttp(c *dynamic.Config) *HttpInfo { - return a.http.Add(c) -} - -func (a *App) GetHttp(name string) *HttpInfo { - return a.http.Get(name) -} - -func (a *App) RemoveHttp(c *dynamic.Config) { - a.http.Remove(c) -} - -func (a *App) ListHttp() []*HttpInfo { - return a.http.List() -} - func (a *App) Search(r search.Request) (search.Result, error) { return a.searchIndex.Search(r) } diff --git a/runtime/runtime_http.go b/runtime/runtime_http.go index 33c5908dc..8c310ccd7 100644 --- a/runtime/runtime_http.go +++ b/runtime/runtime_http.go @@ -29,6 +29,7 @@ type HttpInfo struct { *openapi.Config configs map[string]*dynamic.Config seenPaths map[string]bool + m sync.Mutex } type httpHandler struct { @@ -60,7 +61,6 @@ func (s *HttpStore) List() []*HttpInfo { func (s *HttpStore) Add(c *dynamic.Config) *HttpInfo { s.m.Lock() - defer s.m.Unlock() if len(s.infos) == 0 { s.infos = make(map[string]*HttpInfo) @@ -68,19 +68,26 @@ func (s *HttpStore) Add(c *dynamic.Config) *HttpInfo { cfg := c.Data.(*openapi.Config) name := cfg.Info.Name hc, ok := s.infos[name] - - store, hasStoreConfig := s.cfg.Event.Store[name] - if !hasStoreConfig { - store = s.cfg.Event.Store["default"] - } - if !ok { hc = &HttpInfo{ + Config: cfg, configs: map[string]*dynamic.Config{}, seenPaths: map[string]bool{}, } s.infos[cfg.Info.Name] = hc + } + + s.m.Unlock() + + hc.m.Lock() + defer hc.m.Unlock() + + store, hasStoreConfig := s.cfg.Event.Store[name] + if !hasStoreConfig { + store = s.cfg.Event.Store["default"] + } + if !ok { s.events.ResetStores(events.NewTraits().WithNamespace("http").WithName(name)) s.events.SetStore(int(store.Size), events.NewTraits().WithNamespace("http").WithName(name)) } @@ -204,3 +211,13 @@ func IsHttpConfig(c *dynamic.Config) (*openapi.Config, bool) { func getHttpConfig(c *dynamic.Config) *openapi.Config { return c.Data.(*openapi.Config) } + +func (s *HttpStore) Len() int { + if s == nil { + return 0 + } + + s.m.RLock() + defer s.m.RUnlock() + return len(s.infos) +} diff --git a/runtime/runtime_http_search.go b/runtime/runtime_http_search.go index e6a551da1..f035e674c 100644 --- a/runtime/runtime_http_search.go +++ b/runtime/runtime_http_search.go @@ -2,11 +2,17 @@ package runtime import ( "fmt" + "maps" "mokapi/providers/openapi" openApiSchema "mokapi/providers/openapi/schema" "mokapi/runtime/search" "mokapi/schema/json/schema" + "slices" + "strconv" "strings" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" ) type httpSearchIndexData struct { @@ -28,6 +34,7 @@ type httpPathSearchIndexData struct { Summary string `json:"summary"` Description string `json:"description"` Parameters []httpParameterSearchIndexData `json:"parameters"` + Meta map[string]string `json:"meta"` } type httpParameterSearchIndexData struct { @@ -38,22 +45,31 @@ type httpParameterSearchIndexData struct { } type httpOperationSearchIndexData struct { - Type string `json:"type"` - Discriminator string `json:"discriminator"` - Api string `json:"api"` - Path string `json:"path"` - Method string `json:"method"` - Summary string `json:"summary"` - Description string `json:"description"` - OperationId string `json:"operationId"` - Tags []string `json:"tags"` - Parameters []httpParameterSearchIndexData `json:"parameters"` - RequestBody string `json:"request_body"` - Responses []httpResponseSearchIndexData `json:"responses"` + Type string `json:"type"` + Discriminator string `json:"discriminator"` + Api string `json:"api"` + Path string `json:"path"` + Method string `json:"method"` + Summary string `json:"summary"` + Description string `json:"description"` + OperationId string `json:"operationId"` + Tags []string `json:"tags"` + Parameters []httpParameterSearchIndexData `json:"parameters"` + StatusCode int `json:"statusCode"` + StatusCodeText string `json:"statusCodeText"` + RequestBodies []httpRequestBodySearchIndexData `json:"requestBodies"` + Responses []httpResponseSearchIndexData `json:"responses"` +} + +type httpRequestBodySearchIndexData struct { + Description string `json:"description"` + ContentType string `json:"contentType"` + Schema *schema.IndexData `json:"schema"` } type httpResponseSearchIndexData struct { Description string `json:"description"` + ContentType string `json:"contentType"` Schema *schema.IndexData `json:"schema"` } @@ -86,6 +102,7 @@ func (s *HttpStore) addToIndex(cfg *openapi.Config) { Path: path, Summary: p.Summary, Description: p.Description, + Meta: map[string]string{}, } if pathData.Summary == "" { pathData.Summary = p.Value.Summary @@ -103,28 +120,19 @@ func (s *HttpStore) addToIndex(cfg *openapi.Config) { Schema: schema.NewIndexData(ps), }) } + methods := slices.Collect(maps.Keys(p.Value.Operations())) + pathData.Meta["methods"] = strings.Join(methods, ",") s.index.Add(fmt.Sprintf("http_%s_%s", cfg.Info.Name, path), pathData) for method, op := range p.Value.Operations() { id := fmt.Sprintf("http_%s_%s_%s", cfg.Info.Name, path, method) - opData := httpOperationSearchIndexData{ - Type: "http", - Discriminator: "http_operation", - Api: cfg.Info.Name, - Path: path, - Method: method, - Summary: op.Summary, - Description: op.Description, - OperationId: op.OperationId, - Tags: op.Tags, - Parameters: pathData.Parameters, - } + params := pathData.Parameters for _, param := range op.Parameters { ps := openApiSchema.ConvertToJsonSchema(param.Value.Schema) - opData.Parameters = append(opData.Parameters, httpParameterSearchIndexData{ + params = append(params, httpParameterSearchIndexData{ Name: param.Value.Name, Description: param.Value.Description, Location: param.Value.Type.String(), @@ -132,25 +140,75 @@ func (s *HttpStore) addToIndex(cfg *openapi.Config) { }) } - if op.Responses != nil { + var requestBodies []httpRequestBodySearchIndexData + if op.RequestBody != nil && op.RequestBody.Value != nil { + v := op.RequestBody.Value + for ct, mt := range v.Content { + rs := openApiSchema.ConvertToJsonSchema(mt.Schema) + requestBodies = append(requestBodies, httpRequestBodySearchIndexData{ + Description: v.Description, + ContentType: ct, + Schema: schema.NewIndexData(rs), + }) + } + } + + if op.Responses != nil && op.Responses.Len() > 0 { for it := op.Responses.Iter(); it.Next(); { v := it.Value().Value if v == nil { continue } - for _, mt := range v.Content { - rs := openApiSchema.ConvertToJsonSchema(mt.Schema) + statusCode := 0 + if i, err := strconv.Atoi(it.Key()); err == nil { + statusCode = i + } - opData.Responses = append(opData.Responses, httpResponseSearchIndexData{ + var responses []httpResponseSearchIndexData + for ct, mt := range v.Content { + rs := openApiSchema.ConvertToJsonSchema(mt.Schema) + responses = append(responses, httpResponseSearchIndexData{ Description: v.Description, + ContentType: ct, Schema: schema.NewIndexData(rs), }) } + opData := httpOperationSearchIndexData{ + Type: "http", + Discriminator: "http_operation", + Api: cfg.Info.Name, + Path: path, + Method: method, + Summary: op.Summary, + Description: op.Description, + OperationId: op.OperationId, + Tags: op.Tags, + Parameters: params, + StatusCode: statusCode, + StatusCodeText: it.Key(), + RequestBodies: requestBodies, + Responses: responses, + } + + s.index.Add(id, opData) } + } else { + opData := httpOperationSearchIndexData{ + Type: "http", + Discriminator: "http_operation", + Api: cfg.Info.Name, + Path: path, + Method: method, + Summary: op.Summary, + Description: op.Description, + OperationId: op.OperationId, + Tags: op.Tags, + Parameters: params, + RequestBodies: requestBodies, + } + s.index.Add(id, opData) } - - s.index.Add(id, opData) } } } @@ -180,12 +238,15 @@ func getHttpSearchResult(fields map[string]string, discriminator []string) (sear } case "operation": result.Domain = fields["api"] - result.Title = fmt.Sprintf("%s %s", fields["method"], fields["path"]) + result.Title = fields["path"] result.Params = map[string]string{ "type": strings.ToLower(result.Type), "service": result.Domain, "path": fields["path"], - "method": strings.ToLower(fields["method"]), + "method": strings.ToUpper(fields["method"]), + } + if s, ok := fields["statusCode"]; ok && s != "0" { + result.Params["statusCode"] = s } default: return result, fmt.Errorf("unsupported search result: %s", strings.Join(discriminator, "_")) @@ -203,3 +264,14 @@ func (s *HttpStore) removeFromIndex(cfg *openapi.Config) { } } } + +func AddMappings(m *mapping.DocumentMapping) { + // Statt tief zu verschachteln, mappe den Pfad direkt: + statusFieldMapping := bleve.NewNumericFieldMapping() + statusFieldMapping.Name = "statusCode" + statusFieldMapping.Store = true + + // Direkt auf das oberste Document Mapping anwenden + // Bleve sucht im JSON trotzdem nach dem Pfad, indiziert ihn aber flach + m.AddFieldMappingsAt("event.data.response.statusCode", statusFieldMapping) +} diff --git a/runtime/runtime_http_search_test.go b/runtime/runtime_http_search_test.go index 71cfc150b..4a945aec7 100644 --- a/runtime/runtime_http_search_test.go +++ b/runtime/runtime_http_search_test.go @@ -5,11 +5,14 @@ import ( "mokapi/config/dynamic" "mokapi/config/dynamic/dynamictest" "mokapi/config/static" + "mokapi/engine/enginetest" "mokapi/providers/openapi" "mokapi/providers/openapi/openapitest" "mokapi/runtime" "mokapi/runtime/search" "mokapi/safe" + "net/http" + "net/http/httptest" "testing" "time" @@ -33,7 +36,7 @@ func TestIndex_Http(t *testing.T) { name: "Search by name", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -60,11 +63,16 @@ func TestIndex_Http(t *testing.T) { name: "config should be remove from index", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) + waitSearchIndex(t, func() bool { + r, err := app.Search(search.Request{QueryText: "foo", Limit: 10}) + require.NoError(t, err) + return len(r.Results) == 1 + }) r, err := app.Search(search.Request{QueryText: "foo", Limit: 10}) require.NoError(t, err) require.Len(t, r.Results, 1) - app.RemoveHttp(toConfig(cfg)) + app.Http.Remove(toConfig(cfg)) waitSearchIndex(t, func() bool { r, err = app.Search(search.Request{QueryText: "foo", Limit: 10}) require.NoError(t, err) @@ -76,7 +84,7 @@ func TestIndex_Http(t *testing.T) { name: "Search by substring", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("My petstore API", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -103,7 +111,7 @@ func TestIndex_Http(t *testing.T) { name: "mailpiece should not match mailbox", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("mailbox", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -119,7 +127,7 @@ func TestIndex_Http(t *testing.T) { name: "Search by version", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "1.0", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -149,7 +157,7 @@ func TestIndex_Http(t *testing.T) { openapitest.WithInfo("foo", "1.0", ""), openapitest.WithPath("/pets"), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -181,7 +189,7 @@ func TestIndex_Http(t *testing.T) { openapitest.WithInfo("foo", "1.0", "a description"), openapitest.WithPath("/pets", openapitest.WithPathInfo("", "a description")), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -224,12 +232,12 @@ func TestIndex_Http(t *testing.T) { openapitest.WithInfo("foo", "1.0", "a description"), openapitest.WithPath("/pets", openapitest.WithPathInfo("", "a description"), - openapitest.WithOperation("get", + openapitest.WithOperation(http.MethodGet, openapitest.WithHeaderParam("foo", true, openapitest.WithParamInfo("parameter description")), ), ), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -243,13 +251,13 @@ func TestIndex_Http(t *testing.T) { search.ResultItem{ Type: "HTTP", Domain: "foo", - Title: "GET /pets", + Title: "/pets", Fragments: []string{"parameter description"}, Params: map[string]string{ "type": "http", "service": "foo", "path": "/pets", - "method": "get", + "method": "GET", }, }, r.Results[0]) @@ -266,7 +274,7 @@ func TestIndex_Http(t *testing.T) { ), ), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) // search response should only have one the root OpenAPI object var r search.Result var err error @@ -298,7 +306,7 @@ func TestIndex_Http(t *testing.T) { name: "Search by api with space", test: func(t *testing.T, app *runtime.App) { cfg := openapitest.NewConfig("3.0", openapitest.WithInfo("foo bar", "", "")) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) var r search.Result var err error @@ -332,7 +340,7 @@ func TestIndex_Http(t *testing.T) { ), ), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) // search response should only have one the root OpenAPI object var r search.Result var err error @@ -371,7 +379,7 @@ func TestIndex_Http(t *testing.T) { ), ), ) - app.AddHttp(toConfig(cfg)) + app.Http.Add(toConfig(cfg)) // search response should only have one the root OpenAPI object var r search.Result var err error @@ -409,6 +417,79 @@ func TestIndex_Http(t *testing.T) { } } +func TestIndex_Http_Event(t *testing.T) { + api := openapitest.NewConfig("3.0", + openapitest.WithInfo("Test HTTP Events", "", ""), + openapitest.WithPath("/foo", + openapitest.WithOperation(http.MethodGet, + openapitest.WithResponse(http.StatusOK), + ), + ), + ) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: api, + } + + testcases := []struct { + name string + test func(t *testing.T, h openapi.Handler, app *runtime.App) + }{ + { + name: "search event by method", + test: func(t *testing.T, h openapi.Handler, app *runtime.App) { + req := httptest.NewRequest("GET", "http://localhost/foo", nil) + w := httptest.NewRecorder() + he := h.ServeHTTP(w, req) + require.Nil(t, he) + + r, err := waitSearchResult(t, func() (search.Result, error) { + return app.Search(search.Request{QueryText: "+method:GET +type:event", Limit: 10}) + }, 1) + + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "Event", r.Results[0].Type) + require.Equal(t, "Test HTTP Events", r.Results[0].Domain) + require.Equal(t, "http://localhost/foo", r.Results[0].Title) + require.Len(t, r.Results[0].Fragments, 2) + require.Contains(t, r.Results[0].Fragments, "GET") + require.Contains(t, r.Results[0].Fragments, "event") + require.Len(t, r.Results[0].Params, 6) + require.Equal(t, "event", r.Results[0].Params["type"]) + require.Equal(t, "http", r.Results[0].Params["traits.namespace"]) + require.Equal(t, "Test HTTP Events", r.Results[0].Params["traits.name"]) + require.Equal(t, "/foo", r.Results[0].Params["traits.path"]) + require.Equal(t, "GET", r.Results[0].Params["traits.method"]) + require.Contains(t, r.Results[0].Params, "id") + require.NotEmpty(t, r.Results[0].Time) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + + app.Http.Add(cfg) + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + + h := openapi.NewHandler(api, enginetest.NewEngine(), app.Events) + tc.test(t, h, app) + }) + } +} + func waitSearchIndex(t *testing.T, check func() bool) { deadline := time.Now().Add(2 * time.Second) @@ -422,3 +503,21 @@ func waitSearchIndex(t *testing.T, check func() bool) { time.Sleep(20 * time.Millisecond) } } + +func waitSearchResult(t *testing.T, f func() (search.Result, error), expectedResults int) (search.Result, error) { + deadline := time.Now().Add(2 * time.Second) + + for { + r, err := f() + if err != nil { + return r, err + } + if len(r.Results) == expectedResults { + return r, nil + } + if time.Now().After(deadline) { + t.Fatalf("wait search result reached deadline: last search returned %d results, %d was expected", len(r.Results), expectedResults) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/runtime/runtime_http_test.go b/runtime/runtime_http_test.go index 9dc729b09..776605318 100644 --- a/runtime/runtime_http_test.go +++ b/runtime/runtime_http_test.go @@ -11,6 +11,7 @@ import ( "mokapi/runtime" "mokapi/runtime/events" "mokapi/runtime/events/eventstest" + "mokapi/runtime/metrics" "mokapi/runtime/monitor" "net/http" "net/http/httptest" @@ -28,9 +29,9 @@ func TestApp_AddHttp(t *testing.T) { { name: "event store available", test: func(t *testing.T, app *runtime.App) { - app.AddHttp(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")))) + app.Http.Add(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")))) - require.NotNil(t, app.GetHttp("foo")) + require.NotNil(t, app.Http.Get("foo")) err := app.Events.Push(&eventstest.Event{Name: "bar"}, events.NewTraits().WithNamespace("http").WithName("foo")) require.NoError(t, err, "event store should be available") }, @@ -38,10 +39,10 @@ func TestApp_AddHttp(t *testing.T) { { name: "event store for endpoint available", test: func(t *testing.T, app *runtime.App) { - app.AddHttp(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), + app.Http.Add(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), openapitest.WithPath("bar")))) - require.NotNil(t, app.GetHttp("foo")) + require.NotNil(t, app.Http.Get("foo")) err := app.Events.Push(&eventstest.Event{Name: "bar"}, events.NewTraits().WithNamespace("http").WithName("foo").With("path", "bar")) require.NoError(t, err, "event store should be available") }, @@ -49,7 +50,7 @@ func TestApp_AddHttp(t *testing.T) { { name: "request is counted in monitor", test: func(t *testing.T, app *runtime.App) { - info := app.AddHttp(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), + info := app.Http.Add(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), openapitest.WithPath("/foo", openapitest.WithOperation(http.MethodGet, openapitest.WithResponse(http.StatusOK), @@ -63,13 +64,13 @@ func TestApp_AddHttp(t *testing.T) { err := h.ServeHTTP(rr, r) require.Nil(t, err) - require.Equal(t, float64(1), m.RequestCounter.Sum()) + require.Equal(t, float64(1), m.RequestCounter.Sum(metrics.NewQuery())) }, }, { name: "retrieve configs", test: func(t *testing.T, app *runtime.App) { - info := app.AddHttp(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), + info := app.Http.Add(newConfig(openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", ""), openapitest.WithPath("bar")))) configs := info.Configs() @@ -109,7 +110,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { newConfig("https://mokapi.io/b", openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "bar"))), }, test: func(t *testing.T, app *runtime.App) { - info := app.GetHttp("foo") + info := app.Http.Get("foo") require.Equal(t, "bar", info.Info.Description) configs := info.Configs() require.Len(t, configs, 2) @@ -122,7 +123,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { newConfig("https://mokapi.io/a", openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "bar"))), }, test: func(t *testing.T, app *runtime.App) { - info := app.GetHttp("foo") + info := app.Http.Get("foo") require.Equal(t, "foo", info.Info.Description) }, }, @@ -133,7 +134,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { newConfig("https://mokapi.io/a", openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "bar"))), }, test: func(t *testing.T, app *runtime.App) { - info := app.GetHttp("foo") + info := app.Http.Get("foo") require.Equal(t, "foo", info.Info.Description) }, }, @@ -146,7 +147,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { err := app.Events.Push(&eventstest.Event{Name: "bar"}, events.NewTraits().WithNamespace("http").WithName("foo")) require.NoError(t, err) - app.AddHttp( + app.Http.Add( newConfig("https://mokapi.io/a", openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "bar"))), ) @@ -198,7 +199,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { ), }, test: func(t *testing.T, app *runtime.App) { - info := app.GetHttp("foo") + info := app.Http.Get("foo") p := info.Paths["/foo"] require.NotNil(t, p) op := p.Value.Get @@ -220,7 +221,7 @@ func TestApp_AddHttp_Patching(t *testing.T) { } app := runtime.New(cfg, &dynamictest.Reader{}) for _, c := range tc.configs { - app.AddHttp(c) + app.Http.Add(c) } tc.test(t, app) }) diff --git a/runtime/runtime_kafka.go b/runtime/runtime_kafka.go index e3c8738a1..6d20e1245 100644 --- a/runtime/runtime_kafka.go +++ b/runtime/runtime_kafka.go @@ -1,6 +1,7 @@ package runtime import ( + "errors" "mokapi/config/dynamic" "mokapi/config/dynamic/asyncApi" "mokapi/config/static" @@ -14,6 +15,7 @@ import ( "mokapi/sortedmap" "path/filepath" "sort" + "strings" "sync" log "github.com/sirupsen/logrus" @@ -35,6 +37,7 @@ type KafkaInfo struct { configs map[string]*dynamic.Config seenTopics map[string]bool updateEventAndMetrics func(k *KafkaInfo) + m sync.Mutex } type KafkaHandler struct { @@ -75,15 +78,30 @@ func (s *KafkaStore) List() []*KafkaInfo { } func (s *KafkaStore) Add(c *dynamic.Config, emitter common.EventEmitter) (*KafkaInfo, error) { + cfg := getKafkaConfig(c) + if cfg == nil { + return nil, errors.New("no Kafka config found") + } + s.m.Lock() - defer s.m.Unlock() if len(s.infos) == 0 { s.infos = make(map[string]*KafkaInfo) } - cfg := getKafkaConfig(c) name := cfg.Info.Name ki, ok := s.infos[name] + if !ok { + log.Debugf("starting Kafka Store with topics: %v", len(cfg.Channels)) + ki = newKafkaInfo(store.NewEmpty(emitter, s.events, s.monitor.Kafka), s.updateEventStore) + log.Debugf("end Kafka Store with topics: %v", len(cfg.Channels)) + ki.Config = cfg + s.infos[cfg.Info.Name] = ki + } + + defer s.m.Unlock() + + ki.m.Lock() + defer ki.m.Unlock() eventStore, hasStoreConfig := s.cfg.Event.Store[name] if !hasStoreConfig { @@ -93,9 +111,6 @@ func (s *KafkaStore) Add(c *dynamic.Config, emitter common.EventEmitter) (*Kafka if !ok { s.events.ResetStores(events.NewTraits().WithNamespace("kafka").WithName(cfg.Info.Name)) s.events.SetStore(int(eventStore.Size), events.NewTraits().WithNamespace("kafka").WithName(cfg.Info.Name)) - - ki = newKafkaInfo(store.NewEmpty(emitter, s.events, s.monitor.Kafka), s.updateEventStore) - s.infos[cfg.Info.Name] = ki } ki.addConfig(c, s.reader) @@ -269,3 +284,30 @@ func (s *KafkaStore) updateEventStore(k *KafkaInfo) { k.seenTopics[topicName] = true } } + +func HasKafkaBroker(c *dynamic.Config) (*asyncapi3.Config, bool) { + cfg, ok := IsAsyncApiConfig(c) + if !ok { + return nil, false + } + for it := cfg.Servers.Iter(); it.Next(); { + s := it.Value() + if s.Value == nil { + continue + } + if strings.ToLower(s.Value.Protocol) == "kafka" { + return cfg, true + } + } + return cfg, false +} + +func (s *KafkaStore) Len() int { + if s == nil { + return 0 + } + + s.m.RLock() + defer s.m.RUnlock() + return len(s.infos) +} diff --git a/runtime/runtime_kafka_search.go b/runtime/runtime_kafka_search.go index 03a960b64..543779aaa 100644 --- a/runtime/runtime_kafka_search.go +++ b/runtime/runtime_kafka_search.go @@ -19,6 +19,7 @@ type kafkaSearchIndexData struct { Description string `json:"description"` Contact *asyncapi3.Contact `json:"contact"` Servers []kafkaServerSearchData `json:"servers"` + Meta map[string]string `json:"meta"` } type kafkaServerSearchData struct { @@ -65,6 +66,9 @@ func (s *KafkaStore) addToIndex(cfg *asyncapi3.Config) { Version: cfg.Info.Version, Description: cfg.Info.Description, Contact: cfg.Info.Contact, + Meta: map[string]string{ + "topics": fmt.Sprintf("%d", len(cfg.Channels)), + }, } for it := cfg.Servers.Iter(); it.Next(); { name := it.Key() @@ -87,13 +91,20 @@ func (s *KafkaStore) addToIndex(cfg *asyncapi3.Config) { if topic == nil || topic.Value == nil { continue } + topicName := name + if topic.Value.Name != "" { + topicName = topic.Value.Name + } + if topic.Value.Address != "" { + topicName = topic.Value.Address + } t := kafkaTopicSearchIndexData{ Type: "kafka", Discriminator: "kafka_topic", Api: cfg.Info.Name, ChannelId: name, - Name: topic.Value.Name, + Name: topicName, Title: topic.Value.Title, Address: topic.Value.Address, Summary: topic.Value.Summary, @@ -139,6 +150,12 @@ func getKafkaSearchResult(fields map[string]string, discriminator []string) (sea "type": strings.ToLower(result.Type), "service": result.Title, } + for k, v := range fields { + if strings.HasPrefix(k, "meta.") { + k = strings.Replace(k, "meta.", "", 1) + result.Params[k] = v + } + } return result, nil } diff --git a/runtime/runtime_kafka_search_test.go b/runtime/runtime_kafka_search_test.go index 4b24bd866..b98ea7c56 100644 --- a/runtime/runtime_kafka_search_test.go +++ b/runtime/runtime_kafka_search_test.go @@ -6,8 +6,10 @@ import ( "mokapi/config/dynamic/dynamictest" "mokapi/config/static" "mokapi/engine/enginetest" + "mokapi/kafka" "mokapi/providers/asyncapi3" "mokapi/providers/asyncapi3/asyncapi3test" + "mokapi/providers/asyncapi3/kafka/store" "mokapi/runtime" "mokapi/runtime/search" "mokapi/safe" @@ -51,6 +53,7 @@ func TestIndex_Kafka(t *testing.T) { Params: map[string]string{ "type": "kafka", "service": "Kafka Test server", + "topics": "0", }, }, r.Results[0]) @@ -86,15 +89,27 @@ func TestIndex_Kafka(t *testing.T) { cfg := asyncapi3test.NewConfig( asyncapi3test.WithInfo("Kafka Test server", "", ""), asyncapi3test.WithChannel("foo", - asyncapi3test.WithChannelDescription("description"), + asyncapi3test.WithChannelDescription("first"), ), ) + + second := asyncapi3test.NewChannel( + asyncapi3test.WithChannelDescription("second"), + asyncapi3test.WithChannelAddress("address-name"), + ) + cfg.Channels["bar"] = &asyncapi3.ChannelRef{Value: second} + + third := asyncapi3test.NewChannel( + asyncapi3test.WithChannelDescription("third"), + ) + cfg.Channels["yuh"] = &asyncapi3.ChannelRef{Value: third} + _, err := app.Kafka.Add(toConfig(cfg), enginetest.NewEngine()) require.NoError(t, err) var r search.Result waitSearchIndex(t, func() bool { - r, err = app.Search(search.Request{QueryText: "description", Limit: 10}) + r, err = app.Search(search.Request{QueryText: "first", Limit: 10}) require.NoError(t, err) return len(r.Results) == 1 }) @@ -104,7 +119,7 @@ func TestIndex_Kafka(t *testing.T) { Type: "Kafka", Domain: "Kafka Test server", Title: "Topic foo", - Fragments: []string{"description"}, + Fragments: []string{"first"}, Params: map[string]string{ "type": "kafka", "service": "Kafka Test server", @@ -112,14 +127,50 @@ func TestIndex_Kafka(t *testing.T) { }, }, r.Results[0]) + + r, err = app.Search(search.Request{QueryText: "second", Limit: 10}) + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, + search.ResultItem{ + Type: "Kafka", + Domain: "Kafka Test server", + Title: "Topic address-name", + Fragments: []string{"second"}, + Params: map[string]string{ + "type": "kafka", + "service": "Kafka Test server", + "topic": "address-name", + }, + }, + r.Results[0]) + + r, err = app.Search(search.Request{QueryText: "third", Limit: 10}) + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, + search.ResultItem{ + Type: "Kafka", + Domain: "Kafka Test server", + Title: "Topic yuh", + Fragments: []string{"third"}, + Params: map[string]string{ + "type": "kafka", + "service": "Kafka Test server", + "topic": "yuh", + }, + }, + r.Results[0]) }, }, } + t.Parallel() for _, tc := range testcases { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() + app := runtime.New( &static.Config{ Api: static.Api{ @@ -138,3 +189,119 @@ func TestIndex_Kafka(t *testing.T) { }) } } + +func TestIndex_Kafka_Event(t *testing.T) { + api := asyncapi3test.NewConfig( + asyncapi3test.WithInfo("Kafka Test Events", "", ""), + asyncapi3test.WithChannel("events"), + ) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: api, + } + + testcases := []struct { + name string + test func(t *testing.T, s *store.Store, app *runtime.App) + }{ + { + name: "search event by key", + test: func(t *testing.T, s *store.Store, app *runtime.App) { + + wr, err := s.Topic("events").Partitions[0].Write(kafka.RecordBatch{Records: []*kafka.Record{ + { + Key: kafka.NewBytes([]byte("foo")), + }, + }}) + require.NoError(t, err) + require.Len(t, wr.Records, 0) + + r, err := waitSearchResult(t, func() (search.Result, error) { + return app.Search(search.Request{QueryText: "+key:foo +type:event", Limit: 10}) + }, 1) + + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "Event", r.Results[0].Type) + require.Equal(t, "Kafka Test Events", r.Results[0].Domain) + require.Equal(t, "foo", r.Results[0].Title) + require.Len(t, r.Results[0].Fragments, 2) + require.Contains(t, r.Results[0].Fragments, "foo") + require.Contains(t, r.Results[0].Fragments, "event") + require.Len(t, r.Results[0].Params, 7) + require.Equal(t, "event", r.Results[0].Params["type"]) + require.Equal(t, "kafka", r.Results[0].Params["traits.namespace"]) + require.Equal(t, "Kafka Test Events", r.Results[0].Params["traits.name"]) + require.Equal(t, "message", r.Results[0].Params["traits.type"]) + require.Equal(t, "events", r.Results[0].Params["traits.topic"]) + require.Equal(t, "0", r.Results[0].Params["traits.partition"]) + require.Contains(t, r.Results[0].Params, "id") + require.NotEmpty(t, r.Results[0].Time) + }, + }, + { + name: "search event by header", + test: func(t *testing.T, s *store.Store, app *runtime.App) { + + wr, err := s.Topic("events").Partitions[0].Write(kafka.RecordBatch{Records: []*kafka.Record{ + { + Key: kafka.NewBytes([]byte("foo")), + Headers: []kafka.RecordHeader{ + { + Key: "header-key", + Value: []byte("bar"), + }, + }, + }, + }}) + require.NoError(t, err) + require.Len(t, wr.Records, 0) + + r, err := waitSearchResult(t, func() (search.Result, error) { + return app.Search(search.Request{QueryText: `+"header-key" +type:event`, Limit: 10}) + }, 1) + + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "Event", r.Results[0].Type) + require.Equal(t, "Kafka Test Events", r.Results[0].Domain) + require.Equal(t, "foo", r.Results[0].Title) + require.Len(t, r.Results[0].Fragments, 2) + require.Contains(t, r.Results[0].Fragments, "header-key") + require.Contains(t, r.Results[0].Fragments, "event") + require.Len(t, r.Results[0].Params, 7) + require.Equal(t, "event", r.Results[0].Params["type"]) + require.Equal(t, "kafka", r.Results[0].Params["traits.namespace"]) + require.Equal(t, "Kafka Test Events", r.Results[0].Params["traits.name"]) + require.Equal(t, "message", r.Results[0].Params["traits.type"]) + require.Equal(t, "events", r.Results[0].Params["traits.topic"]) + require.Equal(t, "0", r.Results[0].Params["traits.partition"]) + require.Contains(t, r.Results[0].Params, "id") + require.NotEmpty(t, r.Results[0].Time) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + + info, err := app.Kafka.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + + tc.test(t, info.Store, app) + }) + } +} diff --git a/runtime/runtime_kafka_test.go b/runtime/runtime_kafka_test.go index d253e153d..d63698387 100644 --- a/runtime/runtime_kafka_test.go +++ b/runtime/runtime_kafka_test.go @@ -14,6 +14,7 @@ import ( "mokapi/runtime" "mokapi/runtime/events" "mokapi/runtime/events/eventstest" + "mokapi/runtime/metrics" "mokapi/runtime/monitor" "net/url" "strings" @@ -92,16 +93,16 @@ func TestApp_AddKafka(t *testing.T) { traits := events.NewTraits().WithNamespace("kafka").WithName("foo").With("topic", "foo") _ = app.Events.Push(&eventstest.Event{Name: "bar"}, traits) stores := app.Events.GetStores(traits) - require.Len(t, stores, 3, "expected to find two stores for topic foo") - require.Equal(t, stores[2].Traits, traits) - require.Equal(t, 1, stores[2].NumEvents) + require.Len(t, stores, 4, "expected to find two stores for topic foo") + require.Equal(t, stores[3].Traits, traits) + require.Equal(t, 1, stores[3].NumEvents) traits = events.NewTraits().WithNamespace("kafka").WithName("foo").With("topic", "bar") _ = app.Events.Push(&eventstest.Event{Name: "bar"}, traits) stores = app.Events.GetStores(traits) - require.Len(t, stores, 3, "expected to find two stores for topic bar") - require.Equal(t, stores[2].Traits, traits) - require.Equal(t, 1, stores[2].NumEvents) + require.Len(t, stores, 4, "expected to find two stores for topic bar") + require.Equal(t, stores[3].Traits, traits) + require.Equal(t, 1, stores[3].NumEvents) }, }, { @@ -119,7 +120,7 @@ func TestApp_AddKafka(t *testing.T) { // wait for update monitor time.Sleep(500 * time.Millisecond) - require.Equal(t, float64(1), m.Messages.Sum()) + require.Equal(t, float64(1), m.Messages.Sum(metrics.NewQuery())) }, }, { diff --git a/runtime/runtime_ldap.go b/runtime/runtime_ldap.go index 40daceb0c..82b6f4d8a 100644 --- a/runtime/runtime_ldap.go +++ b/runtime/runtime_ldap.go @@ -203,3 +203,13 @@ func IsLdapConfig(c *dynamic.Config) (*directory.Config, bool) { func getLdapConfig(c *dynamic.Config) *directory.Config { return c.Data.(*directory.Config) } + +func (s *LdapStore) Len() int { + if s == nil { + return 0 + } + + s.m.RLock() + defer s.m.RUnlock() + return len(s.infos) +} diff --git a/runtime/runtime_ldap_search.go b/runtime/runtime_ldap_search.go index af43a7496..510d5ef94 100644 --- a/runtime/runtime_ldap_search.go +++ b/runtime/runtime_ldap_search.go @@ -8,13 +8,14 @@ import ( ) type ldapSearchIndexData struct { - Type string `json:"type"` - Discriminator string `json:"discriminator"` - Api string `json:"api"` - Name string `json:"name"` - Version string `json:"version"` - Server string `json:"server"` - Description string `json:"description"` + Type string `json:"type"` + Discriminator string `json:"discriminator"` + Api string `json:"api"` + Name string `json:"name"` + Version string `json:"version"` + Server string `json:"server"` + Description string `json:"description"` + Meta map[string]string `json:"meta"` } type ldapSearchIndexEntry struct { @@ -44,6 +45,9 @@ func (s *LdapStore) addToIndex(cfg *directory.Config) { Version: cfg.Info.Version, Description: cfg.Info.Description, Server: cfg.Address, + Meta: map[string]string{ + "entries": fmt.Sprintf("%d", cfg.Entries.Len()), + }, } s.index.Add(fmt.Sprintf("ldap_%s", cfg.Info.Name), c) @@ -63,7 +67,7 @@ func (s *LdapStore) addToIndex(cfg *directory.Config) { Values: values, }) } - s.index.Add(fmt.Sprintf("mail_%s_%s", cfg.Info.Name, e.Dn), se) + s.index.Add(fmt.Sprintf("ldap_%s_%s", cfg.Info.Name, e.Dn), se) } } } @@ -79,6 +83,12 @@ func getLdapSearchResult(fields map[string]string, discriminator []string) (sear "type": strings.ToLower(result.Type), "service": result.Title, } + for k, v := range fields { + if strings.HasPrefix(k, "meta.") { + k = strings.Replace(k, "meta.", "", 1) + result.Params[k] = v + } + } return result, nil } @@ -89,6 +99,7 @@ func getLdapSearchResult(fields map[string]string, discriminator []string) (sear result.Params = map[string]string{ "type": strings.ToLower(result.Type), "service": result.Domain, + "entry": fields["dn"], } default: return result, fmt.Errorf("unsupported search result: %s", strings.Join(discriminator, "_")) diff --git a/runtime/runtime_ldap_search_test.go b/runtime/runtime_ldap_search_test.go index b42d6265c..ab03ad006 100644 --- a/runtime/runtime_ldap_search_test.go +++ b/runtime/runtime_ldap_search_test.go @@ -6,8 +6,11 @@ import ( "mokapi/config/dynamic/dynamictest" "mokapi/config/static" "mokapi/engine/enginetest" + "mokapi/ldap" + "mokapi/ldap/ldaptest" "mokapi/providers/directory" "mokapi/runtime" + "mokapi/runtime/monitor" "mokapi/runtime/search" "mokapi/safe" "mokapi/sortedmap" @@ -52,6 +55,7 @@ func TestIndex_Ldap(t *testing.T) { Params: map[string]string{ "type": "ldap", "service": "foo", + "entries": "0", }, }, r.Results[0]) @@ -113,6 +117,7 @@ func TestIndex_Ldap(t *testing.T) { Params: map[string]string{ "type": "ldap", "service": "foo", + "entry": "cn=alice,dc=foo,dc=com", }, }, r.Results[0]) @@ -151,3 +156,70 @@ func TestIndex_Ldap(t *testing.T) { }) } } + +func TestIndex_Ldap_Event(t *testing.T) { + api := &directory.Config{Info: directory.Info{Name: "Test LDAP Events"}} + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: api, + } + + testcases := []struct { + name string + test func(t *testing.T, h ldap.Handler, app *runtime.App) + }{ + { + name: "search event by operation", + test: func(t *testing.T, h ldap.Handler, app *runtime.App) { + h.ServeLDAP(&ldaptest.ResponseRecorder{}, ldaptest.NewRequest(0, &ldap.SearchRequest{ + Scope: ldap.ScopeWholeSubtree, + BaseDN: "ou=people,o=search", + Filter: "(cn=user)", + })) + + r, err := waitSearchResult(t, func() (search.Result, error) { + return app.Search(search.Request{QueryText: "+operation:search +type:event", Limit: 10}) + }, 1) + + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "Event", r.Results[0].Type) + require.Equal(t, "Test LDAP Events", r.Results[0].Domain) + require.Equal(t, "(cn=user)", r.Results[0].Title) + require.Len(t, r.Results[0].Fragments, 2) + require.Contains(t, r.Results[0].Fragments, "Search") + require.Contains(t, r.Results[0].Fragments, "event") + require.Len(t, r.Results[0].Params, 6) + require.Equal(t, "event", r.Results[0].Params["type"]) + require.Equal(t, "ldap", r.Results[0].Params["traits.namespace"]) + require.Equal(t, "Test LDAP Events", r.Results[0].Params["traits.name"]) + require.Equal(t, "search", r.Results[0].Params["traits.operation"]) + require.Equal(t, "ou=people,o=search", r.Results[0].Params["baseDN"]) + require.Contains(t, r.Results[0].Params, "id") + require.NotEmpty(t, r.Results[0].Time) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + + info := app.Ldap.Add(cfg, enginetest.NewEngine()) + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + + h := info.Handler(monitor.NewLdap(), app.Events) + tc.test(t, h, app) + }) + } +} diff --git a/runtime/runtime_ldap_test.go b/runtime/runtime_ldap_test.go index ff4510534..c36940605 100644 --- a/runtime/runtime_ldap_test.go +++ b/runtime/runtime_ldap_test.go @@ -11,6 +11,7 @@ import ( "mokapi/runtime" "mokapi/runtime/events" "mokapi/runtime/events/eventstest" + "mokapi/runtime/metrics" "mokapi/runtime/monitor" "net/url" "testing" @@ -44,7 +45,7 @@ func TestApp_AddLdap(t *testing.T) { rr := ldaptest.NewRecorder() h.ServeLDAP(rr, r) - require.Equal(t, float64(1), m.RequestCounter.Sum()) + require.Equal(t, float64(1), m.RequestCounter.Sum(metrics.NewQuery())) }, }, { diff --git a/runtime/runtime_mail.go b/runtime/runtime_mail.go index df9da9ba5..5bc383beb 100644 --- a/runtime/runtime_mail.go +++ b/runtime/runtime_mail.go @@ -282,3 +282,13 @@ func (h *mailHandler) Idle(w imap.UpdateWriter, done chan struct{}, ctx context. func getSmtpConfig(c *dynamic.Config) *mail.Config { return c.Data.(*mail.Config) } + +func (s *MailStore) Len() int { + if s == nil { + return 0 + } + + s.m.RLock() + defer s.m.RUnlock() + return len(s.infos) +} diff --git a/runtime/runtime_mail_search_test.go b/runtime/runtime_mail_search_test.go index 3487a3d27..9e155fe84 100644 --- a/runtime/runtime_mail_search_test.go +++ b/runtime/runtime_mail_search_test.go @@ -5,12 +5,17 @@ import ( "mokapi/config/dynamic" "mokapi/config/dynamic/dynamictest" "mokapi/config/static" + "mokapi/engine/enginetest" "mokapi/imap" "mokapi/providers/mail" "mokapi/runtime" + "mokapi/runtime/monitor" "mokapi/runtime/search" "mokapi/safe" + "mokapi/smtp" + "mokapi/smtp/smtptest" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -171,3 +176,85 @@ func TestIndex_Mail(t *testing.T) { }) } } + +func TestIndex_Mail_Event(t *testing.T) { + api := &mail.Config{ + Info: mail.Info{Name: "Test Mail Events"}, + } + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: api, + } + + testcases := []struct { + name string + test func(t *testing.T, h runtime.MailHandler, app *runtime.App) + }{ + { + name: "search event by subject", + test: func(t *testing.T, h runtime.MailHandler, app *runtime.App) { + sendMail(h, "alice@foo.bar", "bob@foo.bar", "Test Mail", "A random body text") + + r, err := waitSearchResult(t, func() (search.Result, error) { + return app.Search(search.Request{QueryText: `+subject:"Test Mail" +type:event`, Limit: 10}) + }, 1) + + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "Event", r.Results[0].Type) + require.Equal(t, "Test Mail Events", r.Results[0].Domain) + require.Equal(t, "Test Mail", r.Results[0].Title) + require.Len(t, r.Results[0].Fragments, 2) + require.Contains(t, r.Results[0].Fragments, "Test Mail") + require.Contains(t, r.Results[0].Fragments, "event") + require.Len(t, r.Results[0].Params, 4) + require.Equal(t, "event", r.Results[0].Params["type"]) + require.Equal(t, "mail", r.Results[0].Params["traits.namespace"]) + require.Equal(t, "Test Mail Events", r.Results[0].Params["traits.name"]) + require.Contains(t, r.Results[0].Params, "id") + require.NotEmpty(t, r.Results[0].Time) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + app := runtime.New( + &static.Config{ + Api: static.Api{ + Search: static.Search{ + Enabled: true, + InMemory: true, + }, + }, + }, &dynamictest.Reader{}) + + info := app.Mail.Add(cfg) + pool := safe.NewPool(context.Background()) + app.Start(pool) + defer pool.Stop() + + h := info.Handler(monitor.NewMail(), enginetest.NewEngine(), app.Events) + + tc.test(t, h, app) + }) + } +} + +func sendMail(h runtime.MailHandler, from, to, subject, body string) { + ctx := context.Background() + ctx = smtp.NewClientContext(ctx, "") + + rr := smtptest.NewRecorder() + h.ServeSMTP(rr, smtp.NewMailRequest(from, ctx)) + + h.ServeSMTP(rr, smtp.NewRcptRequest(to, ctx)) + + h.ServeSMTP(rr, smtp.NewDataRequest(&smtp.Message{ + From: []smtp.Address{{Address: from}}, + To: []smtp.Address{{Address: to}}, + Date: time.Now(), + Subject: subject, + Body: body, + }, ctx)) +} diff --git a/runtime/runtime_mail_test.go b/runtime/runtime_mail_test.go index 2cb448e71..4d981d138 100644 --- a/runtime/runtime_mail_test.go +++ b/runtime/runtime_mail_test.go @@ -10,6 +10,7 @@ import ( "mokapi/runtime" "mokapi/runtime/events" "mokapi/runtime/events/eventstest" + "mokapi/runtime/metrics" "mokapi/runtime/monitor" "mokapi/smtp" "mokapi/smtp/smtptest" @@ -45,7 +46,7 @@ func TestApp_AddSmtp(t *testing.T) { rr := smtptest.NewRecorder() h.ServeSMTP(rr, smtp.NewDataRequest(&smtp.Message{}, ctx)) - require.Equal(t, float64(1), m.Mails.Sum()) + require.Equal(t, float64(1), m.Mails.Sum(metrics.NewQuery())) }, }, { diff --git a/runtime/runtime_mqtt.go b/runtime/runtime_mqtt.go index ee02851d1..9d60f1407 100644 --- a/runtime/runtime_mqtt.go +++ b/runtime/runtime_mqtt.go @@ -10,8 +10,11 @@ import ( "mokapi/providers/asyncapi3/mqtt/store" "mokapi/runtime/events" "mokapi/runtime/monitor" + "mokapi/runtime/search" + "mokapi/sortedmap" "path/filepath" "sort" + "strings" "sync" log "github.com/sirupsen/logrus" @@ -23,6 +26,9 @@ type MqttStore struct { cfg *static.Config sm *events.StoreManager m sync.RWMutex + events *events.StoreManager + index search.Index + reader dynamic.Reader } type MqttInfo struct { @@ -38,14 +44,13 @@ type MqttHandler struct { next mqtt.Handler } -func NewMqttInfo(c *dynamic.Config, store *store.Store, updateEventAndMetrics func(info *MqttInfo)) *MqttInfo { +func newMqttInfo(store *store.Store, updateEventAndMetrics func(info *MqttInfo)) *MqttInfo { hc := &MqttInfo{ configs: map[string]*dynamic.Config{}, Store: store, seenTopics: map[string]bool{}, updateEventAndMetrics: updateEventAndMetrics, } - hc.AddConfig(c) return hc } @@ -92,13 +97,16 @@ func (s *MqttStore) Add(c *dynamic.Config, emitter common.EventEmitter) (*MqttIn } if !ok { - s.sm.ResetStores(events.NewTraits().WithNamespace("Mqtt").WithName(cfg.Info.Name)) - s.sm.SetStore(int(eventStore.Size), events.NewTraits().WithNamespace("Mqtt").WithName(cfg.Info.Name)) + s.sm.ResetStores(events.NewTraits().WithNamespace("mqtt").WithName(cfg.Info.Name)) + s.sm.SetStore(int(eventStore.Size), events.NewTraits().WithNamespace("mqtt").WithName(cfg.Info.Name)) - ki = NewMqttInfo(c, store.New(cfg, emitter), s.updateEventStore) + ki = newMqttInfo(store.New(cfg, emitter, s.events, s.monitor.Mqtt), s.updateEventStore) s.infos[cfg.Info.Name] = ki - } else { - ki.AddConfig(c) + } + ki.addConfig(c, s.reader) + + if s.cfg.Api.Search.Enabled { + s.addToIndex(ki.Config) } return ki, nil @@ -122,11 +130,17 @@ func (s *MqttStore) Remove(c *dynamic.Config) { if err != nil { return } - name := cfg.Info.Name - ki := s.infos[name] - ki.Remove(c) - if len(ki.configs) == 0 { + mi := s.infos[name] + + if s.cfg.Api.Search.Enabled { + s.removeFromIndex(mi.Config) + } + + delete(mi.configs, c.Info.Url.String()) + mi.update(s.reader) + + if len(mi.configs) == 0 { s.m.RUnlock() s.m.Lock() delete(s.infos, name) @@ -137,13 +151,13 @@ func (s *MqttStore) Remove(c *dynamic.Config) { } } -func (c *MqttInfo) AddConfig(config *dynamic.Config) { +func (c *MqttInfo) addConfig(config *dynamic.Config, reader dynamic.Reader) { key := config.Info.Url.String() c.configs[key] = config - c.update() + c.update(reader) } -func (c *MqttInfo) update() { +func (c *MqttInfo) update(reader dynamic.Reader) { if len(c.configs) == 0 { c.Config = nil c.Store = nil @@ -175,6 +189,28 @@ func (c *MqttInfo) update() { } } + if len(c.configs) > 1 { + err := cfg.Parse(&dynamic.Config{Data: cfg}, reader) + if err != nil { + log.Errorf("failed to parse config: %s", err) + } + } + + if cfg.Servers.Len() == 0 { + log.Infof("no servers defined in AsyncAPI spec — using default Mokapi broker for cluster '%s'", cfg.Info.Name) + if cfg.Servers == nil { + cfg.Servers = &sortedmap.LinkedHashMap[string, *asyncapi3.ServerRef]{} + } + cfg.Servers.Set("mokapi", &asyncapi3.ServerRef{ + Value: &asyncapi3.Server{ + Host: ":1883", + Protocol: "mqtt", + Title: "Mokapi Default Broker", + Summary: "Automatically added broker because no servers are defined in the AsyncAPI spec", + }, + }) + } + c.Config = cfg c.updateEventAndMetrics(c) c.Store.Update(cfg) @@ -199,37 +235,21 @@ func (h *MqttHandler) ServeMessage(rw mqtt.MessageWriter, req *mqtt.Message) { h.next.ServeMessage(rw, req) } -func IsMqttConfig(c *dynamic.Config) (*asyncapi3.Config, bool) { - var cfg *asyncapi3.Config - if old, ok := c.Data.(*asyncApi.Config); ok { - var err error - cfg, err = old.Convert() - if err != nil { - return nil, false - } - } else { - cfg, ok = c.Data.(*asyncapi3.Config) - if !ok { - return nil, false - } +func HasMqttServer(c *dynamic.Config) (*asyncapi3.Config, bool) { + cfg, ok := IsAsyncApiConfig(c) + if !ok { + return nil, false } - - return cfg, hasMqttBroker(cfg) -} - -func hasMqttBroker(c *asyncapi3.Config) bool { - for it := c.Servers.Iter(); it.Next(); { - server := it.Value() - if server.Value.Protocol == "mqtt" { - return true + for it := cfg.Servers.Iter(); it.Next(); { + s := it.Value() + if s.Value == nil { + continue + } + if strings.ToLower(s.Value.Protocol) == "mqtt" { + return cfg, true } } - return false -} - -func (c *MqttInfo) Remove(cfg *dynamic.Config) { - delete(c.configs, cfg.Info.Url.String()) - c.update() + return cfg, false } func getMqttConfig(c *dynamic.Config) (*asyncapi3.Config, error) { @@ -264,3 +284,13 @@ func (s *MqttStore) updateEventStore(k *MqttInfo) { k.seenTopics[topicName] = true } } + +func (s *MqttStore) Len() int { + if s == nil { + return 0 + } + + s.m.RLock() + defer s.m.RUnlock() + return len(s.infos) +} diff --git a/runtime/runtime_mqtt_search.go b/runtime/runtime_mqtt_search.go new file mode 100644 index 000000000..a75b40206 --- /dev/null +++ b/runtime/runtime_mqtt_search.go @@ -0,0 +1,189 @@ +package runtime + +import ( + "fmt" + "mokapi/providers/asyncapi3" + "mokapi/runtime/search" + "mokapi/schema/json/schema" + "strings" + + log "github.com/sirupsen/logrus" +) + +type mqttSearchIndexData struct { + Type string `json:"type"` + Discriminator string `json:"discriminator"` + Api string `json:"api"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Contact *asyncapi3.Contact `json:"contact"` + Servers []mqttServerSearchData `json:"servers"` + Meta map[string]string `json:"meta"` +} + +type mqttServerSearchData struct { + Name string `json:"name"` + Host string `json:"host"` + Title string `json:"title"` + Summary string `json:"summary"` + Description string `json:"description"` +} + +type mqttTopicSearchIndexData struct { + Type string `json:"type"` + Discriminator string `json:"discriminator"` + Api string `json:"api"` + ChannelId string `json:"channelId"` + Name string `json:"name"` + Title string `json:"title"` + Address string `json:"address"` + Summary string `json:"summary"` + Description string `json:"description"` + Messages []mqttMessageSearchIndexData `json:"messages"` +} + +type mqttMessageSearchIndexData struct { + MessageId string `json:"messageId"` + Name string `json:"name"` + Title string `json:"title"` + Summary string `json:"summary"` + Description string `json:"description"` + Headers *schema.IndexData `json:"headers"` + Payload *schema.IndexData `json:"payload"` +} + +func (s *MqttStore) addToIndex(cfg *asyncapi3.Config) { + if cfg == nil || cfg.Info.Name == "" { + return + } + + c := mqttSearchIndexData{ + Type: "mqtt", + Discriminator: "mqtt", + Api: cfg.Info.Name, + Name: cfg.Info.Name, + Version: cfg.Info.Version, + Description: cfg.Info.Description, + Contact: cfg.Info.Contact, + Meta: map[string]string{ + "topics": fmt.Sprintf("%d", len(cfg.Channels)), + }, + } + for it := cfg.Servers.Iter(); it.Next(); { + name := it.Key() + server := it.Value() + if server == nil || server.Value == nil { + continue + } + c.Servers = append(c.Servers, mqttServerSearchData{ + Name: name, + Host: server.Value.Host, + Title: server.Value.Title, + Summary: server.Value.Summary, + Description: server.Value.Description, + }) + } + + s.index.Add(fmt.Sprintf("mqtt_%s", cfg.Info.Name), c) + + for name, topic := range cfg.Channels { + if topic == nil || topic.Value == nil { + continue + } + topicName := name + if topic.Value.Name != "" { + topicName = topic.Value.Name + } + if topic.Value.Address != "" { + topicName = topic.Value.Address + } + + t := mqttTopicSearchIndexData{ + Type: "mqtt", + Discriminator: "mqtt_topic", + Api: cfg.Info.Name, + ChannelId: name, + Name: topicName, + Title: topic.Value.Title, + Address: topic.Value.Address, + Summary: topic.Value.Summary, + Description: topic.Value.Description, + } + + for messageId, message := range topic.Value.Messages { + if message == nil || message.Value == nil { + continue + } + h, err := getSchema(message.Value.Headers) + if err != nil { + log.Errorf("indexing message for topic '%v' failed for headers: %v", topic.Value.Name, err) + } + p, err := getSchema(message.Value.Headers) + if err != nil { + log.Errorf("indexing message for topic '%v' failed for payload: %v", topic.Value.Name, err) + } + + t.Messages = append(t.Messages, mqttMessageSearchIndexData{ + MessageId: messageId, + Name: message.Value.Name, + Title: message.Value.Title, + Summary: message.Value.Summary, + Description: message.Value.Description, + Headers: h, + Payload: p, + }) + } + id := fmt.Sprintf("mqtt_%s_%s", cfg.Info.Name, name) + s.index.Add(id, t) + } +} + +func getMqttSearchResult(fields map[string]string, discriminator []string) (search.ResultItem, error) { + result := search.ResultItem{ + Type: "MQTT", + } + + if len(discriminator) == 1 { + result.Title = fields["name"] + result.Params = map[string]string{ + "type": strings.ToLower(result.Type), + "service": result.Title, + } + for k, v := range fields { + if strings.HasPrefix(k, "meta.") { + k = strings.Replace(k, "meta.", "", 1) + result.Params[k] = v + } + } + return result, nil + } + + switch discriminator[1] { + case "topic": + title := fields["channelId"] + if len(fields["name"]) > 0 { + title = fields["name"] + } else if len(fields["title"]) > 0 { + title = fields["title"] + } + result.Domain = fields["api"] + result.Title = fmt.Sprintf("Topic %s", title) + result.Params = map[string]string{ + "type": strings.ToLower(result.Type), + "service": result.Domain, + "topic": fields["name"], + } + default: + return result, fmt.Errorf("unsupported search result: %s", strings.Join(discriminator, "_")) + } + return result, nil +} + +func (s *MqttStore) removeFromIndex(cfg *asyncapi3.Config) { + s.index.Delete(fmt.Sprintf("mqtt_%s", cfg.Info.Name)) + + for name := range cfg.Channels { + s.index.Delete(fmt.Sprintf("mqtt_%s_%s", cfg.Info.Name, name)) + } +} diff --git a/runtime/runtime_search.go b/runtime/runtime_search.go index b2947ea0b..300b26a41 100644 --- a/runtime/runtime_search.go +++ b/runtime/runtime_search.go @@ -2,7 +2,12 @@ package runtime import ( "mokapi/config/dynamic" + "mokapi/providers/asyncapi3" + "mokapi/providers/directory" + "mokapi/providers/mail" + "mokapi/providers/openapi" "mokapi/runtime/search" + "path/filepath" "strings" ) @@ -12,16 +17,38 @@ type config struct { Name string `json:"name"` Id string `json:"id"` Data string `json:"data"` + Source string `json:"source"` } func (a *App) addConfigToIndex(cfg *dynamic.Config) { - a.searchIndex.Add(cfg.Info.Key(), config{ + data := config{ Discriminator: "config", Provider: cfg.Info.Provider, Name: cfg.Info.Path(), Id: cfg.Info.Key(), Data: string(cfg.Raw), - }) + } + + switch cfg.Data.(type) { + case *openapi.Config: + data.Source = "OpenAPI" + case *asyncapi3.Config: + data.Source = "AsyncAPI" + case *directory.Config: + data.Source = "LDAP" + case *mail.Config: + data.Source = "LDAP" + default: + ext := filepath.Ext(cfg.Info.Path()) + switch ext { + case ".js", ".ts": + data.Source = "JavaScript" + case ".ldif": + data.Source = "LDIF" + } + } + + a.searchIndex.Add(cfg.Info.Key(), data) } func (a *App) removeConfigFromIndex(cfg *dynamic.Config) { @@ -34,8 +61,9 @@ func getConfigSearchResult(fields map[string]string, _ []string) (search.ResultI Domain: strings.ToUpper(fields["provider"]), Title: fields["name"], Params: map[string]string{ - "type": "config", - "id": fields["id"], + "type": "config", + "id": fields["id"], + "source": fields["source"], }, }, nil } diff --git a/runtime/runtime_search_test.go b/runtime/runtime_search_test.go index fbe7608b2..c863563f0 100644 --- a/runtime/runtime_search_test.go +++ b/runtime/runtime_search_test.go @@ -7,6 +7,7 @@ import ( "mokapi/config/static" "mokapi/engine/enginetest" "mokapi/providers/asyncapi3/asyncapi3test" + "mokapi/providers/openapi" "mokapi/providers/openapi/openapitest" "mokapi/runtime" "mokapi/runtime/search" @@ -38,6 +39,7 @@ func TestIndex_Config(t *testing.T) { cfg := &dynamic.Config{ Info: info, Raw: []byte(`{"name":"test"}`), + Data: &openapi.Config{}, } app.UpdateConfig(dynamic.ConfigEvent{ @@ -60,8 +62,9 @@ func TestIndex_Config(t *testing.T) { Title: "file://foo.yml", Fragments: []string{"{"name":"test"}"}, Params: map[string]string{ - "type": "config", - "id": "64613435-3062-6462-3033-316532633233", + "type": "config", + "id": "64613435-3062-6462-3033-316532633233", + "source": "OpenAPI", }, }, r.Results[0]) @@ -71,7 +74,7 @@ func TestIndex_Config(t *testing.T) { name: "kafka and http indexed", test: func(t *testing.T, app *runtime.App) { h := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) - app.AddHttp(toConfig(h)) + app.Http.Add(toConfig(h)) k := asyncapi3test.NewConfig(asyncapi3test.WithInfo("foo", "", "")) _, err := app.Kafka.Add(toConfig(k), enginetest.NewEngine()) require.NoError(t, err) @@ -85,6 +88,33 @@ func TestIndex_Config(t *testing.T) { require.Len(t, r.Results, 2) }, }, + { + name: "api:foo with operator", + test: func(t *testing.T, app *runtime.App) { + h := openapitest.NewConfig("3.0", openapitest.WithInfo("foo", "", "")) + app.Http.Add(toConfig(h)) + h = openapitest.NewConfig("3.0", openapitest.WithInfo("bar", "", "")) + app.Http.Add(toConfig(h)) + k := asyncapi3test.NewConfig(asyncapi3test.WithInfo("foo", "", "")) + _, err := app.Kafka.Add(toConfig(k), enginetest.NewEngine()) + require.NoError(t, err) + + var r search.Result + waitSearchIndex(t, func() bool { + r, err = app.Search(search.Request{QueryText: "+api:foo", Limit: 10}) + require.NoError(t, err) + return len(r.Results) == 2 + }) + require.Len(t, r.Results, 2) + require.Equal(t, "foo", r.Results[0].Title) + require.Equal(t, "foo", r.Results[1].Title) + + r, err = app.Search(search.Request{QueryText: "-api:foo", Limit: 10}) + require.NoError(t, err) + require.Len(t, r.Results, 1) + require.Equal(t, "bar", r.Results[0].Title) + }, + }, } t.Parallel() diff --git a/runtime/runtime_test.go b/runtime/runtime_test.go index 0a6b17182..f2304a01e 100644 --- a/runtime/runtime_test.go +++ b/runtime/runtime_test.go @@ -18,8 +18,8 @@ func TestNew(t *testing.T) { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) require.NotNil(t, app.Monitor) require.Equal(t, "1.0", app.Version) - require.Len(t, app.ListHttp(), 0) - require.Len(t, app.Kafka.List(), 0) - require.Len(t, app.Ldap.List(), 0) - require.Len(t, app.Mail.List(), 0) + require.Equal(t, 0, app.Http.Len()) + require.Equal(t, 0, app.Kafka.Len()) + require.Equal(t, 0, app.Ldap.Len()) + require.Equal(t, 0, app.Mail.Len()) } diff --git a/runtime/runtimetest/app.go b/runtime/runtimetest/app.go index f3fbaf70a..a0afa7c04 100644 --- a/runtime/runtimetest/app.go +++ b/runtime/runtimetest/app.go @@ -41,7 +41,7 @@ func NewApp(opts ...Options) *runtime.App { func WithHttp(configs ...*openapi.Config) Options { return func(app *runtime.App) { for i, cfg := range configs { - app.AddHttp(&dynamic.Config{ + app.Http.Add(&dynamic.Config{ Info: dynamictest.NewConfigInfo(dynamictest.WithUrl(fmt.Sprintf("%d", i))), Data: cfg, }) @@ -117,3 +117,25 @@ func WithEvent(traits events.Traits, data events.EventData) Options { } } } + +func WithMqttInfo(name string, mi *runtime.MqttInfo) Options { + return func(app *runtime.App) { + app.Mqtt.Set(name, mi) + } +} + +func WithMqtt(configs ...*asyncapi3.Config) Options { + return func(app *runtime.App) { + for i, cfg := range configs { + c := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(dynamictest.WithUrl(fmt.Sprintf("%d", i))), + Data: cfg, + } + + _, err := app.Mqtt.Add(c, app.Engine) + if err != nil { + panic(err) + } + } + } +} diff --git a/schema/encoding/decoder.go b/schema/encoding/decoder.go index 3715106f2..a30c682f1 100644 --- a/schema/encoding/decoder.go +++ b/schema/encoding/decoder.go @@ -13,9 +13,13 @@ type Decoder interface { } type Parser interface { - Parse(data interface{}) (interface{}, error) + Parse(data any) (any, error) } +type NullParser struct{} + +func (p *NullParser) Parse(data any) (any, error) { return data, nil } + var decoders []Decoder func init() { diff --git a/schema/json/schema/marshal.go b/schema/json/schema/marshal.go index d3f2a43e0..e10518c03 100644 --- a/schema/json/schema/marshal.go +++ b/schema/json/schema/marshal.go @@ -82,8 +82,12 @@ func (e *encoder) encode(r *Schema) ([]byte, error) { case *Schema: bVal, err = e.encode(val) case dynamic.Reference[*Schema]: - bVal, err = json.Marshal(val) - b.WriteString(strings.Trim(string(bVal), "{}")) + if val.Ref != "" { + if b.Len() > 1 { + b.Write([]byte{','}) + } + b.WriteString(fmt.Sprintf(`"$ref": "%s"`, val.Ref)) + } continue default: bVal, err = json.Marshal(val) diff --git a/schema/json/schema/marshal_test.go b/schema/json/schema/marshal_test.go index f74a8f671..5462cf8e8 100644 --- a/schema/json/schema/marshal_test.go +++ b/schema/json/schema/marshal_test.go @@ -92,6 +92,30 @@ func TestSchema_MarshalJSON_Recursion(t *testing.T) { require.Equal(t, `{"$ref":"foo","contains":{"$ref":"foo"}}`, s) }, }, + { + name: "definitions", + s: func() *Schema { + var s *Schema + err := json.Unmarshal([]byte(` +{ + "definitions": { + "Status": { + "$id": "status", + "description": "a description" + } + } +} +`), &s) + if err != nil { + panic(err) + } + return s + }, + test: func(t *testing.T, s string, err error) { + require.NoError(t, err) + require.Equal(t, `{"definitions":{"Status":{"$id":"status","description":"a description"}}}`, s) + }, + }, } t.Parallel() diff --git a/server/configwatcher_openapi_test.go b/server/configwatcher_openapi_test.go index d2c715ba4..0cffe8064 100644 --- a/server/configwatcher_openapi_test.go +++ b/server/configwatcher_openapi_test.go @@ -148,7 +148,8 @@ paths: time.Sleep(1 * time.Second) - require.Equal(t, "parse error /root.yml: parsing file /root.yml: parse path '/users' failed: resolve reference '/paths.yml#/paths/users' failed: parsing file /paths.yml: parse path '/users' failed: parse operation 'GET' failed: parse request body failed: resolve reference '/not_found.yaml' failed: not found: /not_found.yaml", hook.LastEntry().Message) + require.Equal(t, "parse request body failed: resolve reference '/not_found.yaml' failed: not found: /not_found.yaml", hook.LastEntry().Message) + require.Equal(t, logrus.Fields{"api": "", "method": "GET", "namespace": "http", "path": "/users"}, hook.LastEntry().Data) }, }, } diff --git a/server/httpmanager_test.go b/server/httpmanager_test.go index baf1a744d..1530cb686 100644 --- a/server/httpmanager_test.go +++ b/server/httpmanager_test.go @@ -10,6 +10,7 @@ import ( "mokapi/providers/openapi" "mokapi/providers/openapi/openapitest" "mokapi/runtime" + "mokapi/runtime/metrics" "mokapi/server/cert" "mokapi/try" "mokapi/version" @@ -43,7 +44,7 @@ func TestHttpServers_Monitor(t *testing.T) { // give server time to start time.Sleep(time.Second * 1) try.GetRequest(t, u+"/foo", map[string]string{}) - require.Equal(t, float64(1), app.Monitor.Http.RequestCounter.Sum()) + require.Equal(t, float64(1), app.Monitor.Http.RequestCounter.Sum(metrics.NewQuery())) } func TestHttpManager_Update(t *testing.T) { @@ -63,7 +64,7 @@ func TestHttpManager_Update(t *testing.T) { c := &openapi.Config{OpenApi: version.New("3.0"), Info: openapi.Info{Name: "foo"}, Servers: []*openapi.Server{{Url: "http://:80"}}} m.Update(dynamic.ConfigEvent{Config: &dynamic.Config{Data: c, Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}}}) - list := m.app.ListHttp() + list := m.app.Http.List() require.Len(t, list, 1) require.Equal(t, "foo", list[0].Info.Name) }, @@ -78,12 +79,12 @@ func TestHttpManager_Update(t *testing.T) { m.Update(dynamic.ConfigEvent{Config: &dynamic.Config{Data: foo, Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}}}) m.Update(dynamic.ConfigEvent{Config: &dynamic.Config{Data: bar, Info: dynamic.ConfigInfo{Url: MustParseUrl("bar.yml")}}}) - list := m.app.ListHttp() + list := m.app.Http.List() require.Len(t, list, 2) }, }, { - name: "add new host http://:X", + name: "add new host", test: func(t *testing.T, m *HttpManager, hook *logtest.Hook) { port := try.GetFreePort() c := &openapi.Config{OpenApi: version.New("3.0"), Info: openapi.Info{Name: "foo"}, Servers: []*openapi.Server{{Url: fmt.Sprintf("http://:%v", port)}}} @@ -163,20 +164,19 @@ func TestHttpManager_Update(t *testing.T) { }, } - for _, data := range testdata { - t.Run(data.name, func(t *testing.T) { + for _, tc := range testdata { + t.Run(tc.name, func(t *testing.T) { logrus.SetOutput(io.Discard) hook := logtest.NewGlobal() - logrus.SetLevel(logrus.DebugLevel) store, err := cert.NewStore(&static.Config{}) require.NoError(t, err) - cfg := &static.Config{} + cfg := &static.Config{Log: static.MokApiLog{Level: "debug"}} m := NewHttpManager(&engine.Engine{}, store, runtime.New(cfg, &dynamictest.Reader{})) defer m.Stop() - data.test(t, m, hook) + tc.test(t, m, hook) }) } diff --git a/server/server.go b/server/server.go index 9697099d2..b0a7ee143 100644 --- a/server/server.go +++ b/server/server.go @@ -46,12 +46,13 @@ func (s *Server) Start() error { <-s.stopChan log.Debug("stopping server") - s.pool.Stop() + s.app.Stop() s.kafka.Stop() s.http.Stop() s.mail.Stop() s.ldap.Stop() s.engine.Close() + s.pool.Stop() return nil } diff --git a/server/server_http.go b/server/server_http.go index 429b47fb3..2c01cc98a 100644 --- a/server/server_http.go +++ b/server/server_http.go @@ -79,9 +79,9 @@ func (m *HttpManager) Update(e dynamic.ConfigEvent) { return } - info := m.app.GetHttp(cfg.Info.Name) + info := m.app.Http.Get(cfg.Info.Name) if e.Event == dynamic.Delete { - m.app.RemoveHttp(e.Config) + m.app.Http.Remove(e.Config) if info.Config == nil { m.removeService(cfg.Info.Name) m.stopEmptyServers() @@ -92,7 +92,7 @@ func (m *HttpManager) Update(e dynamic.ConfigEvent) { if info != nil { servers = info.Servers } - info = m.app.AddHttp(e.Config) + info = m.app.Http.Add(e.Config) if servers != nil { m.cleanupRemovedServers(info, servers) } diff --git a/server/server_kafka.go b/server/server_kafka.go index 646e9190d..3d09749c8 100644 --- a/server/server_kafka.go +++ b/server/server_kafka.go @@ -43,8 +43,12 @@ func NewKafkaManager(emitter common.EventEmitter, app *runtime.App) *KafkaManage } func (m *KafkaManager) UpdateConfig(e dynamic.ConfigEvent) { - cfg, ok := runtime.IsAsyncApiConfig(e.Config) + cfg, ok := runtime.HasKafkaBroker(e.Config) if !ok { + if cfg == nil || m.clusters == nil { + return + } + m.removeCluster(cfg.Info.Name) return } @@ -108,7 +112,7 @@ func (c *kafkaCluster) updateBrokers(cfg *runtime.KafkaInfo, old *sortedmap.Link if !slices.ContainsFunc(servers, func(s *asyncapi3.ServerRef) bool { return s.Value != nil && server.Value != nil && s.Value.Host == server.Value.Host }) { - port, _ := getPortFromUrl(server.Value.Host) + port, _ := getPortFromUrl(server.Value.Host, "9092") if b, ok := c.brokers[port]; ok { log.Infof("removing kafka broker '%v' on port %v from cluster '%v'", name, b.Addr(), cfg.Info.Name) b.Stop() @@ -123,7 +127,7 @@ func (c *kafkaCluster) updateBrokers(cfg *runtime.KafkaInfo, old *sortedmap.Link if server == nil || server.Value == nil { continue } - port, err := getPortFromUrl(server.Value.Host) + port, err := getPortFromUrl(server.Value.Host, "9092") if err != nil { log.Errorf("unable to start broker %v for cluster %v: ", server.Value.Host, cfg.Info.Name) continue @@ -151,7 +155,7 @@ func (m *KafkaManager) Stop() { } } -func getPortFromUrl(urlString string) (string, error) { +func getPortFromUrl(urlString, defaultPort string) (string, error) { u, err := url.Parse(urlString) if err != nil || len(u.Host) == 0 { u, err = url.Parse("//" + urlString) @@ -162,7 +166,7 @@ func getPortFromUrl(urlString string) (string, error) { port := u.Port() if len(port) == 0 { - port = "9092" + port = defaultPort } return port, nil diff --git a/server/server_kafka_test.go b/server/server_kafka_test.go index 329e8b362..365dd96e8 100644 --- a/server/server_kafka_test.go +++ b/server/server_kafka_test.go @@ -49,11 +49,11 @@ func TestKafkaServer(t *testing.T) { func TestKafkaServer_Update(t *testing.T) { testcases := []struct { name string - fn func(t *testing.T, m *KafkaManager) + test func(t *testing.T, m *KafkaManager) }{ { - "add another broker", - func(t *testing.T, m *KafkaManager) { + name: "add another broker", + test: func(t *testing.T, m *KafkaManager) { port1 := try.GetFreePort() addr1 := fmt.Sprintf("127.0.0.1:%v", port1) cfg := asyncapi3test.NewConfig( @@ -92,8 +92,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "add broker", - func(t *testing.T, m *KafkaManager) { + name: "add broker", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -120,8 +120,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "remove broker", - func(t *testing.T, m *KafkaManager) { + name: "remove broker", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -148,8 +148,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "remove broker but still has one", - func(t *testing.T, m *KafkaManager) { + name: "remove broker but still has one", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr1 := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -211,8 +211,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "change broker name", - func(t *testing.T, m *KafkaManager) { + name: "change broker name", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -244,8 +244,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "add topic", - func(t *testing.T, m *KafkaManager) { + name: "add topic", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -282,8 +282,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "remove topic", - func(t *testing.T, m *KafkaManager) { + name: "remove topic", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -318,8 +318,8 @@ func TestKafkaServer_Update(t *testing.T) { }, }, { - "remove cluster", - func(t *testing.T, m *KafkaManager) { + name: "remove cluster", + test: func(t *testing.T, m *KafkaManager) { port := try.GetFreePort() addr := fmt.Sprintf("127.0.0.1:%v", port) cfg := asyncapi3test.NewConfig( @@ -346,6 +346,48 @@ func TestKafkaServer_Update(t *testing.T) { require.Error(t, err) }, }, + { + name: "mqtt topic should not be available", + test: func(t *testing.T, m *KafkaManager) { + port := try.GetFreePort() + addr := fmt.Sprintf("127.0.0.1:%v", port) + cfg := asyncapi3test.NewConfig( + asyncapi3test.WithTitle("foo"), + asyncapi3test.WithServer("mqtt12", "mqtt", addr), + asyncapi3test.WithServer("kafka", "kafka", addr), + asyncapi3test.WithChannel("foo", + asyncapi3test.WithMessage("foo", + asyncapi3test.WithPayload( + &schema.Schema{Type: schema.Types{"string"}}, + ), + ), + asyncapi3test.AssignToServer("#/servers/mqtt12"), + ), + asyncapi3test.WithChannel("bar", + asyncapi3test.WithMessage("bar", + asyncapi3test.WithPayload( + &schema.Schema{Type: schema.Types{"string"}}, + ), + ), + asyncapi3test.AssignToServer("#/servers/kafka"), + ), + ) + c := &dynamic.Config{Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}, Data: cfg} + err := cfg.Parse(c, &dynamictest.Reader{}) + require.NoError(t, err) + + m.UpdateConfig(dynamic.ConfigEvent{Config: c}) + + // wait for mqtt start + time.Sleep(500 * time.Millisecond) + + client := kafkatest.NewClient(addr, "test") + r, err := client.Metadata(0, &metaData.Request{}) + require.NoError(t, err) + require.Len(t, r.Topics, 1) + require.Equal(t, "bar", r.Topics[0].Name) + }, + }, } for _, tc := range testcases { @@ -355,7 +397,7 @@ func TestKafkaServer_Update(t *testing.T) { m := NewKafkaManager(nil, runtime.New(cfg, &dynamictest.Reader{})) defer m.Stop() - tc.fn(t, m) + tc.test(t, m) }) } } diff --git a/server/server_mqtt.go b/server/server_mqtt.go index e95467bdb..c5fc2cefe 100644 --- a/server/server_mqtt.go +++ b/server/server_mqtt.go @@ -32,28 +32,29 @@ func NewMqttManager(emitter common.EventEmitter, app *runtime.App) *MqttManager } func (m *MqttManager) UpdateConfig(e dynamic.ConfigEvent) { - // todo: should be IsAsyncConfig and HasMqttBrokers - cfg, ok := runtime.IsMqttConfig(e.Config) + cfg, ok := runtime.HasMqttServer(e.Config) if !ok { + if cfg == nil || m.clusters == nil { + return + } + m.removeCluster(cfg.Info.Name) return } info := m.app.Mqtt.Get(cfg.Info.Name) - if e.Event == dynamic.Delete { + if e.Event == dynamic.Delete && !info.HasMqttServer() { m.app.Kafka.Remove(e.Config) if info.Config == nil { m.removeCluster(cfg.Info.Name) return } - } else if info == nil { - var err error - info, err = m.app.Mqtt.Add(e.Config, m.emitter) - if err != nil { - log.Errorf("add MQTT config %v failed: %v", e.Config.Info.Url, err) - return - } - } else { - info.AddConfig(e.Config) + } + + var err error + info, err = m.app.Mqtt.Add(e.Config, m.emitter) + if err != nil { + log.Errorf("add MQTT config %v failed: %v", e.Config.Info.Url, err) + return } m.addOrUpdateCluster(info) @@ -100,10 +101,10 @@ func (c *mqttCluster) updateBrokers(cfg *runtime.MqttInfo, monitor *monitor.Mqtt for it := cfg.Servers.Iter(); it.Next(); { name := it.Key() server := it.Value() - if server == nil || server.Value == nil { + if server == nil || server.Value == nil || server.Value.Protocol != "mqtt" { continue } - port, err := getPortFromUrl(server.Value.Host) + port, err := getPortFromUrl(server.Value.Host, "1883") if err != nil { log.Errorf("unable to start MQTT broker %v for cluster %v: ", server.Value.Host, cfg.Info.Name) continue diff --git a/server/server_mqtt_test.go b/server/server_mqtt_test.go index 9c4a8a218..823038add 100644 --- a/server/server_mqtt_test.go +++ b/server/server_mqtt_test.go @@ -5,6 +5,8 @@ import ( "mokapi/config/dynamic" "mokapi/config/dynamic/dynamictest" "mokapi/config/static" + "mokapi/mqtt" + "mokapi/mqtt/mqtttest" "mokapi/providers/asyncapi3/asyncapi3test" "mokapi/runtime" "mokapi/schema/json/schema" @@ -16,29 +18,107 @@ import ( ) func TestMqttServer(t *testing.T) { - port := try.GetFreePort() - addr := fmt.Sprintf("127.0.0.1:%v", port) - c := asyncapi3test.NewConfig( - asyncapi3test.WithTitle("foo"), - asyncapi3test.WithServer("mqtt12", "mqtt", addr), - asyncapi3test.WithChannel("foo", - asyncapi3test.WithMessage("foo", - asyncapi3test.WithPayload( - &schema.Schema{Type: schema.Types{"string"}}, - ), - ), - ), - ) - - cfg := &static.Config{} - m := NewMqttManager(nil, runtime.New(cfg, &dynamictest.Reader{})) - defer m.Stop() - m.UpdateConfig(dynamic.ConfigEvent{Config: &dynamic.Config{Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}, Data: c}}) - - // wait for kafka start - time.Sleep(500 * time.Millisecond) - - require.Len(t, m.clusters, 1) - _, ok := m.clusters["foo"] - require.True(t, ok, "cluster exists") + testcases := []struct { + name string + test func(t *testing.T, m *MqttManager) + }{ + { + name: "TestMqttServer", + test: func(t *testing.T, m *MqttManager) { + port := try.GetFreePort() + addr := fmt.Sprintf("127.0.0.1:%v", port) + c := asyncapi3test.NewConfig( + asyncapi3test.WithTitle("foo"), + asyncapi3test.WithServer("mqtt12", "mqtt", addr), + asyncapi3test.WithChannel("foo", + asyncapi3test.WithMessage("foo", + asyncapi3test.WithPayload( + &schema.Schema{Type: schema.Types{"string"}}, + ), + ), + ), + ) + + m.UpdateConfig(dynamic.ConfigEvent{Config: &dynamic.Config{Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}, Data: c}}) + + // wait for mqtt start + time.Sleep(500 * time.Millisecond) + + client := mqtttest.NewClient(addr) + defer client.Close() + _, err := client.Send(&mqtt.Message{ + Header: &mqtt.Header{Type: mqtt.CONNECT}, + Payload: &mqtt.ConnectRequest{}, + }) + require.NoError(t, err) + }, + }, + { + name: "kafka topic should not be available", + test: func(t *testing.T, m *MqttManager) { + port := try.GetFreePort() + addr := fmt.Sprintf("127.0.0.1:%v", port) + cfg := asyncapi3test.NewConfig( + asyncapi3test.WithTitle("foo"), + asyncapi3test.WithServer("mqtt12", "mqtt", addr), + asyncapi3test.WithServer("kafka", "kafka", fmt.Sprintf("127.0.0.1:%v", try.GetFreePort())), + asyncapi3test.WithChannel("foo", + asyncapi3test.WithMessage("foo", + asyncapi3test.WithPayload( + &schema.Schema{Type: schema.Types{"string"}}, + ), + ), + asyncapi3test.AssignToServer("#/servers/mqtt12"), + ), + asyncapi3test.WithChannel("bar", + asyncapi3test.WithMessage("bar", + asyncapi3test.WithPayload( + &schema.Schema{Type: schema.Types{"string"}}, + ), + ), + asyncapi3test.AssignToServer("#/servers/kafka"), + ), + ) + c := &dynamic.Config{Info: dynamic.ConfigInfo{Url: MustParseUrl("foo.yml")}, Data: cfg} + err := cfg.Parse(c, &dynamictest.Reader{}) + require.NoError(t, err) + + m.UpdateConfig(dynamic.ConfigEvent{Config: c}) + + // wait for mqtt start + time.Sleep(500 * time.Millisecond) + + client := mqtttest.NewClient(addr) + defer client.Close() + msg, err := client.Send(&mqtt.Message{ + Header: &mqtt.Header{Type: mqtt.CONNECT}, + Payload: &mqtt.ConnectRequest{ClientId: "client"}, + }) + require.NoError(t, err) + require.IsType(t, &mqtt.ConnectResponse{}, msg.Payload) + require.Equal(t, mqtt.Success.Code, msg.Payload.(*mqtt.ConnectResponse).ReasonCode.Code) + + msg, err = client.Send(&mqtt.Message{ + Header: &mqtt.Header{Type: mqtt.PUBLISH, QoS: 1}, + Payload: &mqtt.PublishRequest{Topic: "bar", MessageId: uint16(123)}, + }) + require.NoError(t, err) + require.IsType(t, &mqtt.PublishResponse{}, msg.Payload) + require.Equal(t, mqtt.TopicNameInvalid, msg.Payload.(*mqtt.PublishResponse).ReasonCode) + }, + }, + } + + t.Parallel() + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + m := NewMqttManager(nil, runtime.New(&static.Config{}, &dynamictest.Reader{})) + defer m.Stop() + + tc.test(t, m) + }) + } } diff --git a/server/service/mqtt_test.go b/server/service/mqtt_test.go index 1160e7ccb..1b911c7e5 100644 --- a/server/service/mqtt_test.go +++ b/server/service/mqtt_test.go @@ -23,7 +23,7 @@ func TestMqttBroker(t *testing.T) { }, Payload: &mqtt.ConnectResponse{ SessionPresent: false, - ReturnCode: mqtt.Accepted, + ReasonCode: mqtt.Success, }, }) }) diff --git a/try/http.go b/try/http.go index 84be72970..464da2888 100644 --- a/try/http.go +++ b/try/http.go @@ -179,14 +179,39 @@ func BodyMatch(regexp string) ResponseCondition { func BodyContainsData(expected map[string]interface{}) ResponseCondition { return func(t *testing.T, tr *TestResponse) { + var test func(expected, actual any) + test = func(expected any, actual any) { + if actual == nil { + assert.Fail(t, "expected a non-nil actual") + return + } + switch e := expected.(type) { + case map[string]any: + assert.IsType(t, e, actual) + m := actual.(map[string]any) + for k, v := range e { + assert.Contains(t, m, k) + if _, ok := m[k]; !ok { + continue + } + test(v, m[k]) + } + case []any: + assert.IsType(t, e, actual) + assert.Len(t, actual, len(e)) + for i, v := range e { + test(v, actual.([]any)[i]) + } + default: + assert.Equal(t, expected, actual) + } + } + body := tr.GetBody() var actual map[string]interface{} err := json.Unmarshal(body, &actual) assert.NoError(t, err) - for k, v := range expected { - assert.Contains(t, actual, k) - assert.Equal(t, v, actual[k]) - } + test(expected, actual) } } diff --git a/webui/e2e/mocks/common.yml b/webui/e2e/mocks/common.yml index 51ff1bc03..4c80595a9 100644 --- a/webui/e2e/mocks/common.yml +++ b/webui/e2e/mocks/common.yml @@ -9,7 +9,7 @@ components: type: array items: type: string - enum: ["http", "kafka", "mail", "ldap"] + enum: ["http", "kafka", "mail", "ldap", "mqtt"] search: type: object properties: @@ -28,9 +28,11 @@ components: $ref: '#/components/schemas/Contact' type: type: string - enum: ["http", "kafka", "mail", "ldap"] + enum: ["http", "kafka", "mail", "ldap", "mqtt"] metrics: - $ref: '#/components/schemas/Metrics' + type: object + status: + type: string Contact: type: object properties: @@ -108,4 +110,39 @@ components: value: type: string binary: - type: string \ No newline at end of file + type: string + Error: + type: object + properties: + message: + type: string + ConfigInfo: + type: object + properties: + id: + type: string + url: + type: string + provider: + type: string + time: + type: string + Config: + type: object + properties: + id: + type: string + url: + type: string + provider: + type: string + time: + type: string + refs: + type: array + items: + $ref: '#/components/schemas/ConfigInfo' + tags: + type: array + items: + type: string \ No newline at end of file diff --git a/webui/e2e/mocks/dashboard.yml b/webui/e2e/mocks/dashboard.yml index ca1ac5caf..09fb21e83 100644 --- a/webui/e2e/mocks/dashboard.yml +++ b/webui/e2e/mocks/dashboard.yml @@ -20,6 +20,12 @@ paths: get: summary: list of all services operationId: services + parameters: + - name: type + in: query + required: false + schema: + type: string responses: "200": description: the list of services @@ -46,6 +52,35 @@ paths: application/json: schema: $ref: 'http.yml#/components/schemas/HttpInfo' + /api/services/http/{name}/operations: + get: + summary: Information about the http service + operationId: httpOperations + parameters: + - name: name + in: path + required: true + schema: + type: string + - name: path + in: query + required: false + schema: + type: string + - name: method + in: query + required: false + schema: + type: string + responses: + "200": + description: http service + content: + application/json: + schema: + type: array + items: + $ref: 'http.yml#/components/schemas/Operation' /api/services/kafka/{name}: get: summary: Information about the kafka cluster @@ -63,6 +98,23 @@ paths: application/json: schema: $ref: 'kafka.yml#/components/schemas/Kafka' + /api/services/mqtt/{name}: + get: + summary: Information about the MQTT cluster + operationId: serviceMqtt + parameters: + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: name of MQTT cluster + content: + application/json: + schema: + $ref: 'mqtt.yml#/components/schemas/Mqtt' /api/services/mail/{name}: get: summary: Information about the mail servers @@ -287,6 +339,10 @@ paths: application/json: schema: $ref: 'kafka.yml#/components/schemas/Kafka' + /api/services/kafka/{cluster}/groups/{group}: + $ref: './kafka.yml#/components/pathItems/Group' + /api/services/kafka/{cluster}/clients/{clientId}: + $ref: './kafka.yml#/components/pathItems/Client' /api/events: get: summary: list of events @@ -331,7 +387,26 @@ paths: operationId: metrics parameters: - in: query - name: query + name: names + schema: + type: array + items: + type: string + responses: + "200": + description: a list of metrics + content: + application/json: + schema: + $ref: 'common.yml#/components/schemas/Metrics' + /api/metrics/{type}: + get: + summary: get list of metrics + operationId: metricsType + parameters: + - in: path + name: type + required: true schema: type: string responses: diff --git a/webui/e2e/mocks/http.yml b/webui/e2e/mocks/http.yml index 833886050..8458a9707 100644 --- a/webui/e2e/mocks/http.yml +++ b/webui/e2e/mocks/http.yml @@ -1,101 +1,146 @@ components: schemas: HttpInfo: - allOf: - - $ref: 'common.yml#/components/schemas/Service' - - type: object + type: object + properties: + name: + type: string + description: + type: string + version: + type: string + contact: + type: object properties: - servers: - type: array - items: - type: object - properties: - url: - type: string - description: - type: string - tags: - type: array - items: - type: object - properties: - name: - type: string - summary: - type: string - description: - type: string - parent: - type: string - kind: - type: string - paths: + name: + type: string + url: + type: string + format: uri + email: + type: string + format: email + servers: + type: array + items: + type: object + properties: + url: + type: string + description: + type: string + tags: + type: array + items: + type: object + properties: + name: + type: string + summary: + type: string + description: + type: string + parent: + type: string + kind: + type: string + paths: + type: array + items: + type: object + properties: + path: + type: string + summary: + type: string + description: + type: string + status: + type: string + errors: + type: array + items: + $ref: 'common.yml#/components/schemas/Error' + operations: + type: array + items: + type: object + properties: + method: + type: string + summary: + type: string + description: + type: string + operationId: + type: string + deprecated: + type: boolean + tags: + type: array + items: + type: string + status: + type: string + errors: + type: array + items: + $ref: 'common.yml#/components/schemas/Error' + metrics: + type: object + properties: + http_requests_total: + type: integer + http_requests_errors_total: + type: integer + request_timestamp: + type: integer + Operation: + type: object + properties: + requestBody: + type: object + properties: + description: + type: string + contents: type: array items: - type: object - properties: - path: - type: string - summary: - type: string - description: + $ref: '#/components/schemas/MediaType' + required: + type: boolean + parameters: + type: array + items: + $ref: '#/components/schemas/Parameter' + responses: + type: array + items: + type: object + properties: + statusCode: + type: [integer, string] + description: + type: string + headers: + type: array + items: + $ref: '#/components/schemas/Header' + contents: + $ref: '#/components/schemas/MediaTypes' + security: + type: array + items: + type: object + additionalProperties: + type: object + properties: + scopes: + type: array + items: type: string - operations: - type: array - items: - type: object - properties: - method: - type: string - summary: - type: string - description: - type: string - operationId: - type: string - requestBody: - type: object - properties: - description: - type: string - contents: - type: array - items: - $ref: '#/components/schemas/MediaType' - required: - type: boolean - parameters: - type: array - items: - $ref: '#/components/schemas/Parameter' - responses: - type: array - items: - type: object - properties: - statusCode: - type: [integer, string] - description: - type: string - headers: - type: array - items: - $ref: '#/components/schemas/Header' - contents: - $ref: '#/components/schemas/MediaTypes' - security: - type: array - items: - type: object - additionalProperties: - type: object - properties: - scopes: - type: array - items: - type: string - configs: - type: object + configs: + type: object MediaTypes: type: array items: diff --git a/webui/e2e/mocks/http_handler.js b/webui/e2e/mocks/http_handler.js index 8ba8008f9..2ae20cddc 100644 --- a/webui/e2e/mocks/http_handler.js +++ b/webui/e2e/mocks/http_handler.js @@ -1,9 +1,10 @@ import { on, sleep } from 'mokapi' -import { clusters, events as kafkaEvents, configs as kafkaConfigs } from 'kafka.js' -import { apps as httpServices, events as httpEvents, configs as httpConfigs } from 'services_http.js' -import { services as mailServices, mailEvents, getMail, getAttachment } from 'mail.js' -import { server as ldapServers, searches } from 'ldap.js' -import { metrics } from 'metrics.js' +import { clusters as kafkaClusters, events as kafkaEvents, configs as kafkaConfigs } from './kafka.ts' +import { apps as httpServices, events as httpEvents, configs as httpConfigs } from './services_http.js' +import { services as mailServices, mailEvents, getMail, getAttachment } from './mail.js' +import { server as ldapServers, searches } from './ldap.js' +import { clusters as mqttClusters, events as mqttEvents } from './mqtt.js' +import { metrics, sum, max } from './metrics.ts' import { get, post, fetch } from 'mokapi/http' import { base64 } from 'mokapi/encoding'; @@ -25,41 +26,159 @@ export default async function() { switch (request.operationId) { case 'info': { - response.data = { version: "0.11.0", activeServices: ["http", "kafka", "ldap", "mail"], search: { enabled: true } } - return true + response.data = { version: "0.11.0", activeServices: ["http", "kafka", "ldap", "mail", "mqtt"], search: { enabled: true } } + return } case 'services': - response.data = getServices() - return true + response.data = getServices(request.query['type']) + return case 'serviceHttp': const name = request.path['name'] const service = httpServices.find(x => x.name === name) if (service) { - response.data = service + response.data = { + name: service.name, + description: service.description, + version: service.version, + ...service.contact ? { contact: service.contact } : {}, + servers: service.servers, + paths: service.paths.map(p => { + return { + path: p.path, + ...p.summary ? { summary: p.summary } : {}, + ...p.description ? { description: p.description } : {}, + status: p.status, + ...p.errors ? { errors: p.errors } : {}, + operations: p.operations.map(o => { + return { + method: o.method, + ...o.summary ? {summary: o.summary} : {}, + ...o.description ? {description: o.description} : {}, + ...o.operationId ? { operationId: o.operationId } : {}, + deprecated: o.deprecated ?? false, + ...o.tags ? { tags: o.tags } : {}, + status: o.status, + ...o.errors ? { errors: o.errors } : {}, + metrics: { + http_requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + http_request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }) + } + } + }) + } + }), + } } else{ response.statusCode = 404 } - return true + return + case 'httpOperations': { + const name = request.path['name'] + const service = httpServices.find(x => x.name === name) + if (!service) { + response.statusCode = 404 + return + } + const method = request.query['method'] + const path = request.query['path'] + let operations = [] + for (const p of service.paths) { + if (!path || p.path === path) { + for (const o of p.operations) { + if (!method || o.method.toLowerCase() === method.toLowerCase()) { + operations.push({ + ...o, + metrics: { + http_requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + http_request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }) + } + }) + } + } + } + } + response.data = operations + return + } case 'serviceKafka': - response.data = clusters[0] - return true + response.data = { + name: kafkaClusters[0].name, + description: kafkaClusters[0].description, + version: kafkaClusters[0].version, + contact: kafkaClusters[0].contact, + servers: kafkaClusters[0].servers, + topics: kafkaClusters[0].topics.map(x => { + return { + name: x.name, + ...x.summary === undefined ? {} : { summary: x.summary }, + metrics: { + kafka_messages_total: metrics.find(m => m.name === `kafka_messages_total{service="Kafka World",topic="${x.name}"}`).value, + kafka_message_timestamp: metrics.find(m => m.name === `kafka_message_timestamp{service="Kafka World",topic="${x.name}"}`).value + } + } + }), + groups: kafkaClusters[0].groups.map(x => { + return { + name: x.name, + generation: x.generation, + state: x.state, + protocol: x.protocol, + members: x.members.length ?? 0, + metrics: x.metrics + } + }), + clients: kafkaClusters[0].clients.map(c => { + return { + clientId: c.clientId, + address: c.address, + software: c.software, + } + }), + configs: kafkaClusters[0].configs.map(c => { + return { + id: c.id, + url: c.url, + provider: c.provider, + time: c.time + } + }) + } + return + case 'kafkaGroup': + response.data = kafkaClusters[0].groups.find(x => x.name === request.path['group']) + return case 'serviceMail': response.data = mailServices[0] - return true - case 'clusters': - response.data = clusters.map(x => getInfo(x, 'kafka')) return - case 'topics': - response.data = clusters[0].topics + case 'kafkaClusters': + response.data = kafkaClusters.map(x => getInfo(x, 'kafka')) + return + case 'kafkaTopics': + response.data = kafkaClusters[0].topics return - case 'topic': - response.data = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaTopic': + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) + response.data = { + ...topic, + groups: kafkaClusters[0].groups.filter(g => g.topics.includes(topic.name)).map(x => { + return { + name: x.name, + generation: x.generation, + state: x.state, + protocol: x.protocol, + members: x.members.length ?? 0, + metrics: x.metrics + } + }), + } if (response.data === undefined) { response.statusCode = 404 } return - case 'produceTopic': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaProduceTopic': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 } else { @@ -74,8 +193,8 @@ export default async function() { } return } - case 'partitions': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaPartitions': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 return @@ -83,8 +202,8 @@ export default async function() { response.data = topic.partitions return } - case 'partition': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaPartition': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 return @@ -97,8 +216,8 @@ export default async function() { } return } - case 'producePartition': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaProducePartition': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 return @@ -118,8 +237,8 @@ export default async function() { } return } - case 'offsets': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaOffsets': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 return @@ -143,8 +262,8 @@ export default async function() { } return } - case 'offset': { - const topic = clusters[0].topics.find(x => x.name === request.path.topic) + case 'kafkaOffset': { + const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) if (!topic) { response.statusCode = 404 return @@ -172,9 +291,12 @@ export default async function() { } return } + case 'kafkaClient': + response.data = kafkaClusters[0].clients.find(x => x.clientId === request.path['clientId']) + return case 'mailboxes': response.data = mailServices[0].mailboxes - return true + return case 'mailbox': for (const mb of mailServices[0].mailboxes) { if (mb.name === request.path.mailbox) { @@ -182,11 +304,11 @@ export default async function() { ...mb, folders: mb.folders.map(x => x.name) } - return true + return } } response.statusCode = 404 - return true + return case 'smtpMessages': for (const mb of mailServices[0].mailboxes) { if (mb.name === request.path.mailbox) { @@ -195,11 +317,11 @@ export default async function() { result.push(...f.messages) } response.data = result - return true + return } } response.statusCode = 404 - return true + return case 'smtpMail': case 'smtpMessage': const messageId = request.path['messageId'] @@ -212,7 +334,7 @@ export default async function() { data: mail } } - return true + return case 'smtpMailAttachment': case 'smtpAttachment': const attachment = getAttachment(request.path['messageId'], request.path['name']) @@ -222,21 +344,25 @@ export default async function() { response.data = attachment.data response.headers['Content-Type'] = attachment.contentType } - return true + return case 'serviceLdap': response.data = ldapServers[0] - return true + return case 'metrics': - let q = request.query["query"] - if (!q) { + let names = request.query['names'] + if (!names) { response.data = metrics } else{ - response.data = metrics.filter(x => x.name.includes(q)) + response.data = metrics.filter(x => names.includes(x.name)) } - return true + return + case 'metricsType': + let type = request.path['type'] + response.data = metrics.filter(x => x.name.startsWith(type + '_')) + return case 'events': response.data = getEvents(request.query['traits']) - return true + return case 'event': let id = request.path["id"] let e = getEvent(id) @@ -245,13 +371,13 @@ export default async function() { }else{ response.data = e } - return true + return case 'example': { const res = post(`${apiBaseUrl}/api/schema/example`, request.body) response.statusCode = res.statusCode response.headers = res.headers response.body = res.body - return true + return } case 'validate': sleep(1000) @@ -265,35 +391,35 @@ export default async function() { response.headers = res.headers response.body = res.body response.data = null - return true + return case 'configs': response.data = getConfigs() - return true + return case 'config': const config = getConfig(request.path.id) if (config) { response.data = config - return true + return } else { response.statusCode = 404 response.data = '' - return true + return } case 'configData': const configData = configs[request.path.id] if (configData) { response.data = configData.data response.headers = configData.headers - return true + return } else { response.statusCode = 404 response.data = '' - return true + return } case 'fakerTree': const resp = get(`${apiBaseUrl}/mokapi/api/faker/tree`) response.body = resp.body - return true + return case 'search': switch (request.query.q) { case 'bad': @@ -400,13 +526,66 @@ export default async function() { ) } -function getServices() { - let http = httpServices.map(x => getInfo(x, 'http')) - let kafka = clusters.map(x => getInfo(x, 'kafka')) - let smtp = mailServices.map(x => getInfo(x, 'mail')) - let ldap = ldapServers.map(x => getInfo(x, 'ldap')) +function getServices(type) { + const services = [] + if (!type || type === 'http') { + services.push(...httpServices.map(x => { + return { + ...getInfo(x, 'http'), + metrics: { + http_requests_total: sum('http_requests_total', { name: 'service', value: x.name }), + http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: x.name }), + http_request_timestamp: max('http_request_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'kafka') { + services.push(...kafkaClusters.map(x => { + return { + ...getInfo(x, 'kafka'), + metrics: { + kafka_messages_total: sum('kafka_messages_total', { name: 'service', value: x.name }), + kafka_message_timestamp: max('kafka_message_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'mail') { + services.push(...mailServices.map(x => { + return { + ...getInfo(x, 'mail'), + metrics: { + mail_mails_total: sum('mail_mails_total', { name: 'service', value: x.name }), + mail_mail_timestamp: max('mail_mail_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'ldap') { + services.push(...ldapServers.map(x => { + return { + ...getInfo(x, 'ldap'), + metrics: { + ldap_requests_total: sum('ldap_requests_total', { name: 'service', value: x.name }), + ldap_request_timestamp: max('ldap_request_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'mqtt') { + services.push(...mqttClusters.map(x => { + return { + ...getInfo(x, 'mqtt'), + metrics: { + mqtt_messages_total: sum('mqtt_messages_total', { name: 'service', value: x.name }), + mqtt_message_timestamp: max('mqtt_message_timestamp', { name: 'service', value: x.name }) + } + } + })) + } - return http.concat(kafka).concat(smtp).concat(ldap) + return services } function getEvent(id) { @@ -419,6 +598,11 @@ function getEvent(id) { if (e.id === id){ return e } + } + for (const e of mqttEvents){ + if (e.id === id){ + return e + } } for (const e of mailEvents){ if (e.id === id){ @@ -450,6 +634,11 @@ function getEvents(traits) { result.push(e) } } + for (const e of mqttEvents){ + if (matchEvent(e, traits)) { + result.push(e) + } + } for (const e of mailEvents) { if (matchEvent(e, traits)) { result.push(e) @@ -503,8 +692,7 @@ function getInfo(config, type) { name: config.name, description: config.description, version: config.version, - metrics: config.metrics, - type: type + type: type, } if (config.contact) { info.contact = { @@ -513,6 +701,9 @@ function getInfo(config, type) { url: config.contact.url } } + if (config.status) { + info.status = config.status + } return info } @@ -548,29 +739,61 @@ function getSearchResults() { type: 'http', service: 'Swagger Petstore', path: '/pet', + methods: 'get,post' } }, { type: 'HTTP', domain: 'Swagger Petstore', - title: "POST /pet", + title: "/pet", fragments: ['Everything', 'store'], params: { type: 'http', service: 'Swagger Petstore', path: '/pet', - method: 'post' + method: 'post', + statusCode: '200' + } + }, + { + type: 'HTTP', + domain: 'Swagger Petstore', + title: "/foo/longTextPathEndpoint/bar/pets", + fragments: ['Everything', 'store'], + params: { + type: 'http', + service: 'Swagger Petstore', + path: '/foo/bar/pets', + method: 'put', + statusCode: '200' + } + }, + { + type: 'HTTP', + domain: 'Swagger Petstore', + title: "/customer/foo/bar/test/correspondenceAddress", + fragments: ['Everything', 'store'], + params: { + type: 'http', + service: 'Swagger Petstore', + path: '/customer/foo/bar/test/correspondenceAddress', + method: 'post', + statusCode: '200' } }, { type: 'Event', domain: 'Swagger Petstore', - title: "POST http://127.0.0.1:18080/pet", - fragments: [], + title: "http://127.0.0.1:18080/pet", + fragments: ['POST'], + time: '2025-05-23T08:49:25.482366+01:00', params: { type: 'event', - namespace: 'http', - id: '4242' + 'traits.namespace': 'http', + 'traits.name': 'Swagger Petstore', + 'traits.path': '/pet', + 'traits.method': 'POST', + id: '9e058601-562a-4063-a3b2-066ba624893a' } }, { @@ -589,7 +812,8 @@ function getSearchResults() { fragments: ['To ours significant why upon tomorrow'], params: { type: 'kafka', - service: 'Kafka World' + service: 'Kafka World', + topics: '3' } }, { @@ -605,13 +829,18 @@ function getSearchResults() { }, { type: 'Event', - domain: 'Kafka World - mokapi.shop.products', + domain: 'Kafka World', title: "GGOEWXXX0827", fragments: null, + time: '2025-05-01T08:49:25.482366+01:00', params: { type: 'event', - namespace: 'kafka', - id: '123456' + 'traits.namespace': 'kafka', + 'traits.name': 'Kafka World', + 'traits.topic': 'mokapi.shop.products', + 'traits.type': 'message', + 'traits.partition': '0', + id: '5d549ffe-97cd-477d-8db6-27ed4963c08d' } }, { @@ -628,38 +857,44 @@ function getSearchResults() { type: 'Event', domain: 'Mail Testserver', title: "A test mail", - fragments: ['message from Alice'], + fragments: ['test mail'], + time: '2025-05-01T08:49:25.482366+01:00', params: { type: 'event', - namespace: 'mail', - id: '8832' + 'traits.namespace': 'mail', + 'traits.name': 'Mail Testserver', + id: '273cd167-f5a5-4da1-969e-d44213686491' } }, { type: 'LDAP', - domain: 'LDAP Testserver', title: "LDAP Testserver", fragments: ['This is a sample LDAP server'], params: { type: 'ldap', service: 'LDAP Testserver', + entries: '12' } }, { type: 'Event', domain: 'LDAP Testserver', - title: "Search (objectClass=user)", + title: "(objectClass=user)", fragments: ['(objectClass=user)'], + time: '2025-05-01T08:49:25.482366+01:00', params: { type: 'event', - namespace: 'ldap', - id: 'dkads-23124' + 'traits.namespace': 'ldap', + 'traits.name': 'LDAP Testserver', + 'traits.operation': 'search', + id: 'dkads-23124', + baseDn: 'dc=example,dc=org', } } ], facets: { type: [{value: 'HTTP', count: 3}, {value: 'Kafka', count: 2}, {value: 'Mail', count: 1}, {value: 'Event', count: 2}, {value: 'Config', count: 1}] }, - total: 8 + total: 15 } } \ No newline at end of file diff --git a/webui/e2e/mocks/kafka.js b/webui/e2e/mocks/kafka.ts similarity index 74% rename from webui/e2e/mocks/kafka.js rename to webui/e2e/mocks/kafka.ts index f1f48a525..73840c59c 100644 --- a/webui/e2e/mocks/kafka.js +++ b/webui/e2e/mocks/kafka.ts @@ -1,4 +1,4 @@ -import { metrics } from 'metrics.js' +import { metrics, filter as filterMetrics, parseLabels } from './metrics.ts' import { base64 } from 'mokapi/encoding'; const Product = { @@ -103,7 +103,7 @@ export let clusters = [ topics: [ { name: 'mokapi.shop.products', - description: 'Though literature second anywhere fortnightly am this either so me.', + summary: 'Though literature second anywhere fortnightly am this either so me.', partitions: [ { id: 0, @@ -150,7 +150,7 @@ export let clusters = [ }, { name: 'mokapi.shop.userSignedUp', - description: 'This channel contains a message per each user who signs up in our application.', + summary: 'This channel contains a message per each user who signs up in our application.', partitions: [ { id: 0, @@ -209,52 +209,112 @@ export let clusters = [ groups: [ { name: 'foo', + generation: 0, members:[ { name: 'julie', + clientId: 'consumer-1', addr: '127.0.0.1:15001', - clientSoftwareName: 'mokapi', - clientSoftwareVersion: '1.0', + software: 'mokapi 1.0', heartbeat: 1654771269, partitions: { 'mokapi.shop.products': [ 0,1 ], 'mokapi.shop.userSignedUp': [ 0 ] } }, { name: 'hermann', + clientId: 'consumer-2', addr: '127.0.0.1:15002', - clientSoftwareName: 'mokapi', - clientSoftwareVersion: '1.0', + software: 'mokapi 1.0', heartbeat: 1654872269, partitions: { 'mokapi.shop.products': [ 2 ], 'mokapi.shop.userSignedUp': [ ] } } ], - coordinator: 'localhost:9092', leader: 'julie', state: 'Stable', protocol: 'Range', topics: [ 'mokapi.shop.products', 'mokapi.shop.userSignedUp' ], + metrics: { + kafka_rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="foo"}`)?.value || 0, + topics: { + 'mokapi.shop.products': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'foo'}, { name: 'topic', value: 'mokapi.shop.products'}).map(m => { + const labels = parseLabels(m) + return { + partition: labels.partition, + kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="${labels.partition}"}`)?.value || 0, + kafka_consumer_group_commit: m.value + } + }), + 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'foo'}, { name: 'topic', value: 'mokapi.shop.userSignedUp'}).map(m => { + const labels = parseLabels(m) + return { + partition: labels.partition, + kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, + kafka_consumer_group_commit: m.value + } + }) + } + } }, { name: 'bar', + generation: 0, members: [ { name: 'george', + clientId: 'consumer', addr: '127.0.0.1:15003', - clientSoftwareName: 'mokapi', - clientSoftwareVersion: '1.0', + software: 'mokapi 1.0', heartbeat: 1654721269, partitions: { 'mokapi.shop.userSignedUp': [ 0 ] } } ], - coordinator: 'localhost:9092', leader: 'george', state: 'Stable', protocol: 'Range', topics: [ 'mokapi.shop.userSignedUp' ], + metrics: { + kafka_rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="mokapi.shop.userSignedUp"}`)?.value || 0, + topics: { + 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'bar'}, { name: 'topic', value: 'mokapi.shop.userSignedUp'}).map(m => { + const labels = parseLabels(m) + return { + partition: labels.partition, + kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, + kafka_consumer_group_commit: m.value + } + }) + } + } }, ], metrics: metrics.filter(x => x.name.includes("kafka")), configs: [ configs[ 'b6fea8ac-56c7-4e73-a9c0-6337640bdca8' ] + ], + clients: [ + { + clientId: 'consumer-1', + address: '127.0.0.1:15001', + software: 'mokapi 1.0', + groups: [ + { memberId: 'julie', group: 'foo' } + ] + }, + { + clientId: 'consumer-2', + address: '127.0.0.1:15002', + software: 'mokapi 1.0', + groups: [ + { memberId: 'hermann', group: 'foo' } + ] + }, + { + clientId: 'consumer', + address: '127.0.0.1:15003', + software: 'mokapi 1.0', + groups: [ + { memberId: 'george', group: 'bar' } + ] + } ] } ] @@ -344,7 +404,7 @@ export let events = [ data: { offset: 1, key: { - value: 'TEST1' + binary: 'VEVTVDEgw6Qg8JCAgA==' }, message: { value: JSON.stringify( diff --git a/webui/e2e/mocks/kafka.yml b/webui/e2e/mocks/kafka.yml index ecd4d7ce9..c475d4d69 100644 --- a/webui/e2e/mocks/kafka.yml +++ b/webui/e2e/mocks/kafka.yml @@ -3,7 +3,7 @@ components: Clusters: get: summary: Returns a list of Kafka clusters. - operationId: clusters + operationId: kafkaClusters responses: '200': description: The list of Kafka clusters @@ -27,7 +27,7 @@ components: Topics: get: summary: Returns the list of topics that belongs to the specified Kafka cluster - operationId: topics + operationId: kafkaTopics parameters: - in: path name: cluster @@ -42,7 +42,7 @@ components: schema: type: array items: - $ref: '#/components/schemas/Topic' + $ref: '#/components/schemas/TopicInfo' Topic: parameters: - in: path @@ -57,7 +57,7 @@ components: type: string get: summary: Return the topic with the given topic name - operationId: topic + operationId: kafkaTopic responses: '200': description: The topic. @@ -67,7 +67,7 @@ components: $ref: '#/components/schemas/Topic' post: summary: Produce messages to a topic, the partition will be chosen randomly - operationId: produceTopic + operationId: kafkaProduceTopic requestBody: required: true content: @@ -102,7 +102,7 @@ components: Partitions: get: summary: Return the list of partitions that belong to the specified topic. - operationId: partitions + operationId: kafkaPartitions parameters: - in: path name: cluster @@ -142,7 +142,7 @@ components: type: integer get: summary: Return the partition with the given partition id. - operationId: partition + operationId: kafkaPartition responses: '200': description: The list of partitions. @@ -152,7 +152,7 @@ components: $ref: '#/components/schemas/Partition' post: summary: Produce records to specified partition of the topic. - operationId: producePartition + operationId: kafkaProducePartition requestBody: required: true content: @@ -203,7 +203,7 @@ components: type: integer get: summary: Return published records in partition. - operationId: offsets + operationId: kafkaOffsets parameters: - in: query name: offset @@ -254,7 +254,7 @@ components: type: integer get: summary: Return published record in partition with the specified offset. - operationId: offset + operationId: kafkaOffset responses: '200': description: The record. @@ -270,17 +270,73 @@ components: type: array items: $ref: '#/components/schemas/Record' + Group: + parameters: + - in: path + name: cluster + required: true + schema: + type: string + - in: path + name: group + required: true + schema: + type: string + get: + summary: Return the group with the given group id. + operationId: kafkaGroup + responses: + '200': + description: The group details. + content: + application/json: + schema: + $ref: '#/components/schemas/Group' + Client: + parameters: + - in: path + name: cluster + required: true + schema: + type: string + - in: path + name: clientId + required: true + schema: + type: string + get: + summary: Return the client with the given client id. + operationId: kafkaClient + responses: + '200': + description: The client details. + content: + application/json: + schema: + $ref: '#/components/schemas/Client' schemas: KafkaInfo: - allOf: - - $ref: 'common.yml#/components/schemas/Service' - - type: object + type: object + properties: + name: + type: string + description: + type: string + version: + type: string + contact: + type: object properties: - topics: - type: array - items: - type: string + name: + type: string + url: + type: string + format: uri + email: + type: string + format: email + Kafka: type: object properties: @@ -320,19 +376,32 @@ components: topics: type: array items: - $ref: '#/components/schemas/Topic' + $ref: '#/components/schemas/TopicInfo' groups: type: array items: - $ref: '#/components/schemas/Group' - metrics: - $ref: 'common.yml#/components/schemas/Metrics' + $ref: '#/components/schemas/GroupInfo' + configs: + type: array + items: + $ref: './common.yml#/components/schemas/ConfigInfo' + TopicInfo: + type: object + properties: + name: + type: string + summary: + type: string + lastMessageReceived: + type: integer + messages: + type: integer Topic: type: object properties: name: type: string - description: + summary: type: string partitions: type: array @@ -359,6 +428,30 @@ components: $ref: '#/components/schemas/Broker' segments: type: integer + GroupInfo: + type: object + properties: + name: + type: string + generation: + type: integer + state: + type: string + protocol: + type: string + members: + type: integer + metrics: + type: object + properties: + kafka_rebalance_timestamp: + type: integer + topics: + type: object + additionalProperties: + type: array + items: + type: object Group: type: object properties: @@ -382,6 +475,17 @@ components: type: array items: type: string + metrics: + type: object + properties: + kafka_rebalance_timestamp: + type: integer + topics: + type: object + additionalProperties: + type: array + items: + type: object Member: type: object properties: @@ -389,9 +493,7 @@ components: type: string addr: type: string - clientSoftwareName: - type: string - clientSoftwareVersion: + software: type: string heartbeat: type: integer @@ -401,6 +503,26 @@ components: type: array items: type: integer + Client: + type: object + properties: + clientId: + type: string + address: + type: string + software: + type: string + brokerAddress: + type: string + groups: + type: array + items: + type: object + properties: + memberId: + type: string + group: + type: string Broker: type: object properties: diff --git a/webui/e2e/mocks/ldap.js b/webui/e2e/mocks/ldap.js index 8f259b511..ba2556fe7 100644 --- a/webui/e2e/mocks/ldap.js +++ b/webui/e2e/mocks/ldap.js @@ -1,13 +1,9 @@ -import { metrics } from "./metrics.js" -import { SearchScope, ResultCode } from "mokapi/ldap" - export let server = [ { name: "LDAP Testserver", description: "This is a sample LDAP server", version: "1.0", server: "0.0.0.0:389", - metrics: metrics.filter(x => x.name.startsWith("ldap")) } ] diff --git a/webui/e2e/mocks/mail.js b/webui/e2e/mocks/mail.js index 9a9d15d5c..31caf47a4 100644 --- a/webui/e2e/mocks/mail.js +++ b/webui/e2e/mocks/mail.js @@ -1,5 +1,3 @@ -import {metrics} from "./metrics.js"; - export let mails = [ { from: [{name: 'Alice', address: 'alice@mokapi.io'}], @@ -98,7 +96,6 @@ export let services = [ {name: 'spam', recipient: '.*@foo.bar', subject: 'spam', body: 'spam', action: 'deny'} ], settings: { maxRecipients: 0, autoCreateMailbox: true }, - metrics: metrics.filter(x => x.name.startsWith("mail")) } ] diff --git a/webui/e2e/mocks/metrics.js b/webui/e2e/mocks/metrics.js deleted file mode 100644 index 6959147dc..000000000 --- a/webui/e2e/mocks/metrics.js +++ /dev/null @@ -1,77 +0,0 @@ -const app_start = new Date() -app_start.setHours(new Date().getHours() - 2) - -export let metrics = [ - { - name: 'app_start_timestamp', - value: 1652025690 //app_start.getTime() / 1000,// 1652025690, - }, - { - name: 'app_memory_usage_bytes', - value: 52450600 - }, - { - name: 'app_job_run_total', - value: 1 - }, - { - name: 'http_requests_total{service="Swagger Petstore",endpoint="/pet",method="POST"}"', - value: 2 - }, - { - name: 'http_requests_errors_total{service="Swagger Petstore",endpoint="/pet",method="POST"}"', - value: 1 - }, - { - name: 'http_requests_total{service="Swagger Petstore",endpoint="/pet/findByStatus",method="GET"}"', - value: 2 - }, - { - name: 'http_requests_errors_total{service="Swagger Petstore",endpoint="/pet/findByStatus",method="POST"}"', - value: 0 - }, - { - name: 'http_request_timestamp{service="Swagger Petstore",endpoint="/pet",method="POST"}"', - value: 1652235690 - }, - { - name: 'http_request_timestamp{service="Swagger Petstore",endpoint="/pet/findByStatus",method="POST"}"', - value: 1652237690 - }, - { - name: 'kafka_messages_total{service="Kafka World",topic="mokapi.shop.products"}"', - value: 10 - }, - { - name: 'kafka_message_timestamp{service="Kafka World",topic="mokapi.shop.products"}"', - value: 1652135690 - }, - { - name: 'kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="0"}"', - value: 10 - }, - { - name: 'kafka_messages_total{service="Kafka World",topic="bar"}"', - value: 1 - }, - { - name: 'kafka_message_timestamp{service="Kafka World",topic="bar"}"', - value: 1652035690 - }, - { - name: 'mail_mails_total{service="Smtp Testserver"}"', - value: 3 - }, - { - name: 'mail_mail_timestamp{service="Smtp Testserver"}"', - value: 1652635690 - }, - { - name: 'ldap_requests_total{service="LDAP Testserver"}"', - value: 6 - }, - { - name: 'ldap_request_timestamp{service="LDAP Testserver"}"', - value: 1652635690 - }, -] \ No newline at end of file diff --git a/webui/e2e/mocks/metrics.ts b/webui/e2e/mocks/metrics.ts new file mode 100644 index 000000000..148de49b3 --- /dev/null +++ b/webui/e2e/mocks/metrics.ts @@ -0,0 +1,199 @@ +const app_start = new Date() +app_start.setHours(new Date().getHours() - 2) + +export let metrics = [ + { + name: 'app_start_timestamp', + value: 1652025690 //app_start.getTime() / 1000,// 1652025690, + }, + { + name: 'app_memory_usage_bytes', + value: 52450600 + }, + { + name: 'app_job_run_total', + value: 1 + }, + { + name: 'http_requests_total{service="Swagger Petstore",endpoint="/pet",method="POST"}', + value: 2 + }, + { + name: 'http_requests_errors_total{service="Swagger Petstore",endpoint="/pet",method="POST"}', + value: 1 + }, + { + name: 'http_requests_total{service="Swagger Petstore",endpoint="/pet/findByStatus",method="GET"}', + value: 2 + }, + { + name: 'http_requests_errors_total{service="Swagger Petstore",endpoint="/pet/findByStatus",method="POST"}', + value: 0 + }, + { + name: 'http_request_timestamp{service="Swagger Petstore",endpoint="/pet",method="POST"}', + value: 1652235690 + }, + { + name: 'http_request_timestamp{service="Swagger Petstore",endpoint="/pet/findByStatus",method="POST"}', + value: 1652237690 + }, + { + name: 'kafka_messages_total{service="Kafka World",topic="mokapi.shop.products"}', + value: 10 + }, + { + name: 'kafka_message_timestamp{service="Kafka World",topic="mokapi.shop.products"}', + value: 1652135690 + }, + { + name: 'kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="0"}', + value: 10 + }, + { + name: 'kafka_consumer_group_commit{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="0"}', + value: 1 + }, + { + name: 'kafka_rebalance_timestamp{service="Kafka World",group="foo"}', + value: 1652135690 + }, + { + name: 'kafka_messages_total{service="Kafka World",topic="bar"}', + value: 1 + }, + { + name: 'kafka_message_timestamp{service="Kafka World",topic="bar"}', + value: 0 + }, + { + name: 'kafka_messages_total{service="Kafka World",topic="mokapi.shop.userSignedUp"}', + value: 0 + }, + { + name: 'kafka_message_timestamp{service="Kafka World",topic="mokapi.shop.userSignedUp"}', + value: 0 + }, + { + name: 'kafka_messages_total{service="Kafka World",topic="mokapi.shop.avro"}', + value: 0 + }, + { + name: 'kafka_message_timestamp{service="Kafka World",topic="mokapi.shop.avro"}', + value: 1652035690 + }, + { + name: 'mail_mails_total{service="Mail Testserver"}', + value: 3 + }, + { + name: 'mail_mail_timestamp{service="Mail Testserver"}', + value: 1652635690 + }, + { + name: 'ldap_requests_total{service="LDAP Testserver"}', + value: 6 + }, + { + name: 'ldap_request_timestamp{service="LDAP Testserver"}', + value: 1652635690 + }, + // mqtt + { + name: 'mqtt_messages_total{service="MQTT Temperature Sensor API",topic="home/livingroom/temperature"}', + value: 1 + }, + { + name: 'mqtt_message_timestamp{service="MQTT Temperature Sensor API",topic="home/livingroom/temperature"}', + value: 1672182690 + }, + { + name: 'mqtt_messages_total{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data"}', + value: 1 + }, + { + name: 'mqtt_message_timestamp{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data"}', + value: 1771058965 + }, +] + +export function sum(name, ...labels){ + if (!metrics){ + return 0 + } + + let sum = 0 + for (let metric of metrics) { + if (!metric.name.startsWith(name)) { + continue + } + + if (labels.length == 0 || matchLabels(metric, labels)){ + sum += Number(metric.value) + } + } + return sum +} + +export function max(name, ...labels) { + if (!metrics){ + return 0 + } + + let max = 0 + for (let metric of metrics) { + if (!metric.name.startsWith(name)) { + continue + } + + if (labels.length == 0 || matchLabels(metric, labels)){ + const n = Number(metric.value) + if (n > max) { + max = n + } + } + } + return max +} + +export function filter(name, ...labels) { + const result = []; + if (!metrics){ + return []; + } + + for (let metric of metrics) { + if (!metric.name.startsWith(name)) { + continue + } + + if (labels.length == 0 || matchLabels(metric, labels)){ + result.push(metric) + } + } + return result; +} + +export function parseLabels(metric) { + const match = metric.name.match(/\{(.+)}/); + if (!match || match.length < 2 || !match[1]) { + return {}; + } + + return Object.fromEntries( + Array.from( + match[1].matchAll(/(\w+)="([^"]*)"/g), + m => [m[1], m[2]] + ) + ); +} + +function matchLabels(metric, labels){ + for (var label of labels){ + const s = `${label.name}="${label.value}"` + if (!metric.name.includes(s)){ + return false + } + } + return true +} diff --git a/webui/e2e/mocks/mqtt.js b/webui/e2e/mocks/mqtt.js new file mode 100644 index 000000000..2c723199d --- /dev/null +++ b/webui/e2e/mocks/mqtt.js @@ -0,0 +1,238 @@ +import { metrics } from 'metrics.ts'; +import { on } from 'mokapi'; + +const Measurement = { + type: 'object', + properties: { + sensorId: { type: 'string' }, + temperature: { type: 'number', format: 'float' }, + unit: { type: 'string', enum: ['celsius', 'fahrenheit'] }, + timestamp: { type: 'string', format: 'date-time' }, + }, +}; + +export let clusters = [ + { + name: 'MQTT Temperature Sensor API', + description: + 'API for an MQTT-based temperature sensor. The sensor publishes measurement data and receives configuration commands.', + version: '1.0.0', + contact: { + name: 'mokapi', + url: 'https://www.mokapi.io', + email: 'info@mokapi.io', + }, + servers: [ + { + name: 'Broker', + host: 'localhost:1883', + tags: [ + { + name: 'env:test', + description: 'This environment is for running internal tests', + }, + ], + description: 'Development server', + }, + ], + topics: [ + { + name: 'home/livingroom/temperature', + summary: 'Channel for messages FROM the sensor (Publish)', + messages: { + TemperatureMeasurement: { + name: 'TemperatureMeasurement', + title: 'Temperature Measurement', + payload: { schema: Measurement }, + contentType: 'application/json', + }, + }, + bindings: { + qos: 1, + retain: false, + }, + }, + { + name: 'sensors/{sensorId}/data', + messages: { + TemperaturePayload: { + name: 'TemperaturePayload', + title: 'Temperature Payload', + payload: { + schema: { + type: 'object', + properties: { + temp: { + type: 'number' + }, + timestamp: { + type: 'string', + format: 'date-time' + } + } + } + }, + contentType: 'application/json', + }, + }, + instances: [ + { + name: 'sensors/123/data', + parameters: { + sensorId: '123' + } + } + ] + }, + ], + metrics: metrics.filter((x) => x.name.includes('mqtt')), + clients: [ + { clientId: 'mqtt-client-1', address: '127.0.0.1:83374', brokerAddress: 'localhost:1883', protocolVersion: 4 }, + { clientId: 'mqtt-client-2', address: '127.0.0.1:83374', brokerAddress: 'localhost:1883', protocolVersion: 5 } + ] + }, +]; + +export default async function () { + on('http', function (request, response) { + switch (request.operationId) { + case 'serviceMqtt': + response.data = { + name: clusters[0].name, + description: clusters[0].description, + version: clusters[0].version, + contact: clusters[0].contact, + servers: clusters[0].servers, + topics: clusters[0].topics.map(x => { + return { + name: x.name, + ...x.title ? { title: x.title } : {}, + ...x.summary ? { summary: x.summary } : {}, + ...x.description ? { description: x.description } : {}, + messages: x.messages, + instances: x.instances, + metrics: { + mqtt_messages_total: metrics.find(m => m.name === `mqtt_messages_total{service="${clusters[0].name}",topic="${x.name}"}`).value, + mqtt_message_timestamp: metrics.find(m => m.name === `mqtt_message_timestamp{service="${clusters[0].name}",topic="${x.name}"}`).value + } + } + }), + clients: clusters[0].clients + } + } + }); +} + +export let events = [ + { + id: '9f8bd24d-1480-4d2d-a1b4-f022aeddeaf8', + traits: { + namespace: 'mqtt', + name: 'MQTT Temperature Sensor API', + type: 'message', + topic: 'home/livingroom/temperature', + clientId: 'mqtt-client-1', + }, + time: '2026-02-13T09:49:25.482366+01:00', + data: { + topic: 'home/livingroom/temperature', + messageId: 'TemperatureMeasurement', + message: { + value: JSON.stringify({ + sensorId: '12345', + temperature: 30.0, + unit: 'celsius', + timestamp: '2026-02-13T09:49:25.482366+01:00' + }), + }, + clientId: 'mqtt-client-1', + }, + }, + { + id: '0d39cc83-2eda-4b36-9645-107186ae401d', + traits: { + namespace: 'mqtt', + name: 'MQTT Temperature Sensor API', + type: 'message', + topic: 'sensors/{sensorId}/data', + sensorId: '123', + clientId: 'mqtt-client-2', + }, + time: '2026-02-14T09:49:25.482366+01:00', + data: { + topic: 'sensors/123/data', + messageId: 'TemperaturePayload', + message: { + value: JSON.stringify({ + temp: 33, + timestamp: '2026-02-14T09:49:25.482366+01:00' + }), + }, + clientId: 'mqtt-client-2', + }, + }, + { + id: '93834ead-6c61-47eb-8068-c269fb3f606b', + traits: { + namespace: 'mqtt', + name: 'MQTT Temperature Sensor API', + type: 'request', + clientId: 'mqtt-client-2', + }, + time: '2026-02-14T09:49:25.482366+01:00', + data: { + type: 1, + request: { + version: 5, + cleanSession: true, + keepAlive: 60, + message: { + qos: 1, + retain: true, + topic: 'sensors/123/data', + message: '{"temp":33,"timestamp":"2026-02-14T09:49:25.482366+01:00}"}' + } + }, + response: { + sessionPresent: false, + reasonCode: { reason: 'Success', code: 0 } + } + }, + }, + { + id: '8897df7c-c3bb-4e57-8bbe-ef5a1f71d978', + traits: { + namespace: 'mqtt', + name: 'MQTT Temperature Sensor API', + type: 'request', + clientId: 'mqtt-client-2', + }, + time: '2026-02-14T09:49:25.482366+01:00', + data: { + type: 8, + request: { + messageId: 1234, + topics: [ { name: 'sensors/123/data', qos: 1 } ] + }, + response: { + reasonCodes: [ 1 ] + } + }, + }, + { + id: '469a585e-9254-4c05-bbb6-3369fde147a9', + traits: { + namespace: 'mqtt', + name: 'MQTT Temperature Sensor API', + type: 'request', + clientId: 'mqtt-client-2', + }, + time: '2026-02-14T09:49:25.482366+01:00', + data: { + type: 14, + request: { + reason: 0 + } + }, + } +]; diff --git a/webui/e2e/mocks/mqtt.yml b/webui/e2e/mocks/mqtt.yml new file mode 100644 index 000000000..e266906b6 --- /dev/null +++ b/webui/e2e/mocks/mqtt.yml @@ -0,0 +1,114 @@ +components: + schemas: + MqttInfo: + type: object + properties: + name: + type: string + description: + type: string + version: + type: string + contact: + type: object + properties: + name: + type: string + url: + type: string + format: uri + email: + type: string + format: email + + Mqtt: + type: object + properties: + name: + type: string + description: + type: string + version: + type: string + contact: + type: object + properties: + name: + type: string + url: + type: string + format: uri + email: + type: string + format: email + servers: + type: array + items: + type: object + properties: + name: + type: string + host: + type: string + description: + type: string + tags: + $ref: './kafka.yml#/components/schemas/Tags' + configs: + additionalProperties: + type: ['number', 'string'] + topics: + type: array + items: + $ref: '#/components/schemas/MqttTopic' + metrics: + $ref: 'common.yml#/components/schemas/Metrics' + MqttTopic: + type: object + properties: + name: + type: string + description: + type: string + messages: + type: object + additionalProperties: + $ref: '#/components/schemas/MqttMessageConfig' + bindings: + $ref: '#/components/schemas/MqttTopicBindings' + tags: + $ref: './kafka.yml#/components/schemas/Tags' + MqttMessageConfig: + type: object + properties: + name: + type: string + title: + type: string + summary: + type: string + description: + type: string + key: + $ref: 'schema.yml#/components/schemas/Schema' + payload: + $ref: 'schema.yml#/components/schemas/Schema' + header: + $ref: 'schema.yml#/components/schemas/Schema' + contentType: + type: string + MqttTopicBindings: + partitions: + type: integer + retentionBytes: + type: integer + retentionMs: + type: integer + segmentBytes: + type: integer + segmentMs: + type: integer + valueSchemaValidation: + type: boolean + keySchemaValidation: + type: boolean \ No newline at end of file diff --git a/webui/e2e/mocks/services_http.js b/webui/e2e/mocks/services_http.js index e8065dbfb..e69630042 100644 --- a/webui/e2e/mocks/services_http.js +++ b/webui/e2e/mocks/services_http.js @@ -142,10 +142,12 @@ export let apps = [ description: "Server is mocked by *mokapi*" } ], + status: 'valid', paths: [ { path: "/pet", summary: "Everything about your Pets", + status: 'valid', operations: [ { method: "post", @@ -153,6 +155,7 @@ export let apps = [ operationId: "addPet", deprecated: true, tags: ["pet", "post"], + status: 'valid', parameters: [ { type: "query", @@ -268,11 +271,13 @@ export let apps = [ }, { path: "/pet/{petId}", + status: 'valid', operations: [ { method: "post", summary: "Updates a pet in the store with form data", tags: ["pet", "post"], + status: 'valid', parameters: [ { name: "petId", @@ -342,6 +347,7 @@ export let apps = [ method: "get", summary: "Returns a single pet", tags: ["pet"], + status: 'valid', parameters: [ { name: "petId", @@ -373,6 +379,7 @@ export let apps = [ }, { path: "/pet/findByStatus", + status: 'valid', operations: [ { method: "get", @@ -380,6 +387,7 @@ export let apps = [ description: "Multiple status values **can** be provided with comma separated strings", operationId: "findPetsByStatus", tags: ["pet"], + status: 'valid', parameters: [ { name: "status", @@ -463,11 +471,13 @@ export let apps = [ { path: "/Zorem/ipsum/dolor/sit/amet/consetetur/sadipscing/elitr/sed/diam/nonumy/eirmod", summary: "A long path example", + status: 'valid', operations: [ { method: "post", summary: "A long path example", tags: ["custom", "post"], + status: 'valid', requestBody: { description: "Create a new pet in the store", contents: [ @@ -534,14 +544,17 @@ export let apps = [ url: "https://api.example.com/v1", } ], + status: 'invalid', paths: [ { path: "/books", + status: 'valid', operations: [ { method: "get", summary: "Get books from the store", operationId: "listBooks", + status: 'valid', responses: [ { statusCode: 200, @@ -562,6 +575,7 @@ export let apps = [ method: "post", summary: "Add a new book", operationId: "addBook", + status: 'valid', requestBody: { required: true, contents: [ @@ -586,6 +600,19 @@ export let apps = [ } ] }, + { + path: "/users", + status: 'valid', + operations: [ + { + method: "get", + summary: "Get users from the store", + operationId: "listUsers", + status: 'invalid', + errors: [{ message: "An example error message for this operation" }], + } + ] + } ] } ] diff --git a/webui/e2e/tests/components/kafka.ts b/webui/e2e/tests/components/kafka.ts index 13aa410a3..c6d10c6ef 100644 --- a/webui/e2e/tests/components/kafka.ts +++ b/webui/e2e/tests/components/kafka.ts @@ -4,19 +4,19 @@ import { formatDateTime } from "../helpers/format" export interface Topic { name: string - description: string + summary: string lastMessage: string messages: string } export function useKafkaTopics(table: Locator) { - const topics = useTable(table, ['Name', 'Description', 'Last Message', 'Messages']) + const topics = useTable(table, ['Name', 'Summary', 'Last Message', 'Messages']) return { async testTopic(row: number, topic: Topic) { await test.step(`Check Kafka topic in row ${row}`, async () => { const t = topics.getRow(row + 1) await expect(t.getCellByName('Name')).toHaveText(topic.name) - await expect(t.getCellByName('Description')).toHaveText(topic.description) + await expect(t.getCellByName('Summary')).toHaveText(topic.summary) await expect(t.getCellByName('Last Message')).toHaveText(topic.lastMessage) await expect(t.getCellByName('Messages')).toHaveText(topic.messages) }) @@ -81,9 +81,9 @@ export function useKafkaMessages(page: Page) { return { test: async (table: Locator, withTopic: boolean = true) => { await test.step('Check messages log', async () => { - let columns = ['Key', 'Value', 'Topic', 'Time'] + let columns = ['Topic', 'Key', 'Value', 'Time'] if (!withTopic) { - columns.splice(2,1) + columns.splice(0,1) } const messages = await useTable(table, columns) diff --git a/webui/e2e/tests/dashboard-demo/dashboard.spec.ts b/webui/e2e/tests/dashboard-demo/dashboard.spec.ts index 9b7a0df39..9a8ba1bcc 100644 --- a/webui/e2e/tests/dashboard-demo/dashboard.spec.ts +++ b/webui/e2e/tests/dashboard-demo/dashboard.spec.ts @@ -5,8 +5,12 @@ test.use({ colorScheme: 'light' }) // reset storage state test.use({ storageState: { cookies: [], origins: [] } }); -test('Visit Dashboard Demo Overview', async ({ page }) => { - await page.goto('/dashboard-demo'); +test('Visit Dashboard Demo Overview', async ({ page, baseURL }) => { + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + await page.goto('/dashboard-demo') + } await test.step('Verify app metrics', async () => { const main = page.locator('main') diff --git a/webui/e2e/tests/dashboard-demo/kafka.spec.ts b/webui/e2e/tests/dashboard-demo/kafka.spec.ts index f2e647b88..f71dff7a5 100644 --- a/webui/e2e/tests/dashboard-demo/kafka.spec.ts +++ b/webui/e2e/tests/dashboard-demo/kafka.spec.ts @@ -5,9 +5,15 @@ test.use({ colorScheme: 'light' }) // reset storage state test.use({ storageState: { cookies: [], origins: [] } }); -test('Visit Kafka Order Service', async ({ page }) => { +test('Visit Kafka Order Service', async ({ page, baseURL }) => { + let dashboard = '/dashboard' + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + dashboard = '/dashboard-demo' + await page.goto('/dashboard-demo') + } - await page.goto('/dashboard-demo'); await page.getByText('Kafka Order Service API').click(); await test.step('Verify service info', async () => { @@ -38,7 +44,7 @@ test('Visit Kafka Order Service', async ({ page }) => { const rows = table.locator('tbody tr'); await expect(rows).toHaveCount(2); await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('order-topic'); - await expect(await getCellByColumnName(table, 'Description', rows.nth(0))).toHaveText('The Kafka topic for order events.'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText('The Kafka topic for order events.'); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toHaveText('-'); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('2'); @@ -147,7 +153,7 @@ test('Visit Kafka Order Service', async ({ page }) => { await expect(info.getByLabel('Topic', { exact: true })).toHaveText('order-topic'); await expect(info.getByLabel('Cluster')).toHaveText('Kafka Order Service API'); await expect(info.getByLabel('Cluster')).toHaveAttribute('href'); - await expect(info.getByLabel('Description')).toHaveText('The Kafka topic for order events.'); + await expect(info.getByLabel('Summary')).toHaveText('The Kafka topic for order events.'); await expect(info.getByLabel('Type of API')).toHaveText('Kafka'); @@ -157,7 +163,7 @@ test('Visit Kafka Order Service', async ({ page }) => { const meta = page.getByRole('region', { name: 'Meta' }); await expect(meta.getByLabel('Kafka Key')).toHaveText('a914817b-c5f0-433e-8280-1cd2fe44234e'); await expect(meta.getByLabel('Kafka Topic')).toHaveText('order-topic'); - await expect(meta.getByLabel('Kafka Topic')).toHaveAttribute('href', '/dashboard-demo/kafka/service/Kafka%20Order%20Service%20API/topics/order-topic'); + await expect(meta.getByLabel('Kafka Topic')).toHaveAttribute('href', dashboard + '/kafka/service/Kafka%20Order%20Service%20API/topics/order-topic'); await expect(meta.getByLabel('Offset')).toHaveText('1'); await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); await expect(meta.getByLabel('Key Type')).toHaveText('-'); @@ -165,7 +171,7 @@ test('Visit Kafka Order Service', async ({ page }) => { await expect(meta.getByLabel('Client')).toHaveText('producer-1'); const value = page.getByRole('region', { name: 'Value' }); - await expect(value.getByLabel('Content Type')).toHaveText('application/json'); + await expect(value.getByLabel('Content Type', { exact: true })).toHaveText('application/json'); await expect(value.getByLabel('Lines of Code')).toHaveText('8 lines'); await expect(value.getByLabel('Size of Code')).toHaveText('249 B'); await expect(value.getByLabel('Content', { exact: true })).toContainText('"orderId": "a914817b-c5f0-433e-8280-1cd2fe44234e",') @@ -189,7 +195,7 @@ test('Visit Kafka Order Service', async ({ page }) => { const meta = page.getByRole('region', { name: 'Meta' }); await expect(meta.getByLabel('Kafka Key')).toHaveText('random-message-1'); await expect(meta.getByLabel('Kafka Topic')).toHaveText('order-topic'); - await expect(meta.getByLabel('Kafka Topic')).toHaveAttribute('href', '/dashboard-demo/kafka/service/Kafka%20Order%20Service%20API/topics/order-topic'); + await expect(meta.getByLabel('Kafka Topic')).toHaveAttribute('href', dashboard + '/kafka/service/Kafka%20Order%20Service%20API/topics/order-topic'); await expect(meta.getByLabel('Offset')).toHaveText('0'); await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); await expect(meta.getByLabel('Key Type')).toHaveText('-'); diff --git a/webui/e2e/tests/dashboard-demo/ldap.spec.ts b/webui/e2e/tests/dashboard-demo/ldap.spec.ts index f04edbd1f..b10f18732 100644 --- a/webui/e2e/tests/dashboard-demo/ldap.spec.ts +++ b/webui/e2e/tests/dashboard-demo/ldap.spec.ts @@ -5,9 +5,13 @@ test.use({ colorScheme: 'light' }) // reset storage state test.use({ storageState: { cookies: [], origins: [] } }); -test('Visit LDAP Testserver', async ({ page }) => { +test('Visit LDAP Testserver', async ({ page, baseURL }) => { + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + await page.goto('/dashboard-demo') + } - await page.goto('/dashboard-demo'); await page.getByRole('cell').getByText('HR Employee Directory').click(); await test.step('Verify service info', async () => { diff --git a/webui/e2e/tests/dashboard-demo/mail.spec.ts b/webui/e2e/tests/dashboard-demo/mail.spec.ts index 8d9a7467c..f709a62b5 100644 --- a/webui/e2e/tests/dashboard-demo/mail.spec.ts +++ b/webui/e2e/tests/dashboard-demo/mail.spec.ts @@ -5,9 +5,15 @@ test.use({ colorScheme: 'light' }) // reset storage state test.use({ storageState: { cookies: [], origins: [] } }); -test('Visit Mail Server', async ({ page }) => { +test('Visit Mail Server', async ({ page, baseURL }) => { + let dashboard = '/dashboard' + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + dashboard = '/dashboard-demo' + await page.goto('/dashboard-demo') + } - await page.goto('/dashboard-demo'); await page.getByRole('cell').getByText(/^Mail Server$/).click(); await test.step('Verify service info', async () => { @@ -69,7 +75,7 @@ test('Visit Mail Server', async ({ page }) => { await rows.nth(1).click(); await expect(page.getByLabel('Mailbox Name')).toHaveText('bob.miller@example.com'); await expect(page.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', '/dashboard-demo/mail/service/Mail%20Server'); + await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(page.getByLabel('Username')).toHaveText('bob.miller'); await expect(page.getByLabel('Password')).toHaveText('mysecretpassword123'); @@ -88,7 +94,7 @@ test('Visit Mail Server', async ({ page }) => { const info = page.getByRole('region', { name: 'Info' }) await expect(info.getByLabel('Subject')).toHaveText('Reset Your Password'); await expect(info.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(info.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', '/dashboard-demo/mail/service/Mail%20Server'); + await expect(info.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(info.getByLabel('From')).not.toBeEmpty(); await expect(info.getByLabel('From')).toHaveText('zzz@example.com'); await expect(info.getByLabel('To', { exact: true })).toHaveText('Bob Miller '); @@ -112,7 +118,7 @@ test('Visit Mail Server', async ({ page }) => { await expect(page.getByLabel('Mailbox Name')).toHaveText('alice.johnson@example.com'); await expect(page.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', '/dashboard-demo/mail/service/Mail%20Server'); + await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(page.getByLabel('Username')).toHaveText('alice.johnson'); await expect(page.getByLabel('Password')).toHaveText('anothersecretpassword456'); @@ -131,7 +137,7 @@ test('Visit Mail Server', async ({ page }) => { const info = page.getByRole('region', { name: 'Info' }) await expect(info.getByLabel('Subject')).toHaveText('Check Out Our New Arrivals!'); await expect(info.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(info.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', '/dashboard-demo/mail/service/Mail%20Server'); + await expect(info.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(info.getByLabel('From')).not.toBeEmpty(); await expect(info.getByLabel('From')).toHaveText('Bob Miller '); await expect(info.getByLabel('To', { exact: true })).toHaveText('Alice Johnson '); @@ -142,10 +148,12 @@ test('Visit Mail Server', async ({ page }) => { const attachments = page.getByRole('region', { name: 'Attachments '}); await expect(attachments).toBeVisible(); - await expect(attachments.getByRole('link', { name: 'headerimg' })).toHaveAttribute('href', /\/demo\/header.jpg$/) - await expect(attachments.getByRole('link', { name: 'product1' })).toHaveAttribute('href', /\/demo\/product1.jpg$/) - await expect(attachments.getByRole('link', { name: 'product2' })).toHaveAttribute('href', /\/demo\/product2.jpg$/) - await expect(attachments.getByRole('link', { name: 'product3' })).toHaveAttribute('href', /\/demo\/product3.jpg$/) + if (dashboard === 'dashboard-demo') { + await expect(attachments.getByRole('link', { name: 'headerimg' })).toHaveAttribute('href', /\/demo\/header.jpg$/) + await expect(attachments.getByRole('link', { name: 'product1' })).toHaveAttribute('href', /\/demo\/product1.jpg$/) + await expect(attachments.getByRole('link', { name: 'product2' })).toHaveAttribute('href', /\/demo\/product2.jpg$/) + await expect(attachments.getByRole('link', { name: 'product3' })).toHaveAttribute('href', /\/demo\/product3.jpg$/) + } await expect(page.getByLabel('Content-Type')).toHaveText('text/html'); await expect(page.getByLabel('Encoding')).not.toBeVisible(); diff --git a/webui/e2e/tests/dashboard-demo/mqtt.spec.ts b/webui/e2e/tests/dashboard-demo/mqtt.spec.ts new file mode 100644 index 000000000..7b3ad8ef0 --- /dev/null +++ b/webui/e2e/tests/dashboard-demo/mqtt.spec.ts @@ -0,0 +1,114 @@ +import { test, expect } from '../models/fixture-dashboard' +import { getCellByColumnName } from '../helpers/table' + +test('Visit MQTT overview', async ({ page, baseURL }) => { + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + await page.goto('/dashboard-demo') + } + + await test.step('Verify Dashboard', async () => { + + await expect(page.getByLabel('MQTT Messages')).toHaveText('1') + + const table = page.getByRole('table', { name: 'MQTT Clusters' }); + const rows = table.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('Smart Home MQTT API'); + + await page.getByRole('link', { name: 'MQTT', exact: true }).click() + await expect(page.getByRole('table', { name: 'Recent Messages' })).toBeVisible() + + }) + + await test.step('Visit Smart Home MQTT API', async () => { + + await page.getByText('Smart Home MQTT API').click(); + + const region = page.getByRole('region', { name: 'Info' }); + await expect(region.getByLabel('Name')).toHaveText('Smart Home MQTT API') + await expect(region.getByLabel('Description')).toHaveText('Example specification for controlling sensors via MQTT.') + + const table = page.getByRole('table', { name: 'Topics' }); + const rows = table.locator('tbody tr'); + await expect(rows).toHaveCount(1); + + await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('sensors/{sensorId}/data'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText(''); + await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); + await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('1'); + + await test.step('Visit sensors/{sensorId}/data', async () => { + const topics = page.getByRole('table', { name: 'Topics' }); + await topics.getByText('sensors/{sensorId}/data').click(); + + await expect(page.getByLabel('Topic', { exact: true })).toHaveText('sensors/{sensorId}/data'); + await expect(page.getByLabel('Description')).not.toBeVisible(); + await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(messages, 'Topic', rows.nth(0))).toHaveText('sensors/12345/data'); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toContainText('{"temp":24,"timestamp":"'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); + + await test.step('Visit message', async () => { + + await rows.nth(0).click(); + + const meta = page.getByRole('region', { name: 'Meta' }) + await expect(meta.getByLabel('Topic')).toHaveText('sensors/12345/data'); + await expect(meta.getByLabel('Time')).not.toBeEmpty(); + await expect(meta.getByLabel('Client')).toContainText('mqttjs'); + await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); + await expect(meta.getByLabel('Service Type')).toHaveText('MQTT'); + + const value = page.getByRole('region', { name: 'Value' }); + await expect(value.getByLabel('Content Type')).toHaveText('application/json'); + await expect(value.getByLabel('Lines of Code')).toHaveText('4 lines'); + await expect(value.getByLabel('Size of Code')).toHaveText('67 B'); + await expect(value.getByLabel('Content', { exact: true })).toContainText('"temp": 24,'); + + await test.step('Visit client', async () => { + + await meta.getByLabel('Client').getByRole('link').click(); + + const info = page.getByRole('region', { name: 'Info' }) + await expect(info.getByLabel('Client Id')).toContainText('mqttjs'); + await expect(info.getByLabel('Address')).not.toBeEmpty(); + await expect(info.getByLabel('Broker')).toContainText(':1883'); + await expect(info.getByLabel('Protocol Version')).toHaveText('4 (v3.1.1)'); + await expect(info.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + + await rows.nth(0).click() + + }) + + await meta.getByLabel('Topic').click(); + await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + }) + + }) + + await test.step('Verify Clients', async () => { + + await page.getByRole('tab', { name: 'Clients' }).click(); + + const clients = page.getByRole('table', { name: 'Clients' }); + const rows = clients.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(clients, 'Client Id', rows.nth(0))).toContainText('mqttjs'); + await expect(await getCellByColumnName(clients, 'Address', rows.nth(0))).not.toBeEmpty(); + await expect(await getCellByColumnName(clients, 'Protocol Version', rows.nth(0))).toHaveText('4 (v3.1.1)'); + + }) + + }) + +}) \ No newline at end of file diff --git a/webui/e2e/tests/dashboard-demo/petstore.spec.ts b/webui/e2e/tests/dashboard-demo/petstore.spec.ts index 23d7b8e0e..791386c77 100644 --- a/webui/e2e/tests/dashboard-demo/petstore.spec.ts +++ b/webui/e2e/tests/dashboard-demo/petstore.spec.ts @@ -5,9 +5,15 @@ test.use({ colorScheme: 'light' }) // reset storage state test.use({ storageState: { cookies: [], origins: [] } }); -test('Visit Petstore Demo', async ({ page }) => { +test('Visit Petstore Demo', async ({ page, baseURL }) => { + let dashboard = '/dashboard' + if (baseURL === 'http://localhost:8080') { + await page.goto('/dashboard') + } else { + dashboard = '/dashboard-demo' + await page.goto('/dashboard-demo') + } - await page.goto('/dashboard-demo'); await page.getByText('Swagger Petstore').click(); await test.step('Verify service info', async () => { @@ -119,7 +125,7 @@ test('Visit Petstore Demo', async ({ page }) => { await expect(page.getByLabel('Path', { exact: true })).toHaveText('/pet/{petId}') await expect(page.getByLabel('Service', { exact: true })).toHaveText('Swagger Petstore') - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', '/dashboard-demo/http/services/Swagger%20Petstore') + await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/http/services/Swagger%20Petstore') const region = page.getByRole('region', { name: 'Methods' }); await expect(region).toBeVisible(); diff --git a/webui/e2e/tests/dashboard/http/books.spec.ts b/webui/e2e/tests/dashboard/http/books.spec.ts index a0e897d15..96cc8dd38 100644 --- a/webui/e2e/tests/dashboard/http/books.spec.ts +++ b/webui/e2e/tests/dashboard/http/books.spec.ts @@ -7,7 +7,10 @@ test.describe('Visit Books API', () => { test('Verify overview', async ({ dashboard, page }) => { await dashboard.open() - await page.getByRole('link', { name: 'Books API' }).click(); + const service = page.getByRole('link', { name: 'Books API' }) + await expect(service.getByRole('img')).toBeVisible(); + + await service.click(); await test.step('Verify service info', async () => { @@ -19,6 +22,14 @@ test.describe('Visit Books API', () => { }); + await test.step('Verify errors', async () => { + const region = page.getByRole('region', { name: 'Errors' }); + await expect(region).toBeVisible(); + const table = page.getByRole('table', { name: 'Errors' }); + const rows = table.locator('tbody tr'); + await expect(await getCellByColumnName(table, 'Message', rows.nth(0))).toHaveText('An example error message for this operation'); + }) + await test.step('Verify servers', async () => { await page.getByRole('tab', { name: 'Servers' }).click(); @@ -48,13 +59,21 @@ test.describe('Visit Books API', () => { const table = page.getByRole('table', { name: 'Paths' }); const rows = table.locator('tbody tr'); - await expect(rows).toHaveCount(1); + await expect(rows).toHaveCount(2); await expect(await getCellByColumnName(table, 'Path', rows.nth(0))).toHaveText('/books'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText(''); await expect(await getCellByColumnName(table, 'Operations', rows.nth(0))).toHaveText('GET POST'); await expect(await getCellByColumnName(table, 'Last Request', rows.nth(0))).toHaveText('-'); await expect(await getCellByColumnName(table, 'Req / Err', rows.nth(0))).toHaveText('0 / 0'); + await expect(await getCellByColumnName(table, 'Path', rows.nth(1))).toHaveText('/users'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText('Get users from the store'); + await expect(await getCellByColumnName(table, 'Operations', rows.nth(1))).toHaveText('GET'); + const operations = await getCellByColumnName(table, 'Operations', rows.nth(1)) + await expect(operations.getByTitle('An example error message for this operation')).toHaveCSS('border-color', 'rgb(220, 53, 69)'); + await expect(await getCellByColumnName(table, 'Last Request', rows.nth(1))).toHaveText('-'); + await expect(await getCellByColumnName(table, 'Req / Err', rows.nth(1))).toHaveText('0 / 0'); + await test.step('Verify path', async () => { await page.getByRole('link', { name: '/books' }).click(); diff --git a/webui/e2e/tests/dashboard/kafka/cluster.spec.ts b/webui/e2e/tests/dashboard/kafka/cluster.spec.ts index e746fd550..0fb348c13 100644 --- a/webui/e2e/tests/dashboard/kafka/cluster.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/cluster.spec.ts @@ -90,8 +90,8 @@ test('Visit Kafka cluster config file', async ({ page, context }) => { const { test: testSourceView } = useSourceView(dashboard.getByRole('region', { name: 'Content' })) await testSourceView({ - lines: '342 lines', - size: '8.94 kB', + lines: '112 lines', + size: '2.85 kB', content: /"name": "Kafka World"/, filename: 'asyncapi.json', clipboard: '"name": "Kafka World"' diff --git a/webui/e2e/tests/dashboard/kafka/cluster.ts b/webui/e2e/tests/dashboard/kafka/cluster.ts index 60ae6e487..c39a6d822 100644 --- a/webui/e2e/tests/dashboard/kafka/cluster.ts +++ b/webui/e2e/tests/dashboard/kafka/cluster.ts @@ -17,7 +17,9 @@ export const cluster = { topics: [ { name: 'mokapi.shop.products', - description: 'Though literature second anywhere fortnightly am this either so me.', + title: '', + summary: 'Though literature second anywhere fortnightly am this either so me.', + description: '', lastMessage: formatTimestamp(1652135690), messages: '10', partitions: [ @@ -56,7 +58,9 @@ export const cluster = { }, { name: 'mokapi.shop.userSignedUp', - description: 'This channel contains a message per each user who signs up in our application.', + title: '', + summary: 'This channel contains a message per each user who signs up in our application.', + description: '', lastMessage: '-', messages: '0', partitions: [ @@ -70,6 +74,7 @@ export const cluster = { messageConfigs: [ { name: 'second', + title: '', summary: '', description: '', contentType: 'application/json', diff --git a/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts b/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts index 8d903c818..b75db8875 100644 --- a/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts @@ -28,7 +28,15 @@ test('Visit Kafka topic mokapi.shop.products', async ({ page, context }) => { await expect(info.getByLabel('Topic')).toHaveText(topic.name) await expect(info.getByLabel('Cluster')).toHaveText(cluster.name) await expect(info.getByLabel('Type of API')).toHaveText('Kafka') - await expect(info.getByLabel('Description')).toHaveText(topic.description) + if (topic.title) { + await expect(info.getByLabel('Title')).toHaveText(topic.title) + } + if (topic.summary) { + await expect(info.getByLabel('Summary')).toHaveText(topic.summary) + } + if (topic.description) { + await expect(info.getByLabel('Description')).toHaveText(topic.description) + } }) await useKafkaMessages(page).test(page.getByRole('table', { name: 'Recent Messages' }), false) diff --git a/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts b/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts index 16d67f8c0..3c0f2e2ec 100644 --- a/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts @@ -25,7 +25,7 @@ test('Visit Kafka topic mokapi.shop.userSignedUp', async ({ page, context }) => await expect(info.getByLabel('Topic')).toHaveText(topic.name) await expect(info.getByLabel('Cluster')).toHaveText(cluster.name) await expect(info.getByLabel('Type of API')).toHaveText('Kafka') - await expect(info.getByLabel('Description')).toHaveText(topic.description) + await expect(info.getByLabel('Summary')).toHaveText(topic.summary) }) const tabList = page.getByRole('region', { name: 'Topic Data' }).getByRole('tablist') diff --git a/webui/e2e/tests/dashboard/mqtt/overview.spec.ts b/webui/e2e/tests/dashboard/mqtt/overview.spec.ts new file mode 100644 index 000000000..c521e7d25 --- /dev/null +++ b/webui/e2e/tests/dashboard/mqtt/overview.spec.ts @@ -0,0 +1,176 @@ +import { test, expect } from '../../models/fixture-dashboard' +import { getCellByColumnName } from '../../helpers/table' + +test('Visit MQTT overview', async ({ page }) => { + await page.goto('/dashboard') + + await test.step('Verify Dashboard', async () => { + + await expect(page.getByLabel('MQTT Messages')).toHaveText('2') + + const table = page.getByRole('table', { name: 'MQTT Clusters' }); + const rows = table.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('MQTT Temperature Sensor API'); + + await page.getByRole('link', { name: 'MQTT', exact: true }).click() + await expect(page.getByRole('table', { name: 'Recent Messages' })).toBeVisible() + + }) + + await test.step('Visit MQTT Temperature Sensor API', async () => { + + await page.getByText('MQTT Temperature Sensor API').click(); + + const region = page.getByRole('region', { name: 'Info' }); + await expect(region.getByLabel('Name')).toHaveText('MQTT Temperature Sensor API') + await expect(region.getByLabel('Description')).toHaveText('API for an MQTT-based temperature sensor. The sensor publishes measurement data and receives configuration commands.') + + const table = page.getByRole('table', { name: 'Topics' }); + const rows = table.locator('tbody tr'); + await expect(rows).toHaveCount(2); + await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('home/livingroom/temperature'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText('Channel for messages FROM the sensor (Publish)'); + await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); + await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('1'); + + await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('sensors/{sensorId}/data'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText(''); + await expect(await getCellByColumnName(table, 'Last Message', rows.nth(1))).not.toBeEmpty(); + await expect(await getCellByColumnName(table, 'Messages', rows.nth(1))).toHaveText('1'); + + await test.step('Visit home/livingroom/temperature', async () => { + const topics = page.getByRole('table', { name: 'Topics' }); + await topics.getByText('home/livingroom/temperature').click(); + + await expect(page.getByLabel('Topic', { exact: true })).toHaveText('home/livingroom/temperature'); + await expect(page.getByLabel('Summary')).toHaveText('Channel for messages FROM the sensor (Publish)'); + await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toHaveText('{"sensorId":"12345","temperature":30,"unit":"celsius","timestamp":"2026-02-13T09:49:25.482366+01:00"}'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); + + await test.step('Visit message', async () => { + + await rows.nth(0).click(); + + const meta = page.getByRole('region', { name: 'Meta' }) + await expect(meta.getByLabel('Topic')).toHaveText('home/livingroom/temperature'); + await expect(meta.getByLabel('Time')).not.toBeEmpty(); + await expect(meta.getByLabel('Client')).toHaveText('mqtt-client-1'); + await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); + await expect(meta.getByLabel('Service Type')).toHaveText('MQTT'); + + const value = page.getByRole('region', { name: 'Value' }); + await expect(value.getByLabel('Content Type')).toHaveText('application/json'); + await expect(value.getByLabel('Lines of Code')).toHaveText('6 lines'); + await expect(value.getByLabel('Size of Code')).toHaveText('118 B'); + await expect(value.getByLabel('Content', { exact: true })).toContainText('"sensorId": "12345",'); + + await test.step('Visit client mqtt-client-1', async () => { + + await meta.getByLabel('Client').getByRole('link').click(); + + const info = page.getByRole('region', { name: 'Info' }) + await expect(info.getByLabel('Client Id')).toHaveText('mqtt-client-1'); + await expect(info.getByLabel('Address')).toHaveText('127.0.0.1:83374'); + await expect(info.getByLabel('Broker')).toHaveText('localhost:1883'); + await expect(info.getByLabel('Protocol Version')).toHaveText('4 (v3.1.1)'); + await expect(info.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + + await rows.nth(0).click() + + }) + + await meta.getByLabel('Topic').click(); + await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + }) + + }) + + await test.step('Visit sensors/{sensorId}/data', async () => { + const topics = page.getByRole('table', { name: 'Topics' }); + await topics.getByText('sensors/{sensorId}/data').click(); + + await expect(page.getByLabel('Topic', { exact: true })).toHaveText('sensors/{sensorId}/data'); + await expect(page.getByLabel('Title')).not.toBeVisible(); + await expect(page.getByLabel('Summary')).not.toBeVisible(); + await expect(page.getByLabel('Description')).not.toBeVisible(); + await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(messages, 'Topic', rows.nth(0))).toHaveText('sensors/123/data'); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toHaveText('{"temp":33,"timestamp":"2026-02-14T09:49:25.482366+01:00"}'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); + + await test.step('Visit message', async () => { + + await rows.nth(0).click(); + + const meta = page.getByRole('region', { name: 'Meta' }) + await expect(meta.getByLabel('Topic')).toHaveText('sensors/123/data'); + await expect(meta.getByLabel('Time')).not.toBeEmpty(); + await expect(meta.getByLabel('Client')).toHaveText('mqtt-client-2'); + await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); + await expect(meta.getByLabel('Service Type')).toHaveText('MQTT'); + + const value = page.getByRole('region', { name: 'Value' }); + await expect(value.getByLabel('Content Type')).toHaveText('application/json'); + await expect(value.getByLabel('Lines of Code')).toHaveText('4 lines'); + await expect(value.getByLabel('Size of Code')).toHaveText('67 B'); + await expect(value.getByLabel('Content', { exact: true })).toContainText('"temp": 33,'); + + await test.step('Visit client mqtt-client-2', async () => { + + await meta.getByLabel('Client').getByRole('link').click(); + + const info = page.getByRole('region', { name: 'Info' }) + await expect(info.getByLabel('Client Id')).toHaveText('mqtt-client-2'); + await expect(info.getByLabel('Address')).toHaveText('127.0.0.1:83374'); + await expect(info.getByLabel('Broker')).toHaveText('localhost:1883'); + await expect(info.getByLabel('Protocol Version')).toHaveText('5 (v5)'); + await expect(info.getByLabel('Type of API')).toHaveText('MQTT'); + + const messages = page.getByRole('table', { name: 'Messages' }); + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(1); + + await rows.nth(0).click() + + }) + + await meta.getByLabel('Topic').click(); + await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + }) + + }) + + await test.step('Verify Clients', async () => { + + await page.getByRole('tab', { name: 'Clients' }).click(); + + const clients = page.getByRole('table', { name: 'Clients' }); + const rows = clients.locator('tbody tr'); + await expect(rows).toHaveCount(2); + await expect(await getCellByColumnName(clients, 'Client Id', rows.nth(0))).toHaveText('mqtt-client-1'); + await expect(await getCellByColumnName(clients, 'Address', rows.nth(0))).toHaveText('127.0.0.1:83374'); + await expect(await getCellByColumnName(clients, 'Protocol Version', rows.nth(0))).toHaveText('4 (v3.1.1)'); + + await expect(await getCellByColumnName(clients, 'Client Id', rows.nth(1))).toHaveText('mqtt-client-2'); + await expect(await getCellByColumnName(clients, 'Address', rows.nth(1))).toHaveText('127.0.0.1:83374'); + await expect(await getCellByColumnName(clients, 'Protocol Version', rows.nth(1))).toHaveText('5 (v5)'); + + }) + + }) + +}) \ No newline at end of file diff --git a/webui/package-lock.json b/webui/package-lock.json index 9c1b2fca3..7d35162a0 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -30,10 +30,11 @@ "markdown-it": "^14.1.1", "markdown-it-container": "^4.0.0", "mime-types": "^3.0.2", + "mqtt": "^5.15.1", "ncp": "^2.0.0", - "nodemailer": "^8.0.5", - "vue": "^3.5.32", - "vue-router": "^5.0.4", + "nodemailer": "^8.0.7", + "vue": "^3.5.34", + "vue-router": "^5.0.6", "vue3-ace-editor": "^2.2.4", "whatwg-mimetype": "^5.0.0", "xml-formatter": "^3.7.0" @@ -43,18 +44,18 @@ "@rushstack/eslint-patch": "^1.16.1", "@types/js-yaml": "^4.0.9", "@types/markdown-it-container": "^4.0.0", - "@types/node": "^25.6.0", + "@types/node": "^25.6.2", "@vitejs/plugin-vue": "^6.0.6", "@vue/eslint-config-prettier": "^10.2.0", "@vue/eslint-config-typescript": "^14.7.0", "@vue/tsconfig": "^0.9.1", - "eslint": "^10.2.0", - "eslint-plugin-vue": "^10.8.0", + "eslint": "^10.2.1", + "eslint-plugin-vue": "^10.9.1", "npm-run-all": "^4.1.5", "prettier": "^3.8.3", "typescript": "~6.0.3", - "vite": "^8.0.8", - "vue-tsc": "^3.2.6", + "vite": "^8.0.11", + "vue-tsc": "^3.2.8", "xml2js": "^0.6.2" } }, @@ -93,9 +94,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", + "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", "license": "MIT", "dependencies": { "@babel/types": "^7.29.0" @@ -107,6 +108,15 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/types": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", @@ -121,9 +131,9 @@ } }, "node_modules/@emnapi/core": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", - "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", "dev": true, "license": "MIT", "optional": true, @@ -133,9 +143,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", - "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", "dev": true, "license": "MIT", "optional": true, @@ -346,9 +356,9 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz", - "integrity": "sha512-xK9sGVbJWYb08+mTJt3/YV24WxvxpXcXtP6B172paPZ+Ts69Re9dAr7lKwJoeIx8OoeuimEiRZ7umkiUVClmmQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", "dev": true, "license": "MIT", "optional": true, @@ -400,9 +410,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.124.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.124.0.tgz", - "integrity": "sha512-VBFWMTBvHxS11Z5Lvlr3IWgrwhMTXV+Md+EQF0Xf60+wAdsGFTBx7X7K/hP4pi8N7dcm1RvcHwDxZ16Qx8keUg==", + "version": "0.128.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.128.0.tgz", + "integrity": "sha512-huv1Y/LzBJkBVHt3OlC7u0zHBW9qXf1FdD7sGmc1rXc2P1mTwHssYv7jyGx5KAACSCH+9B3Bhn6Z9luHRvf7pQ==", "dev": true, "license": "MIT", "funding": { @@ -449,9 +459,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-YYe6aWruPZDtHNpwu7+qAHEMbQ/yRl6atqb/AhznLTnD3UY99Q1jE7ihLSahNWkF4EqRPVC4SiR4O0UkLK02tA==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.18.tgz", + "integrity": "sha512-lIDyUAfD7U3+BWKzdxMbJcsYHuqXqmGz40aeRqvuAm3y5TkJSYTBW2RDrn65DJFPQqVjUAUqq5uz8urzQ8aBdQ==", "cpu": [ "arm64" ], @@ -466,9 +476,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-oArR/ig8wNTPYsXL+Mzhs0oxhxfuHRfG7Ikw7jXsw8mYOtk71W0OkF2VEVh699pdmzjPQsTjlD1JIOoHkLP1Fg==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.18.tgz", + "integrity": "sha512-apJq2ktnGp27nSInMR5Vcj8kY6xJzDAvfdIFlpDcAK/w4cDO58qVoi1YQsES/SKiFNge/6e4CUzgjfHduYqWpQ==", "cpu": [ "arm64" ], @@ -483,9 +493,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.15.tgz", - "integrity": "sha512-YzeVqOqjPYvUbJSWJ4EDL8ahbmsIXQpgL3JVipmN+MX0XnXMeWomLN3Fb+nwCmP/jfyqte5I3XRSm7OfQrbyxw==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.18.tgz", + "integrity": "sha512-5Ofot8xbs+pxRHJqm9/9N/4sTQOvdrwEsmPE9pdLEEoAbdZtG6F2LMDfO1sp6ZAtXJuJV/21ew2srq3W8NXB5g==", "cpu": [ "x64" ], @@ -500,9 +510,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.15.tgz", - "integrity": "sha512-9Erhx956jeQ0nNTyif1+QWAXDRD38ZNjr//bSHrt6wDwB+QkAfl2q6Mn1k6OBPerznjRmbM10lgRb1Pli4xZPw==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.18.tgz", + "integrity": "sha512-7h8eeOTT1eyqJyx64BFCnWZpNm486hGWt2sqeLLgDxA0xI1oGZ9H7gK1S85uNGmBhkdPwa/6reTxfFFKvIsebw==", "cpu": [ "x64" ], @@ -517,9 +527,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.15.tgz", - "integrity": "sha512-cVwk0w8QbZJGTnP/AHQBs5yNwmpgGYStL88t4UIaqcvYJWBfS0s3oqVLZPwsPU6M0zlW4GqjP0Zq5MnAGwFeGA==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.18.tgz", + "integrity": "sha512-eRcm/HVt9U/JFu5RKAEKwGQYtDCKWLiaH6wOnsSEp6NMBb/3Os8LgHZlNyzMpFVNmiiMFlfb2zEnebfzJrHFmg==", "cpu": [ "arm" ], @@ -534,9 +544,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-eBZ/u8iAK9SoHGanqe/jrPnY0JvBN6iXbVOsbO38mbz+ZJsaobExAm1Iu+rxa4S1l2FjG0qEZn4Rc6X8n+9M+w==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.18.tgz", + "integrity": "sha512-SOrT/cT4ukTmgnrEz/Hg3m7LBnuCLW9psDeMKrimRWY4I8DmnO7Lco8W2vtqPmMkbVu8iJ+g4GFLVLLOVjJ9DQ==", "cpu": [ "arm64" ], @@ -551,9 +561,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.15.tgz", - "integrity": "sha512-ZvRYMGrAklV9PEkgt4LQM6MjQX2P58HPAuecwYObY2DhS2t35R0I810bKi0wmaYORt6m/2Sm+Z+nFgb0WhXNcQ==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.18.tgz", + "integrity": "sha512-QWjdxN1HJCpBTAcZ5N5F7wju3gVPzRzSpmGzx7na0c/1qpN9CFil+xt+l9lV/1M6/gqHSNXCiqPfwhVJPeLnug==", "cpu": [ "arm64" ], @@ -568,9 +578,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-VDpgGBzgfg5hLg+uBpCLoFG5kVvEyafmfxGUV0UHLcL5irxAK7PKNeC2MwClgk6ZAiNhmo9FLhRYgvMmedLtnQ==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.18.tgz", + "integrity": "sha512-ugCOyj7a4d9h3q9B+wXmf6g3a68UsjGh6dob5DHevHGMwDUbhsYNbSPxJsENcIttJZ9jv7qGM2UesLw5jqIhdg==", "cpu": [ "ppc64" ], @@ -585,9 +595,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-y1uXY3qQWCzcPgRJATPSOUP4tCemh4uBdY7e3EZbVwCJTY3gLJWnQABgeUetvED+bt1FQ01OeZwvhLS2bpNrAQ==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.18.tgz", + "integrity": "sha512-kKWRhbsotpXkGbcd5dllUWg5gEXcDAa8u5YnP9AV5DYNbvJHGzzuwv7dpmhc8NqKMJldl0a+x76IHbspEpEmdA==", "cpu": [ "s390x" ], @@ -602,9 +612,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.15.tgz", - "integrity": "sha512-023bTPBod7J3Y/4fzAN6QtpkSABR0rigtrwaP+qSEabUh5zf6ELr9Nc7GujaROuPY3uwdSIXWrvhn1KxOvurWA==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.18.tgz", + "integrity": "sha512-uCo8ElcCIAMyYAZyuIZ81oFkhTSIllNvUCHCAlbhlN4ji3uC28h7IIdlXyIvGO7HsuqnV9p3rD/bpH7XhIyhRw==", "cpu": [ "x64" ], @@ -619,9 +629,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.15.tgz", - "integrity": "sha512-witB2O0/hU4CgfOOKUoeFgQ4GktPi1eEbAhaLAIpgD6+ZnhcPkUtPsoKKHRzmOoWPZue46IThdSgdo4XneOLYw==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.18.tgz", + "integrity": "sha512-XNOQZtuE6yUIvx4rwGemwh8kpL1xvU41FXy/s9K7T/3JVcqGzo3NfKM2HrbrGgfPYGFW42f07Wk++aOC6B9NWA==", "cpu": [ "x64" ], @@ -636,9 +646,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.15.tgz", - "integrity": "sha512-UCL68NJ0Ud5zRipXZE9dF5PmirzJE4E4BCIOOssEnM7wLDsxjc6Qb0sGDxTNRTP53I6MZpygyCpY8Aa8sPfKPg==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.18.tgz", + "integrity": "sha512-tSn/kzrfa7tNOXr7sEacDBN4YsIqTyLqh45IO0nHDwtpKIDNDJr+VFojt+4klSpChxB29JLyduSsE0MKEwa65A==", "cpu": [ "arm64" ], @@ -653,9 +663,9 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.15.tgz", - "integrity": "sha512-ApLruZq/ig+nhaE7OJm4lDjayUnOHVUa77zGeqnqZ9pn0ovdVbbNPerVibLXDmWeUZXjIYIT8V3xkT58Rm9u5Q==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.18.tgz", + "integrity": "sha512-+J9YGmc+czgqlhYmwun3S3O0FIZhsH8ep2456xwjAdIOmuJxM7xz4P4PtrxU+Bz17a/5bqPA8o3HAAoX0teUdg==", "cpu": [ "wasm32" ], @@ -663,18 +673,18 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.9.2", - "@emnapi/runtime": "1.9.2", - "@napi-rs/wasm-runtime": "^1.1.3" + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" }, "engines": { - "node": ">=14.0.0" + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.15.tgz", - "integrity": "sha512-KmoUoU7HnN+Si5YWJigfTws1jz1bKBYDQKdbLspz0UaqjjFkddHsqorgiW1mxcAj88lYUE6NC/zJNwT+SloqtA==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.18.tgz", + "integrity": "sha512-zsu47DgU0FQzSwi6sU9dZoEdUv7pc1AptSEz/Z8HBg54sV0Pbs3N0+CrIbTsgiu6EyoaNN9CHboqbLaz9lhOyQ==", "cpu": [ "arm64" ], @@ -689,9 +699,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.15.tgz", - "integrity": "sha512-3P2A8L+x75qavWLe/Dll3EYBJLQmtkJN8rfh+U/eR3MqMgL/h98PhYI+JFfXuDPgPeCB7iZAKiqii5vqOvnA0g==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.18.tgz", + "integrity": "sha512-7H+3yqGgmnlDTRRhw/xpYY9J1kf4GC681nVc4GqKhExZTDrVVrV2tsOR9kso0fvgBdcTCcQShx4SLLoHgaLwhg==", "cpu": [ "x64" ], @@ -749,9 +759,9 @@ } }, "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", "dev": true, "license": "MIT", "optional": true, @@ -838,9 +848,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.6.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", - "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "version": "25.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz", + "integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==", "license": "MIT", "dependencies": { "undici-types": "~7.19.0" @@ -855,12 +865,30 @@ "@types/node": "*" } }, + "node_modules/@types/readable-stream": { + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.23.tgz", + "integrity": "sha512-wwXrtQvbMHxCbBgjHaMGEmImFTQxxpfMOR/ZoQnXxB1woqkUbdLGFDgauo00Py9IudiaqSeiBiulSV9i6XIPig==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/whatwg-mimetype": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@types/whatwg-mimetype/-/whatwg-mimetype-5.0.0.tgz", "integrity": "sha512-YYiBDCoqBgDIF2ByYn4qDb4RaXZ46cOQ6j2We1Ni3bikFNI7YFeL8jxSiYowWsriZrb1mw09CLZ+b+ZkIhsLVw==", "license": "MIT" }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.58.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.0.tgz", @@ -1178,53 +1206,53 @@ } }, "node_modules/@vue/compiler-core": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.32.tgz", - "integrity": "sha512-4x74Tbtqnda8s/NSD6e1Dr5p1c8HdMU5RWSjMSUzb8RTcUQqevDCxVAitcLBKT+ie3o0Dl9crc/S/opJM7qBGQ==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.34.tgz", + "integrity": "sha512-s9cLyK5mLcvZ4Agva5QgRsQyLKvts9WbU9DB6NqiZkkGEdwmcEiylj5Jbwkp680drF/NNCV8OlAJSe+yMLxaJw==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.2", - "@vue/shared": "3.5.32", + "@babel/parser": "^7.29.3", + "@vue/shared": "3.5.34", "entities": "^7.0.1", "estree-walker": "^2.0.2", "source-map-js": "^1.2.1" } }, "node_modules/@vue/compiler-dom": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.32.tgz", - "integrity": "sha512-ybHAu70NtiEI1fvAUz3oXZqkUYEe5J98GjMDpTGl5iHb0T15wQYLR4wE3h9xfuTNA+Cm2f4czfe8B4s+CCH57Q==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.34.tgz", + "integrity": "sha512-EbF/T++k0e2MMZlJsBhzK8Sgwt0HcIPOhzn1CTB/lv6sQcyk+OWf8YeiLxZp3ro7MbbLcAfAJ6sEvjFWuNgUCw==", "license": "MIT", "dependencies": { - "@vue/compiler-core": "3.5.32", - "@vue/shared": "3.5.32" + "@vue/compiler-core": "3.5.34", + "@vue/shared": "3.5.34" } }, "node_modules/@vue/compiler-sfc": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.32.tgz", - "integrity": "sha512-8UYUYo71cP/0YHMO814TRZlPuUUw3oifHuMR7Wp9SNoRSrxRQnhMLNlCeaODNn6kNTJsjFoQ/kqIj4qGvya4Xg==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.34.tgz", + "integrity": "sha512-D/ihr6uZeIt6r+pVZf46RWT1fAsLFMbUP7k8G1VkiiWexriED9GrX3echHd4Abbt17zjlfiFJ8z7a3BxZOPNjg==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.2", - "@vue/compiler-core": "3.5.32", - "@vue/compiler-dom": "3.5.32", - "@vue/compiler-ssr": "3.5.32", - "@vue/shared": "3.5.32", + "@babel/parser": "^7.29.3", + "@vue/compiler-core": "3.5.34", + "@vue/compiler-dom": "3.5.34", + "@vue/compiler-ssr": "3.5.34", + "@vue/shared": "3.5.34", "estree-walker": "^2.0.2", "magic-string": "^0.30.21", - "postcss": "^8.5.8", + "postcss": "^8.5.14", "source-map-js": "^1.2.1" } }, "node_modules/@vue/compiler-ssr": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.32.tgz", - "integrity": "sha512-Gp4gTs22T3DgRotZ8aA/6m2jMR+GMztvBXUBEUOYOcST+giyGWJ4WvFd7QLHBkzTxkfOt8IELKNdpzITLbA2rw==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.34.tgz", + "integrity": "sha512-cDtTHKibkThKGHH1SP+WdccquNRYQDFH6rRjQCqT9G2ltFAfoR5pUftpab/z+aM5mW9HLLVQW7hfKKQe/1GBeQ==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.32", - "@vue/shared": "3.5.32" + "@vue/compiler-dom": "3.5.34", + "@vue/shared": "3.5.34" } }, "node_modules/@vue/devtools-api": { @@ -1302,25 +1330,25 @@ } }, "node_modules/@vue/language-core": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-3.2.6.tgz", - "integrity": "sha512-xYYYX3/aVup576tP/23sEUpgiEnujrENaoNRbaozC1/MA9I6EGFQRJb4xrt/MmUCAGlxTKL2RmT8JLTPqagCkg==", + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-3.2.8.tgz", + "integrity": "sha512-9OiSPQFiAAWNVnXb0d2dcTmcKnFQamhuNES6ayyISrb/mwPWVgoGdAqSfCWqKhQpa3D5gDTcYD+w7ObiheZ81g==", "dev": true, "license": "MIT", "dependencies": { "@volar/language-core": "2.4.28", "@vue/compiler-dom": "^3.5.0", "@vue/shared": "^3.5.0", - "alien-signals": "^3.0.0", + "alien-signals": "^3.1.2", "muggle-string": "^0.4.1", "path-browserify": "^1.0.1", - "picomatch": "^4.0.2" + "picomatch": "^4.0.4" } }, "node_modules/@vue/language-core/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -1331,53 +1359,53 @@ } }, "node_modules/@vue/reactivity": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.32.tgz", - "integrity": "sha512-/ORasxSGvZ6MN5gc+uE364SxFdJ0+WqVG0CENXaGW58TOCdrAW76WWaplDtECeS1qphvtBZtR+3/o1g1zL4xPQ==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.34.tgz", + "integrity": "sha512-y9XDjCEuBp+98k+UL5dbYkh57AHU4o6cxZedOPXw3bmrZZYLQsVHguGurq7hVrPCSrQtrnz1f9dssyFr+dMXfQ==", "license": "MIT", "dependencies": { - "@vue/shared": "3.5.32" + "@vue/shared": "3.5.34" } }, "node_modules/@vue/runtime-core": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.32.tgz", - "integrity": "sha512-pDrXCejn4UpFDFmMd27AcJEbHaLemaE5o4pbb7sLk79SRIhc6/t34BQA7SGNgYtbMnvbF/HHOftYBgFJtUoJUQ==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.34.tgz", + "integrity": "sha512-mKeBYvu8tcMSLhypAHBmriUFfWXKTCF/23Z4jiCoYK3UtWepkliViNLuR90V9XOyD62mUxs9p1jsrpK3CCGIzw==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.32", - "@vue/shared": "3.5.32" + "@vue/reactivity": "3.5.34", + "@vue/shared": "3.5.34" } }, "node_modules/@vue/runtime-dom": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.32.tgz", - "integrity": "sha512-1CDVv7tv/IV13V8Nip1k/aaObVbWqRlVCVezTwx3K07p7Vxossp5JU1dcPNhJk3w347gonIUT9jQOGutyJrSVQ==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.34.tgz", + "integrity": "sha512-e8kZzERmCwUnBRVsgSQlAfrfU2rGoy0FFKPBXSlfEjc/O3KfA7QP0t1/2ZylrbchjmIKB4dPTd07A6WPr0eOrg==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.32", - "@vue/runtime-core": "3.5.32", - "@vue/shared": "3.5.32", + "@vue/reactivity": "3.5.34", + "@vue/runtime-core": "3.5.34", + "@vue/shared": "3.5.34", "csstype": "^3.2.3" } }, "node_modules/@vue/server-renderer": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.32.tgz", - "integrity": "sha512-IOjm2+JQwRFS7W28HNuJeXQle9KdZbODFY7hFGVtnnghF51ta20EWAZJHX+zLGtsHhaU6uC9BGPV52KVpYryMQ==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.34.tgz", + "integrity": "sha512-nHxmJoTrKsmrkbILRhkC9gY1G3moZbJTqCzDd7DOOzG5KH9oeJ0Unqrff5f9v0pW//jES05ZkJcNtfE8JjOIew==", "license": "MIT", "dependencies": { - "@vue/compiler-ssr": "3.5.32", - "@vue/shared": "3.5.32" + "@vue/compiler-ssr": "3.5.34", + "@vue/shared": "3.5.34" }, "peerDependencies": { - "vue": "3.5.32" + "vue": "3.5.34" } }, "node_modules/@vue/shared": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.32.tgz", - "integrity": "sha512-ksNyrmRQzWJJ8n3cRDuSF7zNNontuJg1YHnmWRJd2AMu8Ij2bqwiiri2lH5rHtYPZjj4STkNcgcmiQqlOjiYGg==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.34.tgz", + "integrity": "sha512-24uqU4OIiX29ryC3MeWid/Xf2fa2EFRUVLb77nRhk+UrTVrh/XiGtFAFmJBAtBRbjwNdsPRP+jj/OL27Eg1NDA==", "license": "MIT" }, "node_modules/@vue/tsconfig": { @@ -1399,6 +1427,18 @@ } } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -1574,6 +1614,26 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/birpc": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/birpc/-/birpc-2.9.0.tgz", @@ -1583,6 +1643,18 @@ "url": "https://github.com/sponsors/antfu" } }, + "node_modules/bl": { + "version": "6.1.6", + "resolved": "https://registry.npmjs.org/bl/-/bl-6.1.6.tgz", + "integrity": "sha512-jLsPgN/YSvPUg9UX0Kd73CXpm2Psg9FxMeCSXnk3WBO3CMT10JMwijubhGfHCnFu6TPn1ei3b975dxv7K2pWVg==", + "license": "MIT", + "dependencies": { + "@types/readable-stream": "^4.0.0", + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^4.2.0" + } + }, "node_modules/body-parser": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", @@ -1700,6 +1772,48 @@ "node": ">=8" } }, + "node_modules/broker-factory": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/broker-factory/-/broker-factory-3.1.14.tgz", + "integrity": "sha512-L45k5HMbPIrMid0nTOZ/UPXG/c0aRuQKVrSDFIb1zOkvfiyHgYmIjc3cSiN1KwQIvRDOtKE0tfb3I9EZ3CmpQQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "fast-unique-numbers": "^9.0.27", + "tslib": "^2.8.1", + "worker-factory": "^7.0.49" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1781,6 +1895,12 @@ "node": ">= 10" } }, + "node_modules/commist": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/commist/-/commist-3.2.0.tgz", + "integrity": "sha512-4PIMoPniho+LqXmpS5d3NuGYncG6XWlkBSVGiWycL22dd42OYdUGil2CWuzklaJoNxyxUSpO4MKIBU94viWNAw==", + "license": "MIT" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1788,6 +1908,35 @@ "dev": true, "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/confbox": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.2.tgz", @@ -2695,18 +2844,18 @@ } }, "node_modules/eslint": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.0.tgz", - "integrity": "sha512-+L0vBFYGIpSNIt/KWTpFonPrqYvgKw1eUI5Vn7mEogrQcWtWYtNQ7dNqC+px/J0idT3BAkiWrhfS7k+Tum8TUA==", + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.2.1.tgz", + "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", - "@eslint/config-array": "^0.23.4", - "@eslint/config-helpers": "^0.5.4", - "@eslint/core": "^1.2.0", - "@eslint/plugin-kit": "^0.7.0", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.5.5", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", @@ -2798,9 +2947,9 @@ } }, "node_modules/eslint-plugin-vue": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-10.8.0.tgz", - "integrity": "sha512-f1J/tcbnrpgC8suPN5AtdJ5MQjuXbSU9pGRSSYAuF3SHoiYCOdEX6O22pLaRyLHXvDcOe+O5ENgc1owQ587agA==", + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-10.9.1.tgz", + "integrity": "sha512-cHB0Tf4Duvzwecwd/AqWzZvF/QszE13BhjVUpVXWCy9AeMR5GjkAjP3i85vqgLgOuTmkHR1OJ5oMeqLHtuw8zg==", "dev": true, "license": "MIT", "dependencies": { @@ -2818,7 +2967,7 @@ "@stylistic/eslint-plugin": "^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0", "@typescript-eslint/parser": "^7.0.0 || ^8.0.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "vue-eslint-parser": "^10.0.0" + "vue-eslint-parser": "^10.3.0" }, "peerDependenciesMeta": { "@stylistic/eslint-plugin": { @@ -2966,6 +3115,24 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/express": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", @@ -3071,6 +3238,19 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-unique-numbers": { + "version": "9.0.27", + "resolved": "https://registry.npmjs.org/fast-unique-numbers/-/fast-unique-numbers-9.0.27.tgz", + "integrity": "sha512-nDA9ADeINN8SA2u2wCtU+siWFTTDqQR37XvgPIDDmboWQeExz7X0mImxuaN+kJddliIqy2FpVRmnvRZ+j8i1/A==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=18.2.0" + } + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -3491,6 +3671,12 @@ "node": ">= 0.4" } }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "license": "MIT" + }, "node_modules/highlight.js": { "version": "11.11.1", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.11.1.tgz", @@ -3551,6 +3737,26 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -3601,6 +3807,15 @@ "node": ">=12" } }, + "node_modules/ip-address": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", + "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4062,6 +4277,16 @@ "dev": true, "license": "ISC" }, + "node_modules/js-sdsl": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.3.0.tgz", + "integrity": "sha512-mifzlm2+5nZ+lEcLJMoBK0/IH/bDg8XnJfd/Wq6IP+xoCjLZsTOnV2QpxlVbX9bMnkl5PdEjNtBJ9Cj1NjifhQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -4490,6 +4715,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -4669,6 +4900,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/mitt": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", @@ -4704,6 +4944,49 @@ "pathe": "^2.0.1" } }, + "node_modules/mqtt": { + "version": "5.15.1", + "resolved": "https://registry.npmjs.org/mqtt/-/mqtt-5.15.1.tgz", + "integrity": "sha512-V1WnkGuJh3ec9QXzy5Iylw8OOBK+Xu1WhxcQ9mMpLThG+/JZIMV1PgLNRgIiqXhZnvnVLsuyxHl5A/3bHHbcAA==", + "license": "MIT", + "dependencies": { + "@types/readable-stream": "^4.0.21", + "@types/ws": "^8.18.1", + "commist": "^3.2.0", + "concat-stream": "^2.0.0", + "debug": "^4.4.1", + "help-me": "^5.0.0", + "lru-cache": "^10.4.3", + "minimist": "^1.2.8", + "mqtt-packet": "^9.0.2", + "number-allocator": "^1.0.14", + "readable-stream": "^4.7.0", + "rfdc": "^1.4.1", + "socks": "^2.8.6", + "split2": "^4.2.0", + "worker-timers": "^8.0.23", + "ws": "^8.18.3" + }, + "bin": { + "mqtt": "build/bin/mqtt.js", + "mqtt_pub": "build/bin/pub.js", + "mqtt_sub": "build/bin/sub.js" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/mqtt-packet": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/mqtt-packet/-/mqtt-packet-9.0.2.tgz", + "integrity": "sha512-MvIY0B8/qjq7bKxdN1eD+nrljoeaai+qjLJgfRn3TiMuz0pamsIWY2bFODPZMSNmabsLANXsLl4EMoWvlaTZWA==", + "license": "MIT", + "dependencies": { + "bl": "^6.0.8", + "debug": "^4.3.4", + "process-nextick-args": "^2.0.1" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4767,9 +5050,9 @@ "license": "MIT" }, "node_modules/nodemailer": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.5.tgz", - "integrity": "sha512-0PF8Yb1yZuQfQbq+5/pZJrtF6WQcjTd5/S4JOHs9PGFxuTqoB/icwuB44pOdURHJbRKX1PPoJZtY7R4VUoCC8w==", + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.7.tgz", + "integrity": "sha512-pkjE4mkBzQjdJT4/UmlKl3pX0rC9fZmjh7c6C9o7lv66Ac6w9WCnzPzhbPNxwZAzlF4mdq4CSWB5+FbK6FWCow==", "license": "MIT-0", "engines": { "node": ">=6.0.0" @@ -5012,6 +5295,16 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/number-allocator": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/number-allocator/-/number-allocator-1.0.14.tgz", + "integrity": "sha512-OrL44UTVAvkKdOdRQZIJpLkAdjXGTRda052sN4sO77bKEzYYqWKMBjQvrJFzqygI99gL6Z4u2xctPW1tB8ErvA==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.1", + "js-sdsl": "4.3.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5443,6 +5736,21 @@ "node": ">=6.0.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5594,6 +5902,22 @@ "node": ">=4" } }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/readdirp": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", @@ -5701,14 +6025,14 @@ "license": "Unlicense" }, "node_modules/rolldown": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.15.tgz", - "integrity": "sha512-Ff31guA5zT6WjnGp0SXw76X6hzGRk/OQq2hE+1lcDe+lJdHSgnSX6nK3erbONHyCbpSj9a9E+uX/OvytZoWp2g==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.18.tgz", + "integrity": "sha512-phmyKBpuBdRYDf4hgyynGAYn/rDDe+iZXKVJ7WX5b1zQzpLkP5oJRPGsfJuHdzPMlyyEO/4sPW6yfSx2gf7lVg==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.124.0", - "@rolldown/pluginutils": "1.0.0-rc.15" + "@oxc-project/types": "=0.128.0", + "@rolldown/pluginutils": "1.0.0-rc.18" }, "bin": { "rolldown": "bin/cli.mjs" @@ -5717,27 +6041,27 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.0-rc.15", - "@rolldown/binding-darwin-arm64": "1.0.0-rc.15", - "@rolldown/binding-darwin-x64": "1.0.0-rc.15", - "@rolldown/binding-freebsd-x64": "1.0.0-rc.15", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.15", - "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.15", - "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.15", - "@rolldown/binding-linux-x64-musl": "1.0.0-rc.15", - "@rolldown/binding-openharmony-arm64": "1.0.0-rc.15", - "@rolldown/binding-wasm32-wasi": "1.0.0-rc.15", - "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.15", - "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.15" + "@rolldown/binding-android-arm64": "1.0.0-rc.18", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.18", + "@rolldown/binding-darwin-x64": "1.0.0-rc.18", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.18", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.18", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.18", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.18", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.18", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.18", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.18", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.18", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.18", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.18", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.18", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.18" } }, "node_modules/rolldown/node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.15", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.15.tgz", - "integrity": "sha512-UromN0peaE53IaBRe9W7CjrZgXl90fqGpK+mIZbA3qSTeYqg3pqpROBdIPvOG3F5ereDHNwoHBI2e50n1BDr1g==", + "version": "1.0.0-rc.18", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.18.tgz", + "integrity": "sha512-CUY5Mnhe64xQBGZEEXQ5WyZwsc1JU3vAZLIxtrsBt3LO6UOb+C8GunVKqe9sT8NeWb4lqSaoJtp2xo6GxT1MNw==", "dev": true, "license": "MIT" }, @@ -5806,6 +6130,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-push-apply": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-push-apply/-/safe-push-apply-1.0.0.tgz", @@ -6096,6 +6440,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.8.tgz", + "integrity": "sha512-NlGELfPrgX2f1TAAcz0WawlLn+0r3FyhhCRpFFK2CemXenPYvzMWWZINv3eDNo9ucdwme7oCHRY0Jnbs4aIkog==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -6150,6 +6518,15 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -6179,6 +6556,15 @@ "integrity": "sha512-Nk/brWYpD85WlOgzw5h173aci0Teyv8YdIAEtV+N88nDB0dLlazZyJMIsN6eo1/AR61l+p6CJTG1JIyFaoNEEA==", "license": "ISC" }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string.prototype.padend": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.6.tgz", @@ -6309,13 +6695,13 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -6342,9 +6728,9 @@ } }, "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", "engines": { "node": ">=12" @@ -6391,9 +6777,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/type-check": { "version": "0.4.0", @@ -6500,6 +6884,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typescript": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", @@ -6664,7 +7054,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/validate-npm-package-license": { @@ -6688,17 +7077,17 @@ } }, "node_modules/vite": { - "version": "8.0.8", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.8.tgz", - "integrity": "sha512-dbU7/iLVa8KZALJyLOBOQ88nOXtNG8vxKuOT4I2mD+Ya70KPceF4IAmDsmU0h1Qsn5bPrvsY9HJstCRh3hG6Uw==", + "version": "8.0.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.11.tgz", + "integrity": "sha512-Jz1mxtUBR5xTT65VOdJZUUeoyLtqljmFkiUXhPTLZka3RDc9vpi/xXkyrnsdRcm2lIi3l3GPMnAidTsEGIj3Ow==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", - "postcss": "^8.5.8", - "rolldown": "1.0.0-rc.15", - "tinyglobby": "^0.2.15" + "postcss": "^8.5.14", + "rolldown": "1.0.0-rc.18", + "tinyglobby": "^0.2.16" }, "bin": { "vite": "bin/vite.js" @@ -6714,7 +7103,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.0", + "@vitejs/devtools": "^0.1.18", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -6801,16 +7190,16 @@ "license": "MIT" }, "node_modules/vue": { - "version": "3.5.32", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.32.tgz", - "integrity": "sha512-vM4z4Q9tTafVfMAK7IVzmxg34rSzTFMyIe0UUEijUCkn9+23lj0WRfA83dg7eQZIUlgOSGrkViIaCfqSAUXsMw==", + "version": "3.5.34", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.34.tgz", + "integrity": "sha512-WdLBG9gm02OgJIG9axd5Hpx0TFLdzVgfG2evFFu8Rur5O/IoGc5cMjnjh3tPL6GnRGsYvUhBSKVPYVcxRKpMCA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.32", - "@vue/compiler-sfc": "3.5.32", - "@vue/runtime-dom": "3.5.32", - "@vue/server-renderer": "3.5.32", - "@vue/shared": "3.5.32" + "@vue/compiler-dom": "3.5.34", + "@vue/compiler-sfc": "3.5.34", + "@vue/runtime-dom": "3.5.34", + "@vue/server-renderer": "3.5.34", + "@vue/shared": "3.5.34" }, "peerDependencies": { "typescript": "*" @@ -6859,9 +7248,9 @@ } }, "node_modules/vue-router": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-5.0.4.tgz", - "integrity": "sha512-lCqDLCI2+fKVRl2OzXuzdSWmxXFLQRxQbmHugnRpTMyYiT+hNaycV0faqG5FBHDXoYrZ6MQcX87BvbY8mQ20Bg==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-5.0.6.tgz", + "integrity": "sha512-9+kmUTGbKMyW9Asoy98IXXYIzrTMT7JDAdpDDeEkorHvybpUvBI2wsrSM5jFOXrFydpzRFJ9vAh+80DN2PGu9w==", "license": "MIT", "dependencies": { "@babel/generator": "^7.28.6", @@ -6916,14 +7305,14 @@ } }, "node_modules/vue-tsc": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-3.2.6.tgz", - "integrity": "sha512-gYW/kWI0XrwGzd0PKc7tVB/qpdeAkIZLNZb10/InizkQjHjnT8weZ/vBarZoj4kHKbUTZT/bAVgoOr8x4NsQ/Q==", + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-3.2.8.tgz", + "integrity": "sha512-27vTLJ6Q2370obOd0PFYoYoKnmXJ521uUIedrs3Zhhhg/8YG10VOCMmwt+JQslatpAMTDbnWiitLnoD5VlIvog==", "dev": true, "license": "MIT", "dependencies": { "@volar/typescript": "2.4.28", - "@vue/language-core": "3.2.6" + "@vue/language-core": "3.2.8" }, "bin": { "vue-tsc": "bin/vue-tsc.js" @@ -7075,12 +7464,80 @@ "node": ">=0.10.0" } }, + "node_modules/worker-factory": { + "version": "7.0.49", + "resolved": "https://registry.npmjs.org/worker-factory/-/worker-factory-7.0.49.tgz", + "integrity": "sha512-lW7tpgy6aUv2dFsQhv1yv+XFzdkCf/leoKRTGMPVK5/die6RrUjqgJHJf556qO+ZfytNG6wPXc17E8zzsOLUDw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "fast-unique-numbers": "^9.0.27", + "tslib": "^2.8.1" + } + }, + "node_modules/worker-timers": { + "version": "8.0.31", + "resolved": "https://registry.npmjs.org/worker-timers/-/worker-timers-8.0.31.tgz", + "integrity": "sha512-ngkq5S6JuZyztom8tDgBzorLo9byhBMko/sXfgiUD945AuzKGg1GCgDMCC3NaYkicLpGKXutONM36wEX8UbBCA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "tslib": "^2.8.1", + "worker-timers-broker": "^8.0.16", + "worker-timers-worker": "^9.0.14" + } + }, + "node_modules/worker-timers-broker": { + "version": "8.0.16", + "resolved": "https://registry.npmjs.org/worker-timers-broker/-/worker-timers-broker-8.0.16.tgz", + "integrity": "sha512-JyP3AvUGyPGbBGW7XiUewm2+0pN/aYo1QpVf5kdXAfkDZcN3p7NbWrG6XnyDEpDIvfHk/+LCnOW/NsuiU9riYA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "broker-factory": "^3.1.14", + "fast-unique-numbers": "^9.0.27", + "tslib": "^2.8.1", + "worker-timers-worker": "^9.0.14" + } + }, + "node_modules/worker-timers-worker": { + "version": "9.0.14", + "resolved": "https://registry.npmjs.org/worker-timers-worker/-/worker-timers-worker-9.0.14.tgz", + "integrity": "sha512-/qF06C60sXmSLfUl7WglvrDIbspmPOM8UrG63Dnn4bi2x4/DfqHS/+dxF5B+MdHnYO5tVuZYLHdAodrKdabTIg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.29.2", + "tslib": "^2.8.1", + "worker-factory": "^7.0.49" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/ws": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xml-formatter": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/xml-formatter/-/xml-formatter-3.7.0.tgz", diff --git a/webui/package.json b/webui/package.json index 1f9f2d3ed..e74ffd2dd 100644 --- a/webui/package.json +++ b/webui/package.json @@ -39,10 +39,11 @@ "markdown-it": "^14.1.1", "markdown-it-container": "^4.0.0", "mime-types": "^3.0.2", + "mqtt": "^5.15.1", "ncp": "^2.0.0", - "nodemailer": "^8.0.5", - "vue": "^3.5.32", - "vue-router": "^5.0.4", + "nodemailer": "^8.0.7", + "vue": "^3.5.34", + "vue-router": "^5.0.6", "vue3-ace-editor": "^2.2.4", "whatwg-mimetype": "^5.0.0", "xml-formatter": "^3.7.0" @@ -52,18 +53,18 @@ "@rushstack/eslint-patch": "^1.16.1", "@types/js-yaml": "^4.0.9", "@types/markdown-it-container": "^4.0.0", - "@types/node": "^25.6.0", + "@types/node": "^25.6.2", "@vitejs/plugin-vue": "^6.0.6", "@vue/eslint-config-prettier": "^10.2.0", "@vue/eslint-config-typescript": "^14.7.0", "@vue/tsconfig": "^0.9.1", - "eslint": "^10.2.0", - "eslint-plugin-vue": "^10.8.0", + "eslint": "^10.2.1", + "eslint-plugin-vue": "^10.9.1", "npm-run-all": "^4.1.5", "prettier": "^3.8.3", "typescript": "~6.0.3", - "vite": "^8.0.8", - "vue-tsc": "^3.2.6", + "vite": "^8.0.11", + "vue-tsc": "^3.2.8", "xml2js": "^0.6.2" } } diff --git a/webui/playwright.config.ts b/webui/playwright.config.ts index f367bd38f..cba59b000 100644 --- a/webui/playwright.config.ts +++ b/webui/playwright.config.ts @@ -103,6 +103,31 @@ const config: PlaywrightTestConfig = { }, testIgnore: ["/e2e/tests/**/*.dashboard.spec.ts", "/e2e/tests/dashboard/**/*.spec.ts"], }, + { + // use this for testing demo-dashboard + // setup: + // run mokapi in scripts/dashboard-demo/demo-configs + // run ci.ts + name: 'demo-dashboard', + use: { + ...devices['Desktop Chrome'], + baseURL: 'http://localhost:8080', + storageState: { + cookies: [], + origins: [ + { + origin: 'http://localhost:8080', + localStorage: [ + { + name: 'theme', value: 'dark' + } + ] + } + ] + }, + }, + testMatch: ["/e2e/tests/dashboard-demo/**/*.spec.ts"], + }, // { // name: 'firefox', // use: { diff --git a/webui/scripts/dashboard-demo/ci.ts b/webui/scripts/dashboard-demo/ci.ts index f481cc5fd..7f19631b1 100644 --- a/webui/scripts/dashboard-demo/ci.ts +++ b/webui/scripts/dashboard-demo/ci.ts @@ -1,6 +1,7 @@ import { collectDashboard } from './collect-dashboard.ts'; import { driveHttp } from './drive-http.ts'; import { driveKafka, closeKafka } from './drive-kafka.ts'; +import { driveMqtt } from './drive-mqtt.ts'; import { driveMail } from './drive-mail.ts'; import { driveLdap } from './drive-ldap.ts'; @@ -8,6 +9,7 @@ async function main() { try { await driveHttp(); await driveKafka(); + await driveMqtt(); await driveMail(); await driveLdap(); diff --git a/webui/scripts/dashboard-demo/collect-dashboard.ts b/webui/scripts/dashboard-demo/collect-dashboard.ts index ea93dcc03..80f3a918f 100644 --- a/webui/scripts/dashboard-demo/collect-dashboard.ts +++ b/webui/scripts/dashboard-demo/collect-dashboard.ts @@ -13,9 +13,16 @@ export async function collectDashboard() { services: { path: '/api/services' , loader: loadJson }, metrics: { path: '/api/metrics?q=app', loader: loadJson }, 'service_Swagger Petstore': { path: '/api/services/http/Swagger%20Petstore', loader: loadJson }, + 'service_Swagger Petstore_operations': { path: '/api/services/http/Swagger%20Petstore/operations', loader: loadJson }, 'service_Kafka Order Service API': { path: '/api/services/kafka/Kafka%20Order%20Service%20API', loader: loadJson }, + 'service_Kafka Order Service API_topic_order-topic': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/topics/order-topic', loader: loadJson }, + 'service_Kafka Order Service API_topic_user-events': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/topics/user-events', loader: loadJson }, + 'service_Kafka Order Service API_group_order-status-group-100': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/groups/order-status-group-100', loader: loadJson }, + 'service_Kafka Order Service API_client_consumer-1': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/clients/consumer-1', loader: loadJson }, + 'service_Kafka Order Service API_client_producer-1': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/clients/producer-1', loader: loadJson }, 'service_Mail Server': { path: '/api/services/mail/Mail%20Server', loader: loadJson }, 'service_HR Employee Directory': { path: '/api/services/ldap/HR%20Employee%20Directory', loader: loadJson }, + 'service_Smart Home MQTT API': { path: '/api/services/mqtt/Smart%20Home%20MQTT%20API', loader: loadJson }, events: { path: '/api/events', loader: fetchEvents }, 'mailbox_alice.johnson@example.com': { path: '/api/services/mail/Mail%20Server/mailboxes/alice.johnson@example.com', loader: loadJson }, 'mailbox_bob.miller@example.com': { path: '/api/services/mail/Mail%20Server/mailboxes/bob.miller@example.com', loader: loadJson }, diff --git a/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml b/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml index 5082f5220..347c24e0c 100644 --- a/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml +++ b/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml @@ -13,14 +13,14 @@ servers: channels: order_events: address: order-topic - description: The Kafka topic for order events. + summary: The Kafka topic for order events. messages: OrderCreated: $ref: '#/components/messages/OrderCreated' tags: - name: order user-events: - description: The Kafka topic for user events. + summary: The Kafka topic for user events. tags: - name: user diff --git a/webui/scripts/dashboard-demo/demo-configs/mqtt.yaml b/webui/scripts/dashboard-demo/demo-configs/mqtt.yaml new file mode 100644 index 000000000..2eff32ec4 --- /dev/null +++ b/webui/scripts/dashboard-demo/demo-configs/mqtt.yaml @@ -0,0 +1,41 @@ +asyncapi: 3.1.0 +info: + title: Smart Home MQTT API + version: 1.0.0 + description: Example specification for controlling sensors via MQTT. +servers: + production: + host: localhost:1883 + protocol: mqtt + description: Public HiveMQ broker for testing purposes. +channels: + sensorData: + address: 'sensors/{sensorId}/data' + messages: + temperatureUpdate: + $ref: '#/components/messages/TemperaturePayload' + parameters: + sensorId: + description: The unique ID of the sensor. +operations: + receiveSensorData: + action: receive + channel: + $ref: '#/channels/sensorData' + summary: Receives temperature data from sensors. + userBindings: + mqtt: + qos: 1 + +components: + messages: + TemperaturePayload: + payload: + type: object + properties: + temp: + type: number + description: Temperature in degrees Celsius. + timestamp: + type: string + format: date-time \ No newline at end of file diff --git a/webui/scripts/dashboard-demo/drive-mqtt.ts b/webui/scripts/dashboard-demo/drive-mqtt.ts new file mode 100644 index 000000000..0debf079b --- /dev/null +++ b/webui/scripts/dashboard-demo/drive-mqtt.ts @@ -0,0 +1,36 @@ +import mqtt from 'mqtt'; + +export async function driveMqtt(): Promise { + const client = mqtt.connect('mqtt://localhost:1883') + + await new Promise((resolve, reject) => { + client.on('connect', () => { + console.log('mqtt: subscribe topic') + + client.subscribe('sensors/12345/data', (err) => { + if (err) { + console.error('mqtt: ', err) + reject(err); + return; + } + + console.log('mqtt: publish message') + client.publish('sensors/12345/data', JSON.stringify({ + temp: 24, + timestamp: '2026-02-13T09:49:25.482366+01:00' + })); + }); + }); + + client.on('message', (topic, message) => { + console.log(message.toString()); + client.end(); + resolve(); + }); + + client.on('error', (err) => { + console.error('mqtt: ', err) + reject(err); + }); + }); +} \ No newline at end of file diff --git a/webui/scripts/dashboard-demo/index.ts b/webui/scripts/dashboard-demo/index.ts index 6ab0aecbd..4acfd549d 100644 --- a/webui/scripts/dashboard-demo/index.ts +++ b/webui/scripts/dashboard-demo/index.ts @@ -1,6 +1,7 @@ import { collectDashboard } from './collect-dashboard.ts'; import { driveHttp } from './drive-http.ts'; import { driveKafka, closeKafka } from './drive-kafka.ts'; +import { driveMqtt } from './drive-mqtt.ts'; import { startMokapi, stopMokapi } from './mokapi.ts'; import { driveMail } from './drive-mail.ts'; import { driveLdap } from './drive-ldap.ts'; @@ -12,6 +13,7 @@ async function main() { await driveHttp(); await driveKafka(); + await driveMqtt(); await driveMail(); await driveLdap(); diff --git a/webui/src/assets/dashboard.css b/webui/src/assets/dashboard.css index 92326061f..7b6bef787 100644 --- a/webui/src/assets/dashboard.css +++ b/webui/src/assets/dashboard.css @@ -230,4 +230,9 @@ background: #f5f5f5; padding: 1px 4px; border-radius: 3px; +} + +.status { + text-transform: capitalize; + color: var(--color-red); } \ No newline at end of file diff --git a/webui/src/assets/main.css b/webui/src/assets/main.css index b87e10bbc..4d298c7c3 100644 --- a/webui/src/assets/main.css +++ b/webui/src/assets/main.css @@ -97,6 +97,7 @@ pre{ .yellow { color: var(--color-yellow) } +.red { color: var(--color-red); } .logo { margin-top: 2rem; margin-bottom: 2rem; diff --git a/webui/src/components/dashboard/Events.vue b/webui/src/components/dashboard/Events.vue new file mode 100644 index 000000000..03197b434 --- /dev/null +++ b/webui/src/components/dashboard/Events.vue @@ -0,0 +1,85 @@ + + + \ No newline at end of file diff --git a/webui/src/components/dashboard/JobCard.vue b/webui/src/components/dashboard/JobCard.vue index 9faa758e6..4e295aad0 100644 --- a/webui/src/components/dashboard/JobCard.vue +++ b/webui/src/components/dashboard/JobCard.vue @@ -8,7 +8,7 @@ import { usePrettyText } from '@/composables/usePrettyText' import { getRouteName, useDashboard } from '@/composables/dashboard' const { dashboard } = useDashboard() -const { events, close } = dashboard.value.getEvents('job') +const { events, close } = dashboard.value.getEvents({ name: 'namespace', value: 'jobs' }) const { format, duration: prettyDuration } = usePrettyDates() const { parseUrls } = usePrettyText() dayjs.extend(duration) diff --git a/webui/src/components/dashboard/SchemaValidate.vue b/webui/src/components/dashboard/SchemaValidate.vue index 335cdc503..2178aed52 100644 --- a/webui/src/components/dashboard/SchemaValidate.vue +++ b/webui/src/components/dashboard/SchemaValidate.vue @@ -35,7 +35,6 @@ const props = withDefaults(defineProps<{ }>(), { title: 'Data Validator' }) -console.log(props) const { createGuid } = useGuid() const { formatLanguage } = usePrettyLanguage() diff --git a/webui/src/components/dashboard/Search.vue b/webui/src/components/dashboard/Search.vue index a9d5496de..586aa88a5 100644 --- a/webui/src/components/dashboard/Search.vue +++ b/webui/src/components/dashboard/Search.vue @@ -1,12 +1,17 @@ + + + + \ No newline at end of file diff --git a/webui/src/components/dashboard/SourceView.vue b/webui/src/components/dashboard/SourceView.vue index 437a941e5..6130fa275 100644 --- a/webui/src/components/dashboard/SourceView.vue +++ b/webui/src/components/dashboard/SourceView.vue @@ -46,7 +46,6 @@ if (props.source.preview) { } else { throw new Error('preview and binary not defined') } -console.log(current.value) watch(() => props.source, (source) => { if (!current.value) { return diff --git a/webui/src/components/dashboard/Tabs.vue b/webui/src/components/dashboard/Tabs.vue index ec593dfa1..399ceb43f 100644 --- a/webui/src/components/dashboard/Tabs.vue +++ b/webui/src/components/dashboard/Tabs.vue @@ -19,9 +19,11 @@ const tabItems = computed(() => [ { text: 'Overview', isVisible: true, to: { name: getRouteName('dashboard').value }, cssClass: 'overview' }, { text: 'HTTP', isVisible: isServiceAvailable('http'), to: { name: getRouteName('http').value } }, { text: 'Kafka', isVisible: isServiceAvailable('kafka'), to: { name: getRouteName('kafka').value } }, + { text: 'MQTT', isVisible: isServiceAvailable('mqtt'), to: { name: getRouteName('mqtt').value } }, { text: 'Mail', isVisible: isServiceAvailable('mail'), to: { name: getRouteName('mail').value } }, { text: 'LDAP', isVisible: isServiceAvailable('ldap'), to: { name: getRouteName('ldap').value } }, { text: 'Jobs', isVisible: hasJobs.value, to: { name: getRouteName('jobs').value } }, + // { text: 'Events', isVisible: true, to: { name: getRouteName('events').value } }, { text: 'Configs', isVisible: true, to: { name: getRouteName('configs').value } }, { text: 'Faker', isVisible: getMode() === 'live', to: { name: getRouteName('tree').value } }, { text: 'Search', isVisible: props.appInfo.search.enabled && getMode() !== 'demo', to: { name: getRouteName('search').value } }, diff --git a/webui/src/components/dashboard/http/EndpointsCard.vue b/webui/src/components/dashboard/http/EndpointsCard.vue index 7c9c29801..09f419f17 100644 --- a/webui/src/components/dashboard/http/EndpointsCard.vue +++ b/webui/src/components/dashboard/http/EndpointsCard.vue @@ -75,19 +75,33 @@ function goToPath(path: HttpPath, openInNewTab = false){ } } function lastRequest(path: HttpPath){ - const n = sum(props.service.metrics, 'http_request_timestamp', {name: 'endpoint', value: path.path}) - if (n == 0){ + let lastRequest = 0 + for (const op of path.operations) { + if (op.metrics.http_request_timestamp > lastRequest) { + lastRequest = op.metrics.http_request_timestamp + } + } + + if (lastRequest == 0){ return '-' } - return format(n) + return format(lastRequest) } function requests(path: HttpPath){ - return sum(props.service.metrics, 'http_requests_total', {name: 'endpoint', value: path.path}) + let totalRequests = 0 + for (const op of path.operations) { + totalRequests += op.metrics.http_requests_total + } + return totalRequests } function errors(path: HttpPath){ - return sum(props.service.metrics, 'http_requests_errors_total', {name: 'endpoint', value: path.path}) + let totalErrors = 0 + for (const op of path.operations) { + totalErrors += op.metrics.http_requests_errors_total + } + return totalErrors } function allOperationsDeprecated(path: HttpPath): boolean{ @@ -122,7 +136,7 @@ function operations(path: HttpPath) { }) } -function operationOrderValue(operation: HttpOperation): number { +function operationOrderValue(operation: HttpOperationInfo): number { switch (operation.method.toLowerCase()) { case 'get': return 0 case 'post': return 1 @@ -165,6 +179,21 @@ function toggleTag(name: string) { tags.value = tags.value.filter((t) => t !== '__all') } } +function operationTitle(operation: HttpOperationInfo): string { + if (operation.errors && operation.errors.length > 0) { + return `${operation.errors.map(x => x.message).join(' ')}` + } + if (operation.summary) { + return operation.summary + } + return '' +} +function operationCssClass(operation: HttpOperationInfo): string { + if (operation.status === 'valid') { + return operation.method + } + return 'border border-danger text-danger bg-transparent' +}