diff --git a/backend/plugins/linear/README.md b/backend/plugins/linear/README.md new file mode 100644 index 00000000000..e52b4527176 --- /dev/null +++ b/backend/plugins/linear/README.md @@ -0,0 +1,115 @@ + + +# Linear + +## Summary + +This plugin collects data from [Linear](https://linear.app) through its +[GraphQL API](https://linear.app/developers/graphql) and maps it into DevLake's +standardized `ticket` domain, so Linear issues appear in DevLake dashboards +(throughput, lead/cycle time, sprint burndown, etc.). + +The selectable **scope** is a Linear **Team**, which maps to a domain `Board`. + +## Supported data + +| Linear entity | Tool-layer table | Domain-layer table | +|-----------------|-----------------------------------|--------------------------------------------| +| Team | `_tool_linear_teams` (scope) | `boards` | +| User | `_tool_linear_accounts` | `accounts` | +| Workflow state | `_tool_linear_workflow_states` | (drives issue status mapping) | +| Issue | `_tool_linear_issues` | `issues`, `board_issues` | +| Label | `_tool_linear_issue_labels` | `issue_labels` | +| Comment | `_tool_linear_comments` | `issue_comments` | +| Cycle | `_tool_linear_cycles` | `sprints`, `board_sprints`, `sprint_issues`| +| Issue history | `_tool_linear_issue_history` | `issue_changelogs` | + +### Field mapping highlights + +- **Status** — derived deterministically from Linear's `WorkflowState.type` + (no manual mapping needed, unlike Jira): + - `backlog`, `unstarted` → `TODO` + - `started` → `IN_PROGRESS` + - `completed`, `canceled` → `DONE` +- **Priority** — Linear's integer priority maps to a label: `0` No priority, + `1` Urgent, `2` High, `3` Medium, `4` Low. +- **Type** — Linear has no native issue type, so issues default to `REQUIREMENT`. +- **Lead time** — `completedAt − createdAt` (Linear provides `startedAt`/`completedAt` + natively; the history changelog captures every status transition). +- **Story points** — Linear's `estimate`. + +## Authentication + +The plugin uses a Linear **personal API key**, passed verbatim in the +`Authorization` header (no `Bearer` prefix). Create one under +**Settings → Security & access → Personal API keys** in Linear. + +## Configuration + +Create a connection: + +``` +curl 'http://localhost:8080/api/plugins/linear/connections' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear", + "endpoint": "https://api.linear.app/graphql", + "token": "", + "rateLimitPerHour": 1500 +}' +``` + +Add a team scope (the team id is the Linear team UUID): + +``` +curl 'http://localhost:8080/api/plugins/linear/connections//scopes' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "data": [{ "connectionId": , "teamId": "", "name": "Engineering" }] +}' +``` + +## Collecting data + +``` +curl 'http://localhost:8080/api/pipelines' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear pipeline", + "plan": [[{ + "plugin": "linear", + "options": { "connectionId": , "teamId": "" } + }]] +}' +``` + +## Rate limiting + +Linear enforces a per-API-key request budget (1,500 requests/hour) plus a +complexity budget. The collector paces requests against the configured +`rateLimitPerHour` (default 1500). Issues are collected incrementally using +`updatedAt` ordering so re-runs only fetch changes. + +## Limitations / roadmap + +- Authentication is personal API key only; OAuth2 is a planned follow-up. +- Issue type defaults to `REQUIREMENT`; label-based type mapping via the scope + config is a planned follow-up. +- config-ui integration (connection form + team picker) and the website + documentation page are planned follow-ups; for now connections and scopes are + managed via the API calls shown above. diff --git a/backend/plugins/linear/api/blueprint_v200.go b/backend/plugins/linear/api/blueprint_v200.go new file mode 100644 index 00000000000..281d018cefb --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "linear", + subtaskMetas, + scopeConfig.Entities, + tasks.LinearOptions{ + ConnectionId: connection.ID, + TeamId: scope.TeamId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.TeamId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/linear/api/blueprint_v200_test.go b/backend/plugins/linear/api/blueprint_v200_test.go new file mode 100644 index 00000000000..c77d67fe878 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200_test.go @@ -0,0 +1,90 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/stretchr/testify/assert" +) + +func mockLinearPlugin(t *testing.T) { + mockMeta := mockplugin.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/linear") + mockMeta.On("Name").Return("linear").Maybe() + _ = plugin.RegisterPlugin("linear", mockMeta) +} + +func TestMakeScopesV200(t *testing.T) { + mockLinearPlugin(t) + + const connectionId uint64 = 1 + const teamId = "team-1" + const expectDomainScopeId = "linear:LinearTeam:1:team-1" + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: connectionId}, + TeamId: teamId, + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: connectionId}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) +} + +func TestMakeScopesV200WithoutTicketEntity(t *testing.T) { + mockLinearPlugin(t) + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1}, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + // no ticket entity selected => no domain board scope produced + assert.Equal(t, 0, len(scopes)) +} diff --git a/backend/plugins/linear/api/connection_api.go b/backend/plugins/linear/api/connection_api.go new file mode 100644 index 00000000000..c2c95c54d64 --- /dev/null +++ b/backend/plugins/linear/api/connection_api.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +const defaultEndpoint = "https://api.linear.app/graphql" + +type LinearTestConnResponse struct { + shared.ApiBody + Connection *models.LinearConn +} + +func testConnection(ctx context.Context, connection models.LinearConn) (*LinearTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = defaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // Linear is GraphQL-over-HTTP-POST; a minimal viewer query verifies the key. + reqBody := map[string]interface{}{"query": "{ viewer { id name } }"} + res, err := apiClient.Post("", nil, reqBody, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API key") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := LinearTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test linear connection +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param body body models.LinearConn true "json body" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.LinearConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test linear connection by ID +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.LinearConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create linear connection +// @Summary create linear connection +// @Description Create linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch linear connection +// @Summary patch linear connection +// @Description Patch linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a linear connection +// @Summary delete a linear connection +// @Description Delete a linear connection +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all linear connections +// @Summary get all linear connections +// @Description Get all linear connections +// @Tags plugins/linear +// @Success 200 {object} []models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get linear connection detail +// @Summary get linear connection detail +// @Description Get linear connection detail +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go new file mode 100644 index 00000000000..850acf54825 --- /dev/null +++ b/backend/plugins/linear/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.LinearConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.LinearConnection, models.LinearTeam, models.LinearScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.LinearConnection) models.LinearConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.LinearConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination](raProxy, listLinearRemoteScopes) +} diff --git a/backend/plugins/linear/api/remote_api.go b/backend/plugins/linear/api/remote_api.go new file mode 100644 index 00000000000..094c785016d --- /dev/null +++ b/backend/plugins/linear/api/remote_api.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearRemotePagination drives cursor-based pagination through the GraphQL +// `teams` connection when listing remote scopes for the config UI. +type LinearRemotePagination struct { + Cursor string `json:"cursor"` +} + +// linearTeamsGraphqlResponse mirrors the shape of the `teams` query response. +type linearTeamsGraphqlResponse struct { + Data struct { + Teams struct { + Nodes []struct { + Id string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + Description string `json:"description"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + } `json:"teams"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const remoteScopesPageSize = 100 + +// listLinearRemoteScopes lists Linear teams as selectable scopes. Linear teams +// are a flat list, so there are no intermediate groups. +func listLinearRemoteScopes( + _ *models.LinearConnection, + apiClient plugin.ApiClient, + _ string, + page LinearRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], + nextPage *LinearRemotePagination, + err errors.Error, +) { + after := "" + if page.Cursor != "" { + after = fmt.Sprintf(", after: %q", page.Cursor) + } + query := fmt.Sprintf( + "query { teams(first: %d%s) { nodes { id name key description } pageInfo { hasNextPage endCursor } } }", + remoteScopesPageSize, after, + ) + + res, err := apiClient.Post("", nil, map[string]interface{}{"query": query}, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to query Linear teams") + } + var response linearTeamsGraphqlResponse + if err := api.UnmarshalResponse(res, &response); err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Linear teams response") + } + if len(response.Errors) > 0 { + return nil, nil, errors.Default.New("linear graphql teams query failed: " + response.Errors[0].Message) + } + + return mapLinearTeamsToScopeEntries(response), nextPageFrom(response), nil +} + +// mapLinearTeamsToScopeEntries converts a teams response into scope-list +// entries. Each team is a selectable (leaf) scope. +func mapLinearTeamsToScopeEntries(response linearTeamsGraphqlResponse) []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam] { + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], 0, len(response.Data.Teams.Nodes)) + for _, team := range response.Data.Teams.Nodes { + team := team + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: team.Id, + Name: team.Name, + FullName: team.Name, + Data: &models.LinearTeam{ + TeamId: team.Id, + Name: team.Name, + Key: team.Key, + Description: team.Description, + }, + }) + } + return children +} + +// nextPageFrom returns the cursor for the following page, or nil when the +// teams connection has been fully traversed. +func nextPageFrom(response linearTeamsGraphqlResponse) *LinearRemotePagination { + pageInfo := response.Data.Teams.PageInfo + if pageInfo.HasNextPage && pageInfo.EndCursor != "" { + return &LinearRemotePagination{Cursor: pageInfo.EndCursor} + } + return nil +} + +// RemoteScopes lists the Linear teams available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the Linear API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/linear/api/remote_api_test.go b/backend/plugins/linear/api/remote_api_test.go new file mode 100644 index 00000000000..801e0d5029f --- /dev/null +++ b/backend/plugins/linear/api/remote_api_test.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestMapLinearTeamsToScopeEntries(t *testing.T) { + var response linearTeamsGraphqlResponse + body := `{"data":{"teams":{"nodes":[` + + `{"id":"team-uuid-1","name":"Engineering","key":"ENG","description":"core eng"},` + + `{"id":"team-uuid-2","name":"Design","key":"DSG","description":""}` + + `],"pageInfo":{"hasNextPage":false,"endCursor":""}}}}` + assert.NoError(t, json.Unmarshal([]byte(body), &response)) + + entries := mapLinearTeamsToScopeEntries(response) + assert.Len(t, entries, 2) + + assert.Equal(t, api.RAS_ENTRY_TYPE_SCOPE, entries[0].Type) + assert.Nil(t, entries[0].ParentId) + assert.Equal(t, "team-uuid-1", entries[0].Id) + assert.Equal(t, "Engineering", entries[0].Name) + assert.Equal(t, "Engineering", entries[0].FullName) + // the scope payload must carry the team id used as the scope's primary key + assert.NotNil(t, entries[0].Data) + assert.Equal(t, "team-uuid-1", entries[0].Data.TeamId) + assert.Equal(t, "ENG", entries[0].Data.Key) + + assert.Equal(t, "team-uuid-2", entries[1].Id) + assert.Equal(t, "Design", entries[1].Name) +} + +func TestNextPageFrom(t *testing.T) { + var more linearTeamsGraphqlResponse + more.Data.Teams.PageInfo.HasNextPage = true + more.Data.Teams.PageInfo.EndCursor = "cursor-abc" + next := nextPageFrom(more) + assert.NotNil(t, next) + assert.Equal(t, "cursor-abc", next.Cursor) + + // no further pages -> nil, so the helper stops paginating + var last linearTeamsGraphqlResponse + last.Data.Teams.PageInfo.HasNextPage = false + last.Data.Teams.PageInfo.EndCursor = "cursor-xyz" + assert.Nil(t, nextPageFrom(last)) +} diff --git a/backend/plugins/linear/api/scope_api.go b/backend/plugins/linear/api/scope_api.go new file mode 100644 index 00000000000..9871d1d2bfe --- /dev/null +++ b/backend/plugins/linear/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.LinearTeam] +type ScopeDetail api.ScopeDetail[models.LinearTeam, models.LinearScopeConfig] + +// PutScopes create or update linear teams +// @Summary create or update linear teams +// @Description Create or update linear teams +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to linear team +// @Summary patch to linear team +// @Description patch to linear team +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Param scope body models.LinearTeam true "json" +// @Success 200 {object} models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get linear teams +// @Summary get linear teams +// @Description get linear teams +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one linear team +// @Summary get one linear team +// @Description get one linear team +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/linear/api/scope_config_api.go b/backend/plugins/linear/api/scope_config_api.go new file mode 100644 index 00000000000..15be82a4dff --- /dev/null +++ b/backend/plugins/linear/api/scope_config_api.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for Linear +// @Summary create scope config for Linear +// @Description create scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for Linear +// @Summary update scope config for Linear +// @Description update scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/linear +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/linear/e2e/account_test.go b/backend/plugins/linear/e2e/account_test.go new file mode 100644 index 00000000000..9cb8f3a8e15 --- /dev/null +++ b/backend/plugins/linear/e2e/account_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearAccountDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_accounts.csv", "_raw_linear_accounts") + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearAccount{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(crossdomain.Account{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/board_test.go b/backend/plugins/linear/e2e/board_test.go new file mode 100644 index 00000000000..eb4e0487323 --- /dev/null +++ b/backend/plugins/linear/e2e/board_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearBoardDataFlow verifies that a Linear team scope is converted into a +// domain ticket.Board, keyed identically to the board_id that board_issues +// already reference (boardIdGen over LinearTeam). Without this, the boards table +// is empty and board-scoped dashboards return no data. +func TestLinearBoardDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the team scope lives in _tool_linear_teams (populated via the scope API) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_teams.csv", &models.LinearTeam{}) + + // convert: team scope -> domain board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertTeamsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/comment_test.go b/backend/plugins/linear/e2e/comment_test.go new file mode 100644 index 00000000000..f8178401f13 --- /dev/null +++ b/backend/plugins/linear/e2e/comment_test.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCommentDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the comment convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_comments.csv", "_raw_linear_comments") + dataflowTester.FlushTabler(&models.LinearComment{}) + dataflowTester.Subtask(tasks.ExtractCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueComment{}) + dataflowTester.Subtask(tasks.ConvertCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/cycle_test.go b/backend/plugins/linear/e2e/cycle_test.go new file mode 100644 index 00000000000..0a50789fa7d --- /dev/null +++ b/backend/plugins/linear/e2e/cycle_test.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCycleDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // issues drive sprint_issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_cycles.csv", "_raw_linear_cycles") + dataflowTester.FlushTabler(&models.LinearCycle{}) + dataflowTester.Subtask(tasks.ExtractCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearCycle{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_cycles.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: cycles -> sprints + board_sprints + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Sprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardSprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: issues -> sprint_issues + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_leadtime_test.go b/backend/plugins/linear/e2e/issue_history_leadtime_test.go new file mode 100644 index 00000000000..7d4a4c9798a --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_leadtime_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueHistoryLeadTime proves lead time is derived from the recorded +// state transitions. issue-1 was started on 2026-05-02 and completed on +// 2026-05-03 (1440 min of active cycle time), even though its createdAt -> +// resolutionDate span is 2880 min. ConvertIssues sets the coarse 2880 fallback; +// ConvertIssueHistory must then refine it to 1440 from the history. Issues +// without the required transitions keep the fallback (issue-4 = 1440). +func TestLinearIssueHistoryLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // flush accounts so assignee/creator names stay empty in this lead-time- + // focused test regardless of other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + + // seed issues + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // seed history + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + + // ConvertIssues sets the createdAt -> resolutionDate fallback ... + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + + // ... then ConvertIssueHistory refines lead time from the transitions. + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_history_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_test.go b/backend/plugins/linear/e2e/issue_history_test.go new file mode 100644 index 00000000000..c8942141b2b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_test.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueHistoryDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the history convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssueHistory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_history.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_changelogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go new file mode 100644 index 00000000000..783820f696d --- /dev/null +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueNegativeLeadTime guards the lead-time fallback against +// resolution timestamps that precede the creation timestamp (clock skew or +// migrated/imported issues). A negative duration must NOT be cast to uint -- +// doing so wraps to a huge bogus value. The expected behaviour is that no lead +// time is derived (lead_time_minutes stays empty). +func TestLinearIssueNegativeLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // extraction: raw -> tool layer + // Flush accounts so this lead-time-focused test is independent of any + // account rows left behind by other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_negative_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_longtitle_test.go b/backend/plugins/linear/e2e/issue_longtitle_test.go new file mode 100644 index 00000000000..f85062cdb8b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_longtitle_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" + "github.com/stretchr/testify/assert" +) + +// TestLinearIssueLongTitle guards against truncation/insert failure for long +// issue titles and URLs. Linear titles can exceed 255 chars (and the issue URL +// embeds a title slug), which overflowed the old varchar(255) columns and +// failed extraction with "Data too long for column 'title'". The columns are +// now untyped (longtext), matching the domain issues.title and jira's tool +// summary. +func TestLinearIssueLongTitle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ConnectionId: 1, TeamId: "team-1"}, + } + + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_long_title.csv", "_raw_linear_issues") + // must not error with "Data too long for column 'title'" + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + var issue models.LinearIssue + err := dataflowTester.Dal.First(&issue, dal.Where("connection_id = ? AND id = ?", 1, "issue-longtitle")) + assert.NoError(t, err) + assert.Len(t, issue.Title, 300, "full 300-char title must be stored untruncated") +} diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go new file mode 100644 index 00000000000..5f98aad8a61 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer (issues + inline labels) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(models.LinearIssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // accounts must be present so the convertor can resolve assignee/creator + // display names and emit issue_assignees rows. + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // verify conversion: tool layer -> domain layer (issues + board_issues + issue_assignees) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/label_test.go b/backend/plugins/linear/e2e/label_test.go new file mode 100644 index 00000000000..4861f6197da --- /dev/null +++ b/backend/plugins/linear/e2e/label_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearLabelDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // labels are produced inline by the issue extractor + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueLabel{}) + dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv new file mode 100644 index 00000000000..785c47d21b2 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-1"",""Name"":""alice"",""DisplayName"":""Alice Anderson"",""Email"":""alice@example.com"",""AvatarUrl"":""https://linear.app/avatars/alice.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-2"",""Name"":""bob"",""DisplayName"":""Bob Brown"",""Email"":""bob@example.com"",""AvatarUrl"":""https://linear.app/avatars/bob.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-3"",""Name"":""carol"",""DisplayName"":""Carol Clark"",""Email"":""carol@example.com"",""AvatarUrl"":""https://linear.app/avatars/carol.png"",""Active"":false}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv new file mode 100644 index 00000000000..6d6fc21e6a8 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv new file mode 100644 index 00000000000..8eb59ddb145 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-1"",""Number"":1,""Name"":"""",""StartsAt"":""2026-04-20T00:00:00Z"",""EndsAt"":""2026-05-04T00:00:00Z"",""CompletedAt"":""2026-05-04T00:00:00Z""}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-2"",""Number"":2,""Name"":""Sprint 2"",""StartsAt"":""2026-05-04T00:00:00Z"",""EndsAt"":""2026-05-18T00:00:00Z"",""CompletedAt"":null}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv new file mode 100644 index 00000000000..d1cef3c37ab --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv new file mode 100644 index 00000000000..f49f41c73b6 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-1"",""Identifier"":""ENG-1"",""Number"":1,""Title"":""Fix login bug"",""Description"":""Users cannot log in"",""Url"":""https://linear.app/eng/issue/ENG-1"",""Priority"":1,""Estimate"":3,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-03T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":""2026-05-03T00:00:00Z"",""CanceledAt"":null,""State"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""},""Assignee"":{""Id"":""user-1""},""Creator"":{""Id"":""user-2""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l1"",""Name"":""Bug""},{""Id"":""l2"",""Name"":""P1""}]}}",https://api.linear.app/graphql,null,2026-05-03 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-2"",""Identifier"":""ENG-2"",""Number"":2,""Title"":""Add dark mode"",""Description"":""Theme support"",""Url"":""https://linear.app/eng/issue/ENG-2"",""Priority"":2,""Estimate"":5,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l3"",""Name"":""Feature""}]}}",https://api.linear.app/graphql,null,2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-3"",""Identifier"":""ENG-3"",""Number"":3,""Title"":""Investigate flakiness"",""Description"":"""",""Url"":""https://linear.app/eng/issue/ENG-3"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-4"",""Identifier"":""ENG-4"",""Number"":4,""Title"":""Deprecated feature"",""Description"":""No longer needed"",""Url"":""https://linear.app/eng/issue/ENG-4"",""Priority"":3,""Estimate"":2,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T12:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-02T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":{""Id"":""user-3""},""Creator"":{""Id"":""user-1""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-02 12:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-5"",""Identifier"":""ENG-5"",""Number"":5,""Title"":""Write docs"",""Description"":""User guide"",""Url"":""https://linear.app/eng/issue/ENG-5"",""Priority"":4,""Estimate"":1,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T06:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""Assignee"":null,""Creator"":{""Id"":""user-2""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 06:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv new file mode 100644 index 00000000000..4842c33a61c --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-longtitle"",""Identifier"":""ENG-LONG"",""Number"":900,""Title"":""TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"",""Description"":""long title row"",""Url"":""https://linear.app/eng/issue/ENG-LONG/tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":null,""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv new file mode 100644 index 00000000000..f63750db475 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-neg"",""Identifier"":""ENG-NEG"",""Number"":99,""Title"":""Imported issue with skewed timestamps"",""Description"":""canceledAt precedes createdAt"",""Url"":""https://linear.app/eng/issue/ENG-NEG"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-10T00:00:00Z"",""UpdatedAt"":""2026-05-10T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-09T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-10 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv new file mode 100644 index 00000000000..72c0ddc877b --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog"",""Color"":""#bec2c8"",""Position"":0}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted"",""Color"":""#e2e2e2"",""Position"":1}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started"",""Color"":""#f2c94c"",""Position"":2}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed"",""Color"":""#5e6ad2"",""Position"":3}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled"",""Color"":""#95a2b3"",""Position"":4}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv new file mode 100644 index 00000000000..db07f6b4cf7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv @@ -0,0 +1,4 @@ +connection_id,id,name,display_name,email,avatar_url,active +1,user-1,alice,Alice Anderson,alice@example.com,https://linear.app/avatars/alice.png,1 +1,user-2,bob,Bob Brown,bob@example.com,https://linear.app/avatars/bob.png,1 +1,user-3,carol,Carol Clark,carol@example.com,https://linear.app/avatars/carol.png,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv new file mode 100644 index 00000000000..aec4878ecb1 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,body,author_id +1,comment-1,issue-1,Looking into this,user-2 +1,comment-2,issue-1,Fixed in PR 42,user-1 +1,comment-3,issue-2,Any update?,user-1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv new file mode 100644 index 00000000000..9e95c44d986 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv @@ -0,0 +1,3 @@ +connection_id,id,team_id,number,name,starts_at,ends_at,completed_at +1,cycle-1,team-1,1,,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00 +1,cycle-2,team-1,2,Sprint 2,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv new file mode 100644 index 00000000000..bc8fe9ec7ba --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,actor_id,from_state_id,from_state_name,from_state_type,to_state_id,to_state_name,to_state_type +1,hist-1,issue-1,user-2,state-backlog,Backlog,backlog,state-todo,Todo,unstarted +1,hist-2,issue-1,user-1,state-todo,Todo,unstarted,state-inprogress,In Progress,started +1,hist-3,issue-1,user-1,state-inprogress,In Progress,started,state-done,Done,completed diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv new file mode 100644 index 00000000000..20da8a076b0 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv @@ -0,0 +1,4 @@ +connection_id,issue_id,label_name +1,issue-1,Bug +1,issue-1,P1 +1,issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv new file mode 100644 index 00000000000..7c35d53d39e --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv new file mode 100644 index 00000000000..1541f9c95e4 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,team_id,name,key,description +1,0,team-1,Engineering,ENG,Core engineering team diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv new file mode 100644 index 00000000000..bd1d3443ae9 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,name,type,color,position +1,state-backlog,team-1,Backlog,backlog,#bec2c8,0 +1,state-canceled,team-1,Canceled,canceled,#95a2b3,4 +1,state-done,team-1,Done,completed,#5e6ad2,3 +1,state-inprogress,team-1,In Progress,started,#f2c94c,2 +1,state-todo,team-1,Todo,unstarted,#e2e2e2,1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv new file mode 100644 index 00000000000..12c2066d191 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,4 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status +linear:LinearAccount:1:user-1,alice@example.com,Alice Anderson,alice,https://linear.app/avatars/alice.png,,,1 +linear:LinearAccount:1:user-2,bob@example.com,Bob Brown,bob,https://linear.app/avatars/bob.png,,,1 +linear:LinearAccount:1:user-3,carol@example.com,Carol Clark,carol,https://linear.app/avatars/carol.png,,,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9a7d32f9477 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-1 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-2 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-3 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-4 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-5 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..ff0cfe33dc5 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,3 @@ +board_id,sprint_id +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-1 +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-2 diff --git a/backend/plugins/linear/e2e/snapshot_tables/boards.csv b/backend/plugins/linear/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4548d25c385 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +linear:LinearTeam:1:team-1,Engineering,Core engineering team,,,linear diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..38f569f7094 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,3 @@ +issue_id,assignee_id,assignee_name +linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,Alice Anderson +linear:LinearIssue:1:issue-4,linear:LinearAccount:1:user-3,Carol Clark diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..9d53280010f --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,4 @@ +id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date +linear:LinearIssueHistory:1:hist-1,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-2,,state,status,Backlog,Todo,TODO,TODO,2026-05-01T08:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-2,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,Todo,In Progress,TODO,IN_PROGRESS,2026-05-02T00:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-3,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,In Progress,Done,IN_PROGRESS,DONE,2026-05-03T00:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv new file mode 100644 index 00000000000..925e84e5648 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv @@ -0,0 +1,4 @@ +id,issue_id,body,account_id,created_date,updated_date +linear:LinearComment:1:comment-1,linear:LinearIssue:1:issue-1,Looking into this,linear:LinearAccount:1:user-2,2026-05-02T10:00:00.000+00:00,2026-05-02T10:00:00.000+00:00 +linear:LinearComment:1:comment-2,linear:LinearIssue:1:issue-1,Fixed in PR 42,linear:LinearAccount:1:user-1,2026-05-03T09:00:00.000+00:00,2026-05-03T09:30:00.000+00:00 +linear:LinearComment:1:comment-3,linear:LinearIssue:1:issue-2,Any update?,linear:LinearAccount:1:user-1,2026-05-02T11:00:00.000+00:00,2026-05-02T11:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv new file mode 100644 index 00000000000..e7c911e0005 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv @@ -0,0 +1,4 @@ +issue_id,label_name +linear:LinearIssue:1:issue-1,Bug +linear:LinearIssue:1:issue-1,P1 +linear:LinearIssue:1:issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..f55c39e0eb8 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv new file mode 100644 index 00000000000..d11625e3c3d --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv new file mode 100644 index 00000000000..4375d47901b --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-neg,https://linear.app/eng/issue/ENG-NEG,,ENG-NEG,Imported issue with skewed timestamps,canceledAt precedes createdAt,,REQUIREMENT,,DONE,Canceled,,2026-05-09T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..b13b4f85210 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,3 @@ +sprint_id,issue_id +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-1 +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-4 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv new file mode 100644 index 00000000000..1074725b86c --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv @@ -0,0 +1 @@ +sprint_id,issue_id diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..a09ab2ffe70 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,3 @@ +id,name,url,status,started_date,ended_date,completed_date,original_board_id +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 diff --git a/backend/plugins/linear/e2e/sprint_issue_test.go b/backend/plugins/linear/e2e/sprint_issue_test.go new file mode 100644 index 00000000000..81e4508e43f --- /dev/null +++ b/backend/plugins/linear/e2e/sprint_issue_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearSprintIssueStaleCycle guards against stale sprint_issues rows when +// an issue is moved out of a cycle. Sprint membership is derived from the +// issue's cycle_id; once that empties on re-collection, the issue must no +// longer appear in sprint_issues from a previous run. +func TestLinearSprintIssueStaleCycle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // seed issues: issue-1 and issue-4 both belong to cycle-1 + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // first conversion: both issues land in the sprint + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + + // every issue is moved out of its cycle; on re-collection cycle_id empties. + // The second conversion then produces zero sprint issues, which is the case + // the batch divider's lazy delete fails to cover. + if err := dataflowTester.Dal.Exec( + "UPDATE _tool_linear_issues SET cycle_id = '' WHERE connection_id = ? AND team_id = ?", 1, "team-1", + ); err != nil { + t.Fatal(err) + } + + // second conversion (no flush) must drop ALL stale sprint_issues rows + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues_after_leaving_cycle.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/workflow_state_test.go b/backend/plugins/linear/e2e/workflow_state_test.go new file mode 100644 index 00000000000..5af5c4a66b9 --- /dev/null +++ b/backend/plugins/linear/e2e/workflow_state_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearWorkflowStateDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_workflow_states.csv", "_raw_linear_workflow_states") + dataflowTester.FlushTabler(&models.LinearWorkflowState{}) + dataflowTester.Subtask(tasks.ExtractWorkflowStatesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearWorkflowState{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_workflow_states.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go new file mode 100644 index 00000000000..bc64218ff61 --- /dev/null +++ b/backend/plugins/linear/impl/impl.go @@ -0,0 +1,220 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Linear)(nil) + +type Linear struct{} + +func (p Linear) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Linear) Description() string { + return "To collect and enrich data from Linear" +} + +func (p Linear) Name() string { + return "linear" +} + +func (p Linear) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linear" +} + +func (p Linear) Connection() dal.Tabler { + return &models.LinearConnection{} +} + +func (p Linear) Scope() plugin.ToolLayerScope { + return &models.LinearTeam{} +} + +func (p Linear) ScopeConfig() dal.Tabler { + return &models.LinearScopeConfig{} +} + +func (p Linear) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linear) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.LinearConnection{}, + &models.LinearTeam{}, + &models.LinearScopeConfig{}, + &models.LinearAccount{}, + &models.LinearIssue{}, + &models.LinearComment{}, + &models.LinearIssueLabel{}, + &models.LinearWorkflowState{}, + &models.LinearCycle{}, + &models.LinearIssueHistory{}, + } +} + +func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectAccountsMeta, + tasks.ExtractAccountsMeta, + tasks.CollectWorkflowStatesMeta, + tasks.ExtractWorkflowStatesMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, + tasks.CollectCommentsMeta, + tasks.ExtractCommentsMeta, + tasks.CollectCyclesMeta, + tasks.ExtractCyclesMeta, + tasks.CollectIssueHistoryMeta, + tasks.ExtractIssueHistoryMeta, + tasks.ConvertTeamsMeta, + tasks.ConvertAccountsMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, + tasks.ConvertCommentsMeta, + tasks.ConvertCyclesMeta, + tasks.ConvertSprintIssuesMeta, + tasks.ConvertIssueHistoryMeta, + } +} + +func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.LinearOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode Linear options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("linear connectionId is invalid") + } + if op.TeamId == "" { + return nil, errors.BadInput.New("linear teamId is required") + } + + connection := &models.LinearConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Linear plugin") + } + + graphqlClient, err := tasks.NewLinearGraphqlClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create Linear GraphQL client") + } + + taskData := &tasks.LinearTaskData{ + Options: &op, + GraphqlClient: graphqlClient, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + } +} + +func (p Linear) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Linear) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.LinearTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.GraphqlClient != nil { + data.GraphqlClient.Release() + } + return nil +} diff --git a/backend/plugins/linear/impl/impl_test.go b/backend/plugins/linear/impl/impl_test.go new file mode 100644 index 00000000000..d11b3ec5ae1 --- /dev/null +++ b/backend/plugins/linear/impl/impl_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestApiResourcesRegistersRemoteScopes guards the remote-scopes endpoints used +// by the config UI to enumerate Linear teams. +func TestApiResourcesRegistersRemoteScopes(t *testing.T) { + resources := Linear{}.ApiResources() + + remoteScopes, ok := resources["connections/:connectionId/remote-scopes"] + assert.True(t, ok, "remote-scopes route must be registered") + assert.NotNil(t, remoteScopes["GET"], "remote-scopes must handle GET") + + proxy, ok := resources["connections/:connectionId/proxy/rest/*path"] + assert.True(t, ok, "proxy route must be registered") + assert.NotNil(t, proxy["GET"], "proxy must handle GET") +} diff --git a/backend/plugins/linear/linear.go b/backend/plugins/linear/linear.go new file mode 100644 index 00000000000..2cb64a49470 --- /dev/null +++ b/backend/plugins/linear/linear.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Linear //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "linear"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "linear connection id") + teamId := cmd.Flags().StringP("team", "t", "", "linear team id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("team") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "teamId": *teamId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linear/models/account.go b/backend/plugins/linear/models/account.go new file mode 100644 index 00000000000..7d1c9376f98 --- /dev/null +++ b/backend/plugins/linear/models/account.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearAccount is a Linear user (tool layer), converted to crossdomain.Account. +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Name string `gorm:"type:varchar(255)" json:"name"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Email string `gorm:"type:varchar(255)" json:"email"` + AvatarUrl string `gorm:"type:varchar(255)" json:"avatarUrl"` + Active bool `json:"active"` + common.NoPKModel +} + +func (LinearAccount) TableName() string { + return "_tool_linear_accounts" +} diff --git a/backend/plugins/linear/models/connection.go b/backend/plugins/linear/models/connection.go new file mode 100644 index 00000000000..6f63e431a11 --- /dev/null +++ b/backend/plugins/linear/models/connection.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// LinearConn holds the essential information to connect to the Linear API. +// Linear authenticates with a personal API key passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. +type LinearConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the Linear API. +func (lc *LinearConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", lc.Token) + return nil +} + +func (lc *LinearConn) Sanitize() LinearConn { + lc.Token = utils.SanitizeString(lc.Token) + return *lc +} + +// LinearConnection holds LinearConn plus ID/Name for database storage. +type LinearConnection struct { + helper.BaseConnection `mapstructure:",squash"` + LinearConn `mapstructure:",squash"` +} + +func (connection LinearConnection) Sanitize() LinearConnection { + connection.LinearConn = connection.LinearConn.Sanitize() + return connection +} + +func (connection *LinearConnection) MergeFromRequest(target *LinearConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (LinearConnection) TableName() string { + return "_tool_linear_connections" +} diff --git a/backend/plugins/linear/models/cycle.go b/backend/plugins/linear/models/cycle.go new file mode 100644 index 00000000000..4f61370dbb2 --- /dev/null +++ b/backend/plugins/linear/models/cycle.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearCycle is a Linear cycle (sprint-equivalent), converted to ticket.Sprint. +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Number int `json:"number"` + Name string `gorm:"type:varchar(255)" json:"name"` + StartsAt *time.Time `json:"startsAt"` + EndsAt *time.Time `json:"endsAt"` + CompletedAt *time.Time `json:"completedAt"` + common.NoPKModel +} + +func (LinearCycle) TableName() string { + return "_tool_linear_cycles" +} diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go new file mode 100644 index 00000000000..991585de956 --- /dev/null +++ b/backend/plugins/linear/models/issue.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssue is the tool-layer representation of a Linear issue. +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `json:"title"` + Description string `json:"description"` + Url string `json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` + common.NoPKModel +} + +func (LinearIssue) TableName() string { + return "_tool_linear_issues" +} diff --git a/backend/plugins/linear/models/issue_comment.go b/backend/plugins/linear/models/issue_comment.go new file mode 100644 index 00000000000..8ba18505038 --- /dev/null +++ b/backend/plugins/linear/models/issue_comment.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearComment is the tool-layer representation of a comment on a Linear issue. +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + Body string `json:"body"` + AuthorId string `gorm:"type:varchar(255)" json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + common.NoPKModel +} + +func (LinearComment) TableName() string { + return "_tool_linear_comments" +} diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go new file mode 100644 index 00000000000..027f6cb4a07 --- /dev/null +++ b/backend/plugins/linear/models/issue_history.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueHistory is a single entry in a Linear issue's history, used to +// build domain-layer changelogs and derive lead/cycle time. +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + ActorId string `gorm:"type:varchar(255)" json:"actorId"` + FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` + FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + FromStateType string `gorm:"type:varchar(100)" json:"fromStateType"` + ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` + ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + ToStateType string `gorm:"type:varchar(100)" json:"toStateType"` + CreatedAt time.Time `gorm:"index" json:"createdAt"` + common.NoPKModel +} + +func (LinearIssueHistory) TableName() string { + return "_tool_linear_issue_history" +} diff --git a/backend/plugins/linear/models/issue_label.go b/backend/plugins/linear/models/issue_label.go new file mode 100644 index 00000000000..76a4517cb7b --- /dev/null +++ b/backend/plugins/linear/models/issue_label.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueLabel joins a Linear issue to one of its labels. Labels are +// collected inline with issues, so there is no separate label collector. +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)" json:"issueId"` + LabelName string `gorm:"primaryKey;type:varchar(255)" json:"labelName"` + common.NoPKModel +} + +func (LinearIssueLabel) TableName() string { + return "_tool_linear_issue_labels" +} diff --git a/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go new file mode 100644 index 00000000000..9bb577343be --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.LinearConnection{}, + &archived.LinearTeam{}, + &archived.LinearScopeConfig{}, + &archived.LinearAccount{}, + &archived.LinearIssue{}, + &archived.LinearComment{}, + &archived.LinearIssueLabel{}, + &archived.LinearWorkflowState{}, + &archived.LinearCycle{}, + &archived.LinearIssueHistory{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260601000001 +} + +func (*addInitTables) Name() string { + return "linear init schemas" +} diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..1c28035387a --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -0,0 +1,166 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/linear/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type LinearConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (LinearConnection) TableName() string { return "_tool_linear_connections" } + +type LinearTeam struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + TeamId string `json:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + Key string `json:"key" gorm:"type:varchar(255)"` + Description string `json:"description"` +} + +func (LinearTeam) TableName() string { return "_tool_linear_teams" } + +type LinearScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` +} + +func (LinearScopeConfig) TableName() string { return "_tool_linear_scope_configs" } + +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + DisplayName string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + Active bool + archived.NoPKModel +} + +func (LinearAccount) TableName() string { return "_tool_linear_accounts" } + +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string + Description string + Url string + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + archived.NoPKModel +} + +func (LinearIssue) TableName() string { return "_tool_linear_issues" } + +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + Body string + AuthorId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearComment) TableName() string { return "_tool_linear_comments" } + +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + LabelName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (LinearIssueLabel) TableName() string { return "_tool_linear_issue_labels" } + +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Type string `gorm:"type:varchar(100)"` + Color string `gorm:"type:varchar(50)"` + Position float64 + archived.NoPKModel +} + +func (LinearWorkflowState) TableName() string { return "_tool_linear_workflow_states" } + +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Number int + Name string `gorm:"type:varchar(255)"` + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time + archived.NoPKModel +} + +func (LinearCycle) TableName() string { return "_tool_linear_cycles" } + +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + ActorId string `gorm:"type:varchar(255)"` + FromStateId string `gorm:"type:varchar(255)"` + FromStateName string `gorm:"type:varchar(255)"` + FromStateType string `gorm:"type:varchar(100)"` + ToStateId string `gorm:"type:varchar(255)"` + ToStateName string `gorm:"type:varchar(255)"` + ToStateType string `gorm:"type:varchar(100)"` + CreatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearIssueHistory) TableName() string { return "_tool_linear_issue_history" } diff --git a/backend/plugins/linear/models/migrationscripts/register.go b/backend/plugins/linear/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/linear/models/scope_config.go b/backend/plugins/linear/models/scope_config.go new file mode 100644 index 00000000000..1af91db58b0 --- /dev/null +++ b/backend/plugins/linear/models/scope_config.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearScopeConfig is intentionally minimal: Linear's WorkflowState.type maps +// deterministically to TODO/IN_PROGRESS/DONE, so no user status mapping is +// required. It is reserved for future label-based issue-type mapping. +type LinearScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (LinearScopeConfig) TableName() string { + return "_tool_linear_scope_configs" +} + +func (sc *LinearScopeConfig) SetConnectionId(c *LinearScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/linear/models/team.go b/backend/plugins/linear/models/team.go new file mode 100644 index 00000000000..d6226208266 --- /dev/null +++ b/backend/plugins/linear/models/team.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*LinearTeam)(nil) + +// LinearTeam is the data-source scope for the Linear plugin. A Linear Team +// owns issues, cycles, workflow states and labels, mapping cleanly to a +// DevLake domain-layer ticket.Board. +type LinearTeam struct { + common.Scope `mapstructure:",squash"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description"` +} + +func (t LinearTeam) ScopeId() string { + return t.TeamId +} + +func (t LinearTeam) ScopeName() string { + return t.Name +} + +func (t LinearTeam) ScopeFullName() string { + return t.Name +} + +func (t LinearTeam) ScopeParams() interface{} { + return &LinearApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (LinearTeam) TableName() string { + return "_tool_linear_teams" +} + +// LinearApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_linear_* table. +type LinearApiParams struct { + ConnectionId uint64 + TeamId string +} diff --git a/backend/plugins/linear/models/workflow_state.go b/backend/plugins/linear/models/workflow_state.go new file mode 100644 index 00000000000..9273f183c91 --- /dev/null +++ b/backend/plugins/linear/models/workflow_state.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearWorkflowState is a Linear team's workflow state. Its Type +// (backlog|unstarted|started|completed|canceled) drives issue status mapping. +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Type string `gorm:"type:varchar(100)" json:"type"` + Color string `gorm:"type:varchar(50)" json:"color"` + Position float64 `json:"position"` + common.NoPKModel +} + +func (LinearWorkflowState) TableName() string { + return "_tool_linear_workflow_states" +} diff --git a/backend/plugins/linear/tasks/account_collector.go b/backend/plugins/linear/tasks/account_collector.go new file mode 100644 index 00000000000..88fa181f002 --- /dev/null +++ b/backend/plugins/linear/tasks/account_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ACCOUNTS_TABLE = "linear_accounts" + +// GraphqlQueryAccountWrapper is the paginated `users` query envelope. +type GraphqlQueryAccountWrapper struct { + Users struct { + Nodes []GraphqlQueryAccount + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"users(first: $pageSize, after: $skipCursor)"` +} + +type GraphqlQueryAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string + Active bool +} + +var CollectAccountsMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectAccounts, + EnabledByDefault: true, + Description: "Collect workspace users from the Linear GraphQL API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectAccounts + +func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryAccountWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryAccountWrapper) + return query.Users.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryAccountWrapper) + for _, account := range query.Users.Nodes { + messages = append(messages, errors.Must1(json.Marshal(account))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/account_convertor.go b/backend/plugins/linear/tasks/account_convertor.go new file mode 100644 index 00000000000..d3c8efd3686 --- /dev/null +++ b/backend/plugins/linear/tasks/account_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_accounts into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.LinearAccount{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertAccounts + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearAccount{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearAccount{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + account := inputRow.(*models.LinearAccount) + status := 1 + if !account.Active { + status = 0 + } + fullName := account.Name + if account.DisplayName != "" { + fullName = account.DisplayName + } + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, account.Id), + }, + UserName: account.Name, + FullName: fullName, + Email: account.Email, + AvatarUrl: account.AvatarUrl, + Status: status, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/account_extractor.go b/backend/plugins/linear/tasks/account_extractor.go new file mode 100644 index 00000000000..19cc86a04cc --- /dev/null +++ b/backend/plugins/linear/tasks/account_extractor.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractAccountsMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractAccounts, + EnabledByDefault: true, + Description: "Extract raw user data into tool layer table _tool_linear_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractAccounts + +func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiAccount := &GraphqlQueryAccount{} + if err := errors.Convert(json.Unmarshal(row.Data, apiAccount)); err != nil { + return nil, err + } + if apiAccount.Id == "" { + return nil, nil + } + account := &models.LinearAccount{ + ConnectionId: data.Options.ConnectionId, + Id: apiAccount.Id, + Name: apiAccount.Name, + DisplayName: apiAccount.DisplayName, + Email: apiAccount.Email, + AvatarUrl: apiAccount.AvatarUrl, + Active: apiAccount.Active, + } + return []interface{}{account}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/api_client.go b/backend/plugins/linear/tasks/api_client.go new file mode 100644 index 00000000000..0ca6b8da25a --- /dev/null +++ b/backend/plugins/linear/tasks/api_client.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + gocontext "context" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +// linearTransport injects the Linear personal API key into every request. +// Linear expects the key verbatim in the Authorization header (no Bearer prefix). +type linearTransport struct { + token string + base http.RoundTripper +} + +func (t *linearTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", t.token) + return t.base.RoundTrip(req) +} + +// graphqlQueryViewer is a tiny probe used to validate connectivity / liveness. +type graphqlQueryViewer struct { + Viewer struct { + Id graphql.String + } +} + +// defaultRateLimitPerHour is Linear's documented per-API-key request budget. +// Used when the connection does not override RateLimitPerHour. +const defaultRateLimitPerHour = 1500 + +// NewLinearGraphqlClient builds a rate-limited async GraphQL client for the +// Linear API from the given connection. +func NewLinearGraphqlClient(taskCtx plugin.TaskContext, connection *models.LinearConnection) (*helper.GraphqlAsyncClient, errors.Error) { + httpClient, err := newLinearHttpClient(connection) + if err != nil { + return nil, err + } + + endpoint := connection.Endpoint + if endpoint == "" { + endpoint = "https://api.linear.app/graphql" + } + client := graphql.NewClient(endpoint, httpClient) + + rateLimitPerHour := connection.RateLimitPerHour + if rateLimitPerHour <= 0 { + rateLimitPerHour = defaultRateLimitPerHour + } + + return helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + func(ctx gocontext.Context, c *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, e errors.Error) { + // Linear does not expose rate-limit info in the GraphQL body (it uses + // HTTP response headers), so we probe liveness and pace against the + // configured hourly budget. The async client self-throttles from here. + var q graphqlQueryViewer + dataErrors, queryErr := errors.Convert01(c.Query(ctx, &q, nil)) + if queryErr != nil { + return 0, nil, queryErr + } + if len(dataErrors) > 0 { + return 0, nil, errors.Default.Wrap(dataErrors[0], "linear graphql viewer query failed") + } + reset := time.Now().Add(1 * time.Hour) + logger.Info("linear graphql client initialized, pacing against %d req/hour", rateLimitPerHour) + return rateLimitPerHour, &reset, nil + }) +} + +func newLinearHttpClient(connection *models.LinearConnection) (*http.Client, errors.Error) { + base := http.DefaultTransport + if proxy := connection.Proxy; proxy != "" { + pu, err := url.Parse(proxy) + if err != nil { + return nil, errors.BadInput.Wrap(err, "malformed proxy url") + } + base = &http.Transport{Proxy: http.ProxyURL(pu)} + } + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &linearTransport{token: connection.Token, base: base}, + }, nil +} diff --git a/backend/plugins/linear/tasks/board_convertor.go b/backend/plugins/linear/tasks/board_convertor.go new file mode 100644 index 00000000000..9366efed2ab --- /dev/null +++ b/backend/plugins/linear/tasks/board_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// RAW_TEAMS_TABLE labels the raw-data lineage for the team-scope-derived board. +// Teams are added as scopes (no collector), so this is a logical tag only. +const RAW_TEAMS_TABLE = "linear_teams" + +var ConvertTeamsMeta = plugin.SubTaskMeta{ + Name: "Convert Teams", + EntryPoint: ConvertTeams, + EnabledByDefault: true, + Description: "Convert the Linear team scope (_tool_linear_teams) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearTeam{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTeams + +func ConvertTeams(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the issue/sprint convertors so the + // board joins to the board_issues/sprint_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearTeam{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_TEAMS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearTeam{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + team := inputRow.(*models.LinearTeam) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, team.TeamId)}, + Name: team.Name, + Description: team.Description, + Type: "linear", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go new file mode 100644 index 00000000000..14e801ff9e3 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -0,0 +1,148 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_COMMENTS_TABLE = "linear_comments" + +// SimpleLinearIssue is the iterator element used to drive per-issue collection +// of child resources (comments, history). Its JSON form is stored in the raw +// row's `input` column so extractors can recover the owning issue id. +type SimpleLinearIssue struct { + // Id is populated by the DalCursorIterator (the _tool_linear_issues.id column) + // when driving per-issue child collection. + Id string `json:"Id"` + // IssueId is populated when parsing a raw row's `input` column: the GraphQL + // collector stores the query variables there (which carry `issueId`), not the + // iterator element. OwningIssueId resolves whichever is present. + IssueId string `json:"issueId" gorm:"-"` +} + +// OwningIssueId returns the issue id this child row belongs to, tolerating both +// the iterator element shape ({"Id":...}) and the collector's stored variables +// shape ({"issueId":...}). +func (s SimpleLinearIssue) OwningIssueId() string { + if s.IssueId != "" { + return s.IssueId + } + return s.Id +} + +// GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. +type GraphqlQueryCommentWrapper struct { + Issue struct { + Comments struct { + Nodes []GraphqlQueryComment + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"comments(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryComment struct { + Id string + Body string + CreatedAt time.Time + UpdatedAt time.Time + User *struct{ Id string } +} + +var CollectCommentsMeta = plugin.SubTaskMeta{ + Name: "Collect Comments", + EntryPoint: CollectComments, + EnabledByDefault: true, + Description: "Collect comments for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectComments + +func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's comments cannot have changed, so re-fetching every + // issue each run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCommentWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCommentWrapper) + return query.Issue.Comments.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCommentWrapper) + for _, comment := range query.Issue.Comments.Nodes { + messages = append(messages, errors.Must1(json.Marshal(comment))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_convertor.go b/backend/plugins/linear/tasks/comment_convertor.go new file mode 100644 index 00000000000..8c4729f0e5c --- /dev/null +++ b/backend/plugins/linear/tasks/comment_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCommentsMeta = plugin.SubTaskMeta{ + Name: "Convert Comments", + EntryPoint: ConvertComments, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_comments into domain layer table issue_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearComment{}.TableName(), RAW_COMMENTS_TABLE}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertComments + +func ConvertComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + commentIdGen := didgen.NewDomainIdGenerator(&models.LinearComment{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("c.*"), + dal.From("_tool_linear_comments c"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = c.connection_id AND i.id = c.issue_id)"), + dal.Where("c.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearComment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + comment := inputRow.(*models.LinearComment) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(connectionId, comment.Id)}, + IssueId: issueIdGen.Generate(connectionId, comment.IssueId), + Body: comment.Body, + CreatedDate: comment.CreatedAt, + } + if comment.AuthorId != "" { + domainComment.AccountId = accountIdGen.Generate(connectionId, comment.AuthorId) + } + if !comment.UpdatedAt.IsZero() { + domainComment.UpdatedDate = &comment.UpdatedAt + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go new file mode 100644 index 00000000000..15b28a3b969 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCommentsMeta = plugin.SubTaskMeta{ + Name: "Extract Comments", + EntryPoint: ExtractComments, + EnabledByDefault: true, + Description: "Extract raw comment data into tool layer table _tool_linear_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractComments + +func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiComment := &GraphqlQueryComment{} + if err := errors.Convert(json.Unmarshal(row.Data, apiComment)); err != nil { + return nil, err + } + // The owning issue id is carried in the raw row's input column. + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + comment := &models.LinearComment{ + ConnectionId: data.Options.ConnectionId, + Id: apiComment.Id, + IssueId: issueRef.OwningIssueId(), + Body: apiComment.Body, + CreatedAt: apiComment.CreatedAt, + UpdatedAt: apiComment.UpdatedAt, + } + if apiComment.User != nil { + comment.AuthorId = apiComment.User.Id + } + return []interface{}{comment}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_collector.go b/backend/plugins/linear/tasks/cycle_collector.go new file mode 100644 index 00000000000..88572650fcd --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_CYCLES_TABLE = "linear_cycles" + +// GraphqlQueryCycleWrapper is the team-scoped, paginated `cycles` query. +type GraphqlQueryCycleWrapper struct { + Team struct { + Cycles struct { + Nodes []GraphqlQueryCycle + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"cycles(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryCycle struct { + Id string + Number int + Name string + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time +} + +var CollectCyclesMeta = plugin.SubTaskMeta{ + Name: "Collect Cycles", + EntryPoint: CollectCycles, + EnabledByDefault: true, + Description: "Collect cycles (sprints) for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectCycles + +func CollectCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCycleWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCycleWrapper) + return query.Team.Cycles.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCycleWrapper) + for _, cycle := range query.Team.Cycles.Nodes { + messages = append(messages, errors.Must1(json.Marshal(cycle))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_convertor.go b/backend/plugins/linear/tasks/cycle_convertor.go new file mode 100644 index 00000000000..ae4a0dd5a84 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCyclesMeta = plugin.SubTaskMeta{ + Name: "Convert Cycles", + EntryPoint: ConvertCycles, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_cycles into domain layer tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearCycle{}.TableName(), RAW_CYCLES_TABLE}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertCycles + +func ConvertCycles(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearCycle{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearCycle{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + cycle := inputRow.(*models.LinearCycle) + sprintId := cycleIdGen.Generate(connectionId, cycle.Id) + name := cycle.Name + if name == "" { + name = fmt.Sprintf("Cycle %d", cycle.Number) + } + status := "ACTIVE" + if cycle.CompletedAt != nil { + status = "CLOSED" + } + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: name, + Status: status, + StartedDate: cycle.StartsAt, + EndedDate: cycle.EndsAt, + CompletedDate: cycle.CompletedAt, + OriginalBoardID: boardId, + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_extractor.go b/backend/plugins/linear/tasks/cycle_extractor.go new file mode 100644 index 00000000000..43a4233aa88 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_extractor.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCyclesMeta = plugin.SubTaskMeta{ + Name: "Extract Cycles", + EntryPoint: ExtractCycles, + EnabledByDefault: true, + Description: "Extract raw cycle data into tool layer table _tool_linear_cycles", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractCycles + +func ExtractCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiCycle := &GraphqlQueryCycle{} + if err := errors.Convert(json.Unmarshal(row.Data, apiCycle)); err != nil { + return nil, err + } + cycle := &models.LinearCycle{ + ConnectionId: data.Options.ConnectionId, + Id: apiCycle.Id, + TeamId: data.Options.TeamId, + Number: apiCycle.Number, + Name: apiCycle.Name, + StartsAt: apiCycle.StartsAt, + EndsAt: apiCycle.EndsAt, + CompletedAt: apiCycle.CompletedAt, + } + return []interface{}{cycle}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go new file mode 100644 index 00000000000..9f5a0aedd1d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUES_TABLE = "linear_issues" + +// GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. +// Incremental runs filter server-side on updatedAt ($filter) rather than +// relying on result ordering, so collection no longer depends on an undocumented +// default sort direction. +type GraphqlQueryIssueWrapper struct { + Team struct { + Issues struct { + Nodes []GraphqlQueryIssue `graphql:"nodes"` + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt, filter: $filter)"` + } `graphql:"team(id: $teamId)"` +} + +// IssueFilter mirrors the subset of Linear's GraphQL IssueFilter input used to +// restrict collection to issues updated after a point in time. The Go type +// name is significant: the GraphQL client emits it as the variable's type +// ($filter:IssueFilter!). +type IssueFilter struct { + UpdatedAt *DateComparator `json:"updatedAt,omitempty"` +} + +// DateComparator mirrors Linear's DateComparator input (only the `gt` operator +// is needed here). +type DateComparator struct { + Gt *time.Time `json:"gt,omitempty"` +} + +// buildIssueFilter returns an IssueFilter restricting to issues updated after +// `since`. When `since` is nil (a full sync) it returns the empty filter, which +// Linear treats as "match all". +func buildIssueFilter(since *time.Time) IssueFilter { + if since == nil { + return IssueFilter{} + } + return IssueFilter{UpdatedAt: &DateComparator{Gt: since}} +} + +type GraphqlQueryIssue struct { + Id string + Identifier string + Number int + Title string + Description string + Url string + Priority int + Estimate *float64 + CreatedAt time.Time + UpdatedAt time.Time + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + State *struct { + Id string + Name string + Type string + } + Assignee *struct{ Id string } + Creator *struct{ Id string } + Cycle *struct{ Id string } + Parent *struct{ Id string } + Labels struct { + Nodes []struct { + Id string + Name string + } + } `graphql:"labels(first: 50)"` +} + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "Collect Issues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "Collect issues for a Linear team, supports incremental collection", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssues + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }) + if err != nil { + return err + } + + since := apiCollector.GetSince() + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryIssueWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + "filter": buildIssueFilter(since), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryIssueWrapper) + return query.Team.Issues.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryIssueWrapper) + // The server-side $filter already restricts to issues updated after + // `since`, so every returned issue is in scope -- no client-side + // early-stop (and thus no dependency on sort direction) is needed. + for _, issue := range query.Team.Issues.Nodes { + issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) + issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) + issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) + messages = append(messages, errors.Must1(json.Marshal(issue))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector_test.go b/backend/plugins/linear/tasks/issue_collector_test.go new file mode 100644 index 00000000000..2dbbeecf586 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestBuildIssueFilter pins the server-side incremental filter that replaces +// the previous reliance on result ordering plus a client-side early-stop. A +// full sync must produce an empty filter (match all); an incremental run must +// produce Linear's IssueFilter shape `{ updatedAt: { gt: } }`. +func TestBuildIssueFilter(t *testing.T) { + full, err := json.Marshal(buildIssueFilter(nil)) + assert.NoError(t, err) + assert.Equal(t, "{}", string(full)) + + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental, err := json.Marshal(buildIssueFilter(&since)) + assert.NoError(t, err) + assert.JSONEq(t, `{"updatedAt":{"gt":"2026-05-01T00:00:00Z"}}`, string(incremental)) +} diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go new file mode 100644 index 00000000000..2e947700d6a --- /dev/null +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Issues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssues + +func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + // Preload account display names so issues can carry assignee/creator names + // and emit issue_assignees rows, mirroring how the account convertor derives + // the domain account's full name (displayName, falling back to name). + var accounts []models.LinearAccount + if err := db.All(&accounts, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + accountNames := make(map[string]string, len(accounts)) + for _, account := range accounts { + name := account.Name + if account.DisplayName != "" { + name = account.DisplayName + } + accountNames[account.Id] = name + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: ticket.REQUIREMENT, + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, + } + if issue.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + domainIssue.CreatorName = accountNames[issue.CreatorId] + } + if issue.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + domainIssue.AssigneeName = accountNames[issue.AssigneeId] + } + if issue.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) + domainIssue.IsSubtask = true + } + // Resolution date: completedAt, falling back to canceledAt. + if issue.CompletedAt != nil { + domainIssue.ResolutionDate = issue.CompletedAt + } else if issue.CanceledAt != nil { + domainIssue.ResolutionDate = issue.CanceledAt + } + // Fallback lead time when no history-derived value is present. + // Guard against a resolution that precedes creation (clock skew or + // migrated/imported issues): a negative duration cast to uint yields + // platform-dependent garbage, so leave lead time unset instead. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil && + domainIssue.ResolutionDate.After(issue.CreatedAt) { + minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_extractor.go b/backend/plugins/linear/tasks/issue_extractor.go new file mode 100644 index 00000000000..e423076612d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_extractor.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "Extract Issues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "Extract raw issue data into tool layer tables _tool_linear_issues and _tool_linear_issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiIssue := &GraphqlQueryIssue{} + if err := errors.Convert(json.Unmarshal(row.Data, apiIssue)); err != nil { + return nil, err + } + connectionId := data.Options.ConnectionId + issue := &models.LinearIssue{ + ConnectionId: connectionId, + Id: apiIssue.Id, + TeamId: data.Options.TeamId, + Identifier: apiIssue.Identifier, + Number: apiIssue.Number, + Title: apiIssue.Title, + Description: apiIssue.Description, + Url: apiIssue.Url, + Priority: apiIssue.Priority, + PriorityLabel: PriorityLabel(apiIssue.Priority), + Estimate: apiIssue.Estimate, + CreatedAt: apiIssue.CreatedAt, + UpdatedAt: apiIssue.UpdatedAt, + StartedAt: apiIssue.StartedAt, + CompletedAt: apiIssue.CompletedAt, + CanceledAt: apiIssue.CanceledAt, + } + if apiIssue.State != nil { + issue.StateId = apiIssue.State.Id + issue.StateName = apiIssue.State.Name + issue.StateType = apiIssue.State.Type + } + if apiIssue.Assignee != nil { + issue.AssigneeId = apiIssue.Assignee.Id + } + if apiIssue.Creator != nil { + issue.CreatorId = apiIssue.Creator.Id + } + if apiIssue.Cycle != nil { + issue.CycleId = apiIssue.Cycle.Id + } + if apiIssue.Parent != nil { + issue.ParentId = apiIssue.Parent.Id + } + + results := make([]interface{}, 0, len(apiIssue.Labels.Nodes)+1) + results = append(results, issue) + for _, label := range apiIssue.Labels.Nodes { + results = append(results, &models.LinearIssueLabel{ + ConnectionId: connectionId, + IssueId: apiIssue.Id, + LabelName: label.Name, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go new file mode 100644 index 00000000000..11de7c218c7 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUE_HISTORY_TABLE = "linear_issue_history" + +// GraphqlQueryHistoryWrapper is the per-issue, paginated `history` query. +type GraphqlQueryHistoryWrapper struct { + Issue struct { + History struct { + Nodes []GraphqlQueryHistory + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"history(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryHistory struct { + Id string + CreatedAt time.Time + Actor *struct{ Id string } + FromState *struct { + Id string + Name string + Type string + } + ToState *struct { + Id string + Name string + Type string + } +} + +var CollectIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Collect Issue History", + EntryPoint: CollectIssueHistory, + EnabledByDefault: true, + Description: "Collect history events for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssueHistory + +func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's history cannot have changed, so re-fetching it every + // run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryHistoryWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryHistoryWrapper) + return query.Issue.History.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryHistoryWrapper) + for _, event := range query.Issue.History.Nodes { + // Only state transitions are relevant to the status changelog. + if event.FromState == nil && event.ToState == nil { + continue + } + messages = append(messages, errors.Must1(json.Marshal(event))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go new file mode 100644 index 00000000000..f1fa99d0519 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Convert Issue History", + EntryPoint: ConvertIssueHistory, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_history into domain layer table issue_changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueHistory{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUE_HISTORY_TABLE}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueHistory + +func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + historyIdGen := didgen.NewDomainIdGenerator(&models.LinearIssueHistory{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueHistory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + event := inputRow.(*models.LinearIssueHistory) + changelog := &ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{Id: historyIdGen.Generate(connectionId, event.Id)}, + IssueId: issueIdGen.Generate(connectionId, event.IssueId), + FieldId: "state", + FieldName: "status", + OriginalFromValue: event.FromStateName, + OriginalToValue: event.ToStateName, + CreatedDate: event.CreatedAt, + } + if event.FromStateType != "" { + changelog.FromValue = StatusFromStateType(event.FromStateType) + } + if event.ToStateType != "" { + changelog.ToValue = StatusFromStateType(event.ToStateType) + } + if event.ActorId != "" { + changelog.AuthorId = accountIdGen.Generate(connectionId, event.ActorId) + } + return []interface{}{changelog}, nil + }, + }) + if err != nil { + return err + } + if err := converter.Execute(); err != nil { + return err + } + + return deriveLeadTimeFromHistory(db, connectionId, data.Options.TeamId, issueIdGen) +} + +// deriveLeadTimeFromHistory refines each issue's lead time from its recorded +// state transitions: the span from the issue's first transition into an +// in-progress state to its first transition into a done state thereafter (the +// active cycle time). This is the value that genuinely requires history and is +// more accurate than the coarse createdAt -> resolutionDate fallback set by +// ConvertIssues, so it overrides that fallback when the transitions exist. +// Issues whose history lacks an in-progress -> done sequence keep the fallback. +func deriveLeadTimeFromHistory(db dal.Dal, connectionId uint64, teamId string, issueIdGen *didgen.DomainIdGenerator) errors.Error { + var events []models.LinearIssueHistory + if err := db.All(&events, + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, teamId), + dal.Orderby("h.issue_id, h.created_at"), + ); err != nil { + return err + } + + type leadWindow struct { + startedAt *time.Time + doneAt *time.Time + } + windows := map[string]*leadWindow{} + for i := range events { + event := events[i] + window := windows[event.IssueId] + if window == nil { + window = &leadWindow{} + windows[event.IssueId] = window + } + switch StatusFromStateType(event.ToStateType) { + case ticket.IN_PROGRESS: + if window.startedAt == nil { + createdAt := event.CreatedAt + window.startedAt = &createdAt + } + case ticket.DONE: + if window.startedAt != nil && window.doneAt == nil { + createdAt := event.CreatedAt + window.doneAt = &createdAt + } + } + } + + for issueId, window := range windows { + if window.startedAt == nil || window.doneAt == nil || !window.doneAt.After(*window.startedAt) { + continue + } + minutes := uint(window.doneAt.Sub(*window.startedAt).Minutes()) + if err := db.UpdateColumn( + &ticket.Issue{}, "lead_time_minutes", minutes, + dal.Where("id = ?", issueIdGen.Generate(connectionId, issueId)), + ); err != nil { + return err + } + } + return nil +} diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go new file mode 100644 index 00000000000..006b92f9701 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Extract Issue History", + EntryPoint: ExtractIssueHistory, + EnabledByDefault: true, + Description: "Extract raw issue history into tool layer table _tool_linear_issue_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssueHistory + +func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiEvent := &GraphqlQueryHistory{} + if err := errors.Convert(json.Unmarshal(row.Data, apiEvent)); err != nil { + return nil, err + } + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + event := &models.LinearIssueHistory{ + ConnectionId: data.Options.ConnectionId, + Id: apiEvent.Id, + IssueId: issueRef.OwningIssueId(), + CreatedAt: apiEvent.CreatedAt, + } + if apiEvent.Actor != nil { + event.ActorId = apiEvent.Actor.Id + } + if apiEvent.FromState != nil { + event.FromStateId = apiEvent.FromState.Id + event.FromStateName = apiEvent.FromState.Name + event.FromStateType = apiEvent.FromState.Type + } + if apiEvent.ToState != nil { + event.ToStateId = apiEvent.ToState.Id + event.ToStateName = apiEvent.ToState.Name + event.ToStateType = apiEvent.ToState.Type + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/label_convertor.go b/backend/plugins/linear/tasks/label_convertor.go new file mode 100644 index 00000000000..e94826d4775 --- /dev/null +++ b/backend/plugins/linear/tasks/label_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ + Name: "Convert Issue Labels", + EntryPoint: ConvertIssueLabels, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_labels into domain layer table issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueLabel{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.IssueLabel{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueLabels + +func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + + cursor, err := db.Cursor( + dal.Select("l.*"), + dal.From("_tool_linear_issue_labels l"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = l.connection_id AND i.id = l.issue_id)"), + dal.Where("l.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueLabel{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + label := inputRow.(*models.LinearIssueLabel) + domainLabel := &ticket.IssueLabel{ + IssueId: issueIdGen.Generate(connectionId, label.IssueId), + LabelName: label.LabelName, + } + return []interface{}{domainLabel}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go new file mode 100644 index 00000000000..0a8cfe9da7f --- /dev/null +++ b/backend/plugins/linear/tasks/shared.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// issuesToCollectChildrenClauses builds the cursor clauses that drive per-issue +// child collection (comments, history). When `since` is non-nil (an incremental +// run), it restricts the sweep to issues updated since the last successful +// collection, so unchanged issues no longer trigger a request every run. On a +// full sync `since` is nil and all of the team's issues are swept. +func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *time.Time) []dal.Clause { + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + } + if since != nil { + clauses = append(clauses, dal.Where("updated_at > ?", *since)) + } + return clauses +} + +// priorityLabels maps Linear's integer priority to its human-readable label. +// Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. +var priorityLabels = map[int]string{ + 0: "No priority", + 1: "Urgent", + 2: "High", + 3: "Medium", + 4: "Low", +} + +// PriorityLabel returns the human-readable label for a Linear priority value. +func PriorityLabel(priority int) string { + if label, ok := priorityLabels[priority]; ok { + return label + } + return "No priority" +} + +// StatusFromStateType maps a Linear WorkflowState.type to a DevLake standard +// issue status. Linear's state types are standardized, so no user-supplied +// mapping is required: +// +// triage, backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +// +// "triage" is the inbox state issues land in before they are accepted into a +// workflow; it is treated as not-yet-started (TODO). Any unrecognized type +// falls back to OTHER so unexpected API values surface rather than silently +// masquerading as a known status. +func StatusFromStateType(stateType string) string { + switch stateType { + case "triage", "backlog", "unstarted": + return ticket.TODO + case "started": + return ticket.IN_PROGRESS + case "completed", "canceled": + return ticket.DONE + default: + return ticket.OTHER + } +} diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go new file mode 100644 index 00000000000..ad2fc7bc6d0 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestIssuesToCollectChildrenClauses pins the incremental behaviour of the +// per-issue child collectors (comments, history): a full sync sweeps every +// issue, while an incremental run adds an updated_at filter so unchanged issues +// are skipped instead of triggering a request each run. +func TestIssuesToCollectChildrenClauses(t *testing.T) { + // full sync: no `since` -> select/from/where(connection,team) only + full := issuesToCollectChildrenClauses(1, "team-1", nil) + assert.Len(t, full, 3) + + // incremental: a `since` adds the updated_at filter clause + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental := issuesToCollectChildrenClauses(1, "team-1", &since) + assert.Len(t, incremental, 4) +} diff --git a/backend/plugins/linear/tasks/shared_test.go b/backend/plugins/linear/tasks/shared_test.go new file mode 100644 index 00000000000..eaf737c9022 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/stretchr/testify/assert" +) + +// TestStatusFromStateType pins the mapping for every Linear WorkflowState.type +// value. Linear's state types are standardized; "triage" is the inbox state +// issues land in before they are accepted, so it maps to TODO. Any genuinely +// unknown type falls back to OTHER. +func TestStatusFromStateType(t *testing.T) { + cases := map[string]string{ + "backlog": ticket.TODO, + "unstarted": ticket.TODO, + "triage": ticket.TODO, + "started": ticket.IN_PROGRESS, + "completed": ticket.DONE, + "canceled": ticket.DONE, + "": ticket.OTHER, + "something": ticket.OTHER, + } + for stateType, want := range cases { + assert.Equal(t, want, StatusFromStateType(stateType), "state type %q", stateType) + } +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go new file mode 100644 index 00000000000..7ebe467a908 --- /dev/null +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertSprintIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Sprint Issues", + EntryPoint: ConvertSprintIssues, + EnabledByDefault: true, + Description: "Link issues to their cycle (sprint) in the domain layer table sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.SprintIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprintIssues + +func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + + // Sprint membership is derived from each issue's cycle_id. Clear this team's + // existing sprint_issues up front so issues that have since left their cycle + // leave no stale rows: the batch divider only deletes outdated records when + // it produces at least one row of the type, which misses the case where + // every issue has been removed from its cycle. + var teamIssues []models.LinearIssue + if err := db.All(&teamIssues, + dal.Select("id"), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ); err != nil { + return err + } + if len(teamIssues) > 0 { + issueIds := make([]string, len(teamIssues)) + for i, issue := range teamIssues { + issueIds[i] = issueIdGen.Generate(connectionId, issue.Id) + } + if err := db.Delete(&ticket.SprintIssue{}, dal.Where("issue_id IN ?", issueIds)); err != nil { + return err + } + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + sprintIssue := &ticket.SprintIssue{ + SprintId: cycleIdGen.Generate(connectionId, issue.CycleId), + IssueId: issueIdGen.Generate(connectionId, issue.Id), + } + return []interface{}{sprintIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/task_data.go b/backend/plugins/linear/tasks/task_data.go new file mode 100644 index 00000000000..cd72916d074 --- /dev/null +++ b/backend/plugins/linear/tasks/task_data.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearOptions are the per-scope options passed to a pipeline task. +type LinearOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// LinearTaskData is the shared context handed to every Linear subtask. +type LinearTaskData struct { + Options *LinearOptions + GraphqlClient *api.GraphqlAsyncClient + TimeAfter *time.Time +} + +type LinearApiParams models.LinearApiParams diff --git a/backend/plugins/linear/tasks/workflow_state_collector.go b/backend/plugins/linear/tasks/workflow_state_collector.go new file mode 100644 index 00000000000..94c22d1d3a5 --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_WORKFLOW_STATES_TABLE = "linear_workflow_states" + +// GraphqlQueryWorkflowStateWrapper is the team-scoped paginated `states` query. +type GraphqlQueryWorkflowStateWrapper struct { + Team struct { + States struct { + Nodes []GraphqlQueryWorkflowState + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"states(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryWorkflowState struct { + Id string + Name string + Type string + Color string + Position float64 +} + +var CollectWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Collect Workflow States", + EntryPoint: CollectWorkflowStates, + EnabledByDefault: true, + Description: "Collect workflow states for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectWorkflowStates + +func CollectWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryWorkflowStateWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryWorkflowStateWrapper) + return query.Team.States.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryWorkflowStateWrapper) + for _, state := range query.Team.States.Nodes { + messages = append(messages, errors.Must1(json.Marshal(state))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/workflow_state_extractor.go b/backend/plugins/linear/tasks/workflow_state_extractor.go new file mode 100644 index 00000000000..75a2099aeaf --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_extractor.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Extract Workflow States", + EntryPoint: ExtractWorkflowStates, + EnabledByDefault: true, + Description: "Extract raw workflow state data into tool layer table _tool_linear_workflow_states", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractWorkflowStates + +func ExtractWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiState := &GraphqlQueryWorkflowState{} + if err := errors.Convert(json.Unmarshal(row.Data, apiState)); err != nil { + return nil, err + } + state := &models.LinearWorkflowState{ + ConnectionId: data.Options.ConnectionId, + Id: apiState.Id, + TeamId: data.Options.TeamId, + Name: apiState.Name, + Type: apiState.Type, + Color: apiState.Color, + Position: apiState.Position, + } + return []interface{}{state}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index c885d5114b5..c594b8f770b 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -31,6 +31,7 @@ import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; @@ -58,6 +59,7 @@ export const pluginConfigs: IPluginConfig[] = [ GitLabConfig, JenkinsConfig, JiraConfig, + LinearConfig, PagerDutyConfig, RootlyConfig, SlackConfig, diff --git a/config-ui/src/plugins/register/linear/assets/icon.svg b/config-ui/src/plugins/register/linear/assets/icon.svg new file mode 100644 index 00000000000..64cddfdea54 --- /dev/null +++ b/config-ui/src/plugins/register/linear/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/linear/config.tsx b/config-ui/src/plugins/register/linear/config.tsx new file mode 100644 index 00000000000..28cf9f98a95 --- /dev/null +++ b/config-ui/src/plugins/register/linear/config.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const LinearConfig: IPluginConfig = { + plugin: 'linear', + name: 'Linear', + icon: ({ color }) => , + sort: 13, + connection: { + docLink: 'https://developers.linear.app/docs', + initialValues: { + endpoint: 'https://api.linear.app/graphql', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Linear GraphQL API base URL.', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'Your Linear personal API key (Settings → Security & access → Personal API keys).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (1500).', + defaultValue: 1500, + }, + ], + }, + dataScope: { + title: 'Teams', + searchPlaceholder: 'Search teams...', + }, +}; diff --git a/config-ui/src/plugins/register/linear/index.ts b/config-ui/src/plugins/register/linear/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/linear/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index e820f72cdd4..34c6ed63b14 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -45,6 +45,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.name}`; case 'asana': return `${scope.gid}`; + case 'linear': + return `${scope.teamId}`; default: return `${scope.id}`; } diff --git a/grafana/dashboards/Linear.json b/grafana/dashboards/Linear.json new file mode 100644 index 00000000000..1e748264206 --- /dev/null +++ b/grafana/dashboards/Linear.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Linear", + "url": "https://devlake.apache.org/docs/Configuration/Linear" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Linear.\n- Data Source Required: Linear", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "mysql", + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "9.5.15", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'linear%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'linear%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Linear", + "uid": "linear-dashboard", + "version": 1, + "weekStart": "" +}