diff --git a/.circleci/config.yml b/.circleci/config.yml index d9d10891..95b3ffaf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,14 @@ -version: 2 +version: 2.1 -jobs: - build: +executors: + go-executor: docker: - image: cimg/go:1.23 - working_directory: ~/task-tools + working_directory: ~/task-tools + +jobs: + test: + executor: go-executor steps: - checkout - run: go install github.com/jstemmer/go-junit-report/v2@latest @@ -23,4 +27,36 @@ jobs: - store_artifacts: path: ~/task-tools/junit - store_artifacts: - path: tests.out \ No newline at end of file + path: tests.out + + build: + executor: go-executor + steps: + - checkout + - setup_remote_docker: + docker_layer_caching: false + - run: + name: "docker login" + command: echo ${DOCKERHUB_TOKEN} | docker login -u ${DOCKERHUB_USERNAME} --password-stdin + - run: + name: "Push Docker Image" + command: make docker + +workflows: + version: 2 + test_and_build: + jobs: + - test: + filters: + tags: + only: + - /.*/ + - build: + requires: + - test + context: + - DOCKER + filters: + tags: + only: + - /.*/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6f6f2342..c5ce8a25 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,7 @@ apps/utils/file-watcher/file-watcher */stats/stats apps/workers/sql-load/sql-load build +tasks.db +*_preview.html coverage diff --git a/README.md b/README.md index b53d4f12..871beff8 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,16 @@ func (tm *taskMaster) Run(ctx context.Context) error { ## Pre-built Apps ### **Flowlord** -an all-purpose TaskMaster that should be used with workflow files to schedule when tasks should run and the task hierarchy. It can retry failed jobs, alert when tasks fail and has an API that can be used to backload/schedule jobs and give a recap of recent jobs run. +Production-ready task orchestration engine for managing complex workflow dependencies with intelligent scheduling, automatic retries, and real-time monitoring. Features include: -See Additional [docs](apps/flowlord/README.md). +- **Workflow Management** - Multi-phase workflows with parent-child task dependencies +- **Intelligent Scheduling** - Cron-based scheduling with template-based task generation +- **Optional SQLite Cache** - Task history, alerts, and file tracking for troubleshooting (non-critical, stateless operation) +- **Web Dashboard** - Real-time monitoring UI with filtering, pagination, and date navigation +- **Batch Processing** - Generate multiple tasks from date ranges, metadata arrays, or data files +- **RESTful API** - Comprehensive API for backloading, monitoring, and workflow management + +See detailed [documentation](apps/flowlord/README.md). ### Workers diff --git a/apps/flowlord/README.md b/apps/flowlord/README.md index a909c396..f4136ee1 100644 --- a/apps/flowlord/README.md +++ b/apps/flowlord/README.md @@ -1,11 +1,64 @@ # flowlord taskmaster -flowlord schedules and coordinates task dependency across workflows. Flowlord reads tasks from the done topic, failed tasks can be configured to retry a set number of times before being sent to slack and/or a retry_failed topic. Successful tasks will start children tasks. -![](flowlord.drawio.svg) + + +**Flowlord** is a production-ready task orchestration engine that manages complex workflow dependencies with intelligent scheduling, automatic retries, and real-time monitoring. Built on the [task](https://github.com/pcelvng/task) ecosystem, it coordinates distributed workers through message bus communication while providing visibility into task execution through a web dashboard. +**Key Features:** +- **Workflow Management** - Define multi-phase workflows with parent-child task dependencies +- **Intelligent Scheduling** - Cron-based scheduling with template-based task generation +- **Automatic Retries** - Configurable retry logic with exponential backoff and jitter to prevent thundering herd +- **File Watching** - Trigger tasks automatically when files are written to specified paths +- **Batch Processing** - Generate multiple tasks from date ranges, metadata arrays, or data files +- **Alerting** - Slack notifications for failed tasks and incomplete jobs with smart frequency management +- **RESTful API** - Web UI and API for monitoring workflows, viewing task history, and managing alerts [![Static Badge](https://img.shields.io/badge/API%20Docs-green)](https://github.com/pcelvng/task-tools/wiki/Flowlord-API) +
+ +## Overview + +![](flowlord.drawio.svg) + +## Monitoring & Troubleshooting + +SQLite is used to store phases and provides troubleshooting convenience by recording task history. As flowlord is stateless task management system, the cache persistent is not required for flowlord to work, but it convenient to review historical tasks and alerts. It is recommended to backup the task and alert logs for long term storage. + +- **Task Records** - Full execution lifecycle with timing metrics +- **Alert History** - Failed task tracking for debugging +- **File Processing Audit** - Which files triggered which tasks +- **Workflow State** - Phase configuration and dependencies + +The database is optional and non-critical. If deleted, Flowlord continues normally and rebuilds fresh data. Features: +- Automatic backup/restore from remote paths (S3/GCS) +- 90-day default retention with automatic cleanup +- WAL mode for concurrent access + +**Configuration:** +```toml +[cache] + local_path = "./tasks.db" # required local cache + backup_path = "gs://bucket/tasks.db" # Optional backup location + retention = "2160h" # 90 days + task_ttl = "4h" # Alert deadline from task to complete (creation to complete) +``` + +## Web Dashboard + +Built-in web UI for monitoring workflows and troubleshooting. Uses Go templates to render HTML dashboards with: +- Task execution history with filtering and pagination +- Alert summaries grouped by task type +- File processing history +- Workflow phase visualization +- System statistics + +Access at `http://localhost:8080/` (or configured port) + +| Files View | Tasks View | Alerts View | Workflow View | +|:----------:|:----------:|:-----------:|:-------------:| +| [![Files View](../../internal/docs/img/flowlord_files.png)](../../internal/docs/img/flowlord_files.png) | [![Tasks View](../../internal/docs/img/flowlord_tasks.png)](../../internal/docs/img/flowlord_tasks.png) | [![Alerts View](../../internal/docs/img/flowlord_alerts.png)](../../internal/docs/img/flowlord_alerts.png) | [![Workflow View](../../internal/docs/img/flowlord_workflow.png)](../../internal/docs/img/flowlord_workflow.png) | + ## workflow A workflow consists of one or more phases as a way to define of how a set of task is to be scheduled and run and the dependencies between them. diff --git a/apps/flowlord/cache/cache.go b/apps/flowlord/cache/cache.go deleted file mode 100644 index 39d2cc20..00000000 --- a/apps/flowlord/cache/cache.go +++ /dev/null @@ -1,146 +0,0 @@ -package cache - -import ( - "net/url" - "strings" - "sync" - "time" - - "github.com/pcelvng/task" - "github.com/pcelvng/task/bus" -) - -type Cache interface { - Add(task.Task) - Get(id string) TaskJob - - // todo: listener for cache expiry? -} - -func NewMemory(ttl time.Duration) *Memory { - if ttl < time.Hour { - ttl = time.Hour - } - return &Memory{ - ttl: ttl, - cache: make(map[string]TaskJob), - } - -} - -type Memory struct { - ttl time.Duration - cache map[string]TaskJob - mu sync.RWMutex -} - -// todo: name to describe info about completed tasks that are within the cache -type TaskJob struct { - LastUpdate time.Time // time since the last event with id - Completed bool - count int - Events []task.Task -} - -type Stat struct { - Count int - Removed int - ProcessTime time.Duration - Unfinished []task.Task -} - -// Recycle iterates through the cache -// clearing all tasks that have been completed within the cache window -// it returns a list of tasks that have not been completed but have expired -func (c *Memory) Recycle() Stat { - tasks := make([]task.Task, 0) - t := time.Now() - total := len(c.cache) - c.mu.Lock() - for k, v := range c.cache { - // remove expired items - if t.Sub(v.LastUpdate) > c.ttl { - if !v.Completed { - tasks = append(tasks, v.Events[len(v.Events)-1]) - } - delete(c.cache, k) - } - } - c.mu.Unlock() - return Stat{ - Count: len(c.cache), - Removed: total - len(c.cache), - ProcessTime: time.Since(t), - Unfinished: tasks, - } - -} - -// Add a task to the cache -// the task must have an id to be added. -func (c *Memory) Add(t task.Task) { - if t.ID == "" || c == nil { - return - } - c.mu.Lock() - job := c.cache[t.ID] - job.Events = append(job.Events, t) - if t.Result != "" { - job.Completed = true - t, _ := time.Parse(time.RFC3339, t.Ended) - job.LastUpdate = t - } else { - job.Completed = false - t, _ := time.Parse(time.RFC3339, t.Created) - job.LastUpdate = t - } - - c.cache[t.ID] = job - c.mu.Unlock() -} - -func (c *Memory) Recap() map[string]*Stats { - data := map[string]*Stats{} - if c == nil { - return data - } - c.mu.RLock() - for _, v := range c.cache { - for _, t := range v.Events { - job := t.Job - if job == "" { - v, _ := url.ParseQuery(t.Meta) - job = v.Get("job") - } - key := strings.TrimRight(t.Type+":"+job, ":") - stat, found := data[key] - if !found { - stat = &Stats{ - CompletedTimes: make([]time.Time, 0), - ErrorTimes: make([]time.Time, 0), - ExecTimes: &DurationStats{}, - } - data[key] = stat - } - stat.Add(t) - } - } - c.mu.RUnlock() - return data -} - -// Get the TaskJob info with the given id. -// If the id isn't found a _ is returned -func (c *Memory) Get(id string) TaskJob { - c.mu.RLock() - defer c.mu.RUnlock() - return c.cache[id] -} - -// SendFunc extends the given producers send function by adding any task sent to the cache. -func (m *Memory) SendFunc(p bus.Producer) func(string, *task.Task) error { - return func(topic string, tsk *task.Task) error { - m.Add(*tsk) - return p.Send(topic, tsk.JSONBytes()) - } -} diff --git a/apps/flowlord/cache/cache_test.go b/apps/flowlord/cache/cache_test.go deleted file mode 100644 index 7c615123..00000000 --- a/apps/flowlord/cache/cache_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package cache - -import ( - "testing" - "time" - - "github.com/hydronica/trial" - "github.com/pcelvng/task" -) - -func TestAdd(t *testing.T) { - fn := func(tasks []task.Task) (map[string]TaskJob, error) { - cache := &Memory{cache: make(map[string]TaskJob)} - for _, t := range tasks { - cache.Add(t) - } - for k, v := range cache.cache { - v.count = len(v.Events) - v.Events = nil - cache.cache[k] = v - } - return cache.cache, nil - } - cases := trial.Cases[[]task.Task, map[string]TaskJob]{ - "no id": { - Input: []task.Task{ - {Type: "test"}, - }, - }, - "created": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z"}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.TimeDay("2023-01-01"), - count: 1, - }, - }, - }, - "completed": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:01Z", Result: task.CompleteResult}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.Time(time.RFC3339, "2023-01-01T00:00:01Z"), - Completed: true, - count: 2, - }, - }, - }, - "failed": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:01Z", Result: task.ErrResult, Msg: "Error with pull from X"}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.Time(time.RFC3339, "2023-01-01T00:00:01Z"), - Completed: true, - count: 2, - }, - }, - }, - "retry": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:01Z", Result: task.ErrResult, Msg: "Error with pull from X"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:01:00Z", Meta: "retry=1"}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.Time(time.RFC3339, "2023-01-01T00:01:00Z"), - Completed: false, - count: 3, - }, - }, - }, - "child": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Created: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:01Z", Result: task.CompleteResult}, - {Type: "transform", ID: "id1", Info: "/product/2023-01-01/data.txt", Created: "2023-01-01T00:02:00Z"}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.Time(time.RFC3339, "2023-01-01T00:02:00Z"), - Completed: false, - count: 3, - }, - }, - }, - "multi-child": { - Input: []task.Task{ - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Started: "2023-01-01T00:00:00Z"}, - {Type: "pull", ID: "id1", Info: "?date=2023-01-01", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:01Z", Result: task.CompleteResult}, - {Type: "transform", ID: "id1", Info: "/product/2023-01-01/data.txt", Started: "2023-01-01T00:02:00Z"}, - {Type: "transform", ID: "id1", Info: "/product/2023-01-01/data.txt", Started: "2023-01-01T00:02:00Z", Ended: "2023-01-01T00:02:15Z", Result: task.CompleteResult}, - {Type: "load", ID: "id1", Info: "/product/2023-01-01/data.txt?table=schema.product", Started: "2023-01-01T00:04:00Z"}, - {Type: "load", ID: "id1", Info: "/product/2023-01-01/data.txt?table=schema.product", Started: "2023-01-01T00:04:00Z", Ended: "2023-01-01T00:05:12Z", Result: task.CompleteResult}, - }, - Expected: map[string]TaskJob{ - "id1": { - LastUpdate: trial.Time(time.RFC3339, "2023-01-01T00:05:12Z"), - Completed: true, - count: 6, - }, - }, - }, - } - trial.New(fn, cases).SubTest(t) -} - -func TestRecycle(t *testing.T) { - now := time.Now() - cache := Memory{ - ttl: time.Hour, - cache: map[string]TaskJob{ - "keep": { - Completed: false, - LastUpdate: now.Add(-30 * time.Minute), - Events: []task.Task{{Type: "test1"}}, - }, - "expire": { - Completed: true, - LastUpdate: now.Add(-90 * time.Minute), - }, - "not-completed": { - Completed: false, - LastUpdate: now.Add(-90 * time.Minute), - Events: []task.Task{ - {Type: "test1", Created: now.String()}, - {Type: "test1", Created: now.String(), Result: task.CompleteResult}, - {Type: "test2", Created: now.String()}, - }, - }, - }, - } - - stat := cache.Recycle() - stat.ProcessTime = 0 - expected := Stat{ - Count: 1, - Removed: 2, - Unfinished: []task.Task{ - {Type: "test2", Created: now.String()}, - }} - if eq, diff := trial.Equal(stat, expected); !eq { - t.Logf(diff) - } -} - -func TestRecap(t *testing.T) { - fn := func(in []task.Task) (map[string]string, error) { - c := &Memory{cache: map[string]TaskJob{}} - for _, t := range in { - c.Add(t) - } - result := map[string]string{} - for k, v := range c.Recap() { - result[k] = v.String() - } - return result, nil - } - cases := trial.Cases[[]task.Task, map[string]string]{ - "task no job": { - Input: []task.Task{{ID: "abc", Type: "test1", Info: "?date=2020-01-02", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z"}}, - Expected: map[string]string{ - "test1": "min: 10s max: 10s avg: 10s\n\tComplete: 1 2020/01/02\n", - }, - }, - "task:job": { - Input: []task.Task{ - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-01", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z"}, - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-02", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:15Z"}, - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-03", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:05Z"}, - }, - Expected: map[string]string{ - "test1:job1": "min: 5s max: 15s avg: 10s\n\tComplete: 3 2020/01/01-2020/01/03\n", - }, - }, - "with errors": { - Input: []task.Task{ - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-01", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z"}, - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-02", Result: "error", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:15Z"}, - {ID: "abc", Type: "test1", Job: "job1", Info: "?day=2020-01-03", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:05Z"}, - }, - Expected: map[string]string{ - "test1:job1": "min: 5s max: 10s avg: 7.5s\n\tComplete: 2 2020/01/01,2020/01/03\n\tError: 1 2020/01/02\n", - }, - }, - "hourly": { - Input: []task.Task{ - {ID: "abc", Type: "proc", Job: "hour", Info: "?hour=2020-01-01T05", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z"}, - {ID: "abc", Type: "proc", Job: "hour", Info: "?hour_utc=2020-01-01T06", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:15Z"}, - {ID: "abc", Type: "proc", Job: "hour", Info: "?hour=2020-01-01T07", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:05Z"}, - {ID: "abc", Type: "proc", Job: "hour", Info: "?hour=2020-01-01T08", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:47Z"}, - {ID: "abc", Type: "proc", Job: "hour", Info: "?hour=2020-01-01T09", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:01:33Z"}, - }, - Expected: map[string]string{ - "proc:hour": "min: 5s max: 1m33s avg: 34s\n\tComplete: 5 2020/01/01T05-2020/01/01T09\n", - }, - }, - "monthly": { - Input: []task.Task{ - {ID: "abc", Type: "month", Info: "?day=2020-01-01", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z"}, - {ID: "abc", Type: "month", Info: "?day=2020-02-01", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:15Z"}, - }, - Expected: map[string]string{ - "month": "min: 10s max: 15s avg: 12.5s\n\tComplete: 2 2020/01/01,2020/02/01\n", - }, - }, - "meta_job": { - Input: []task.Task{ - {ID: "abc", Type: "test1", Info: "?day=2020-01-01", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:10Z", Meta: "job=job1"}, - {ID: "abc", Type: "test1", Info: "?day=2020-01-02", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:15Z", Meta: "job=job1"}, - {ID: "abc", Type: "test1", Info: "?day=2020-01-03", Result: "complete", Started: "2023-01-01T00:00:00Z", Ended: "2023-01-01T00:00:05Z", Meta: "job=job1"}, - }, - Expected: map[string]string{ - "test1:job1": "min: 5s max: 15s avg: 10s\n\tComplete: 3 2020/01/01-2020/01/03\n", - }, - }, - } - trial.New(fn, cases).SubTest(t) -} diff --git a/apps/flowlord/cache/stats.go b/apps/flowlord/cache/stats.go deleted file mode 100644 index 1fde69c5..00000000 --- a/apps/flowlord/cache/stats.go +++ /dev/null @@ -1,125 +0,0 @@ -package cache - -import ( - "encoding/json" - "fmt" - "time" - - gtools "github.com/jbsmith7741/go-tools" - "github.com/pcelvng/task" - - "github.com/pcelvng/task-tools/tmpl" -) - -const ( - precision = 10 * time.Millisecond -) - -type Stats struct { - CompletedCount int - CompletedTimes []time.Time - - ErrorCount int - ErrorTimes []time.Time - - ExecTimes *DurationStats -} - -func (s *Stats) MarshalJSON() ([]byte, error) { - type count struct { - Count int - Times string - } - - v := struct { - Min string `json:"min"` - Max string `json:"max"` - Average string `json:"avg"` - Complete count `json:"complete"` - Error count `json:"error"` - }{ - Min: gtools.PrintDuration(s.ExecTimes.Min), - Max: gtools.PrintDuration(s.ExecTimes.Max), - Average: gtools.PrintDuration(s.ExecTimes.Average()), - Complete: count{ - Count: s.CompletedCount, - Times: tmpl.PrintDates(s.CompletedTimes), - }, - Error: count{ - Count: s.ErrorCount, - Times: tmpl.PrintDates(s.ErrorTimes), - }, - } - return json.Marshal(v) -} - -func (s Stats) String() string { - r := s.ExecTimes.String() - if s.CompletedCount > 0 { - r += fmt.Sprintf("\n\tComplete: %d %v", s.CompletedCount, tmpl.PrintDates(s.CompletedTimes)) - } - if s.ErrorCount > 0 { - r += fmt.Sprintf("\n\tError: %d %v", s.ErrorCount, tmpl.PrintDates(s.ErrorTimes)) - } - - return r + "\n" -} - -type DurationStats struct { - Min time.Duration - Max time.Duration - sum int64 - count int64 -} - -func (s *DurationStats) Add(d time.Duration) { - if s.count == 0 { - s.Min = d - s.Max = d - } - - if d > s.Max { - s.Max = d - } else if d < s.Min { - s.Min = d - } - // truncate times to milliseconds to preserve space - s.sum += int64(d / precision) - s.count++ -} - -func (s *DurationStats) Average() time.Duration { - if s.count == 0 { - return 0 - } - return time.Duration(s.sum/s.count) * precision -} - -func (s *DurationStats) String() string { - return fmt.Sprintf("min: %v max: %v avg: %v", - s.Min, s.Max, s.Average()) -} - -func (stats *Stats) Add(tsk task.Task) { - tm := tmpl.TaskTime(tsk) - if tsk.Result == task.ErrResult { - stats.ErrorCount++ - stats.ErrorTimes = append(stats.ErrorTimes, tm) - return - } - - stats.CompletedCount++ - stats.CompletedTimes = append(stats.CompletedTimes, tm) - - end, _ := time.Parse(time.RFC3339, tsk.Ended) - start, _ := time.Parse(time.RFC3339, tsk.Started) - stats.ExecTimes.Add(end.Sub(start)) -} - -type pathTime time.Time - -func (p *pathTime) UnmarshalText(b []byte) error { - t := tmpl.PathTime(string(b)) - *p = pathTime(t) - return nil -} diff --git a/apps/flowlord/files.go b/apps/flowlord/files.go index bd6d5de0..d736072f 100644 --- a/apps/flowlord/files.go +++ b/apps/flowlord/files.go @@ -9,6 +9,7 @@ import ( "time" "github.com/pcelvng/task" + "github.com/pcelvng/task-tools/file/stat" "github.com/pcelvng/task-tools/tmpl" "github.com/pcelvng/task-tools/workflow" @@ -93,6 +94,9 @@ func unmarshalStat(b []byte) (sts stat.Stats) { // if a match is found it will create a task and send it out func (tm *taskMaster) matchFile(sts stat.Stats) error { matches := 0 + var taskIDs []string + var taskNames []string + for _, f := range tm.files { if isMatch, _ := filepath.Match(f.SrcPattern, sts.Path); !isMatch { continue @@ -106,19 +110,33 @@ func (tm *taskMaster) matchFile(sts stat.Stats) error { meta.Set("file", sts.Path) meta.Set("filename", filepath.Base(sts.Path)) meta.Set("workflow", f.workflowFile) - // todo: add job if provided in task name ex -> task:job // populate the info string info := tmpl.Parse(f.Template, t) info, _ = tmpl.Meta(info, meta) tsk := task.New(f.Topic(), info) + tsk.Job = f.Job() tsk.Meta, _ = url.QueryUnescape(meta.Encode()) + // Collect task information for storage + taskIDs = append(taskIDs, tsk.ID) + taskName := tsk.Type + if tsk.Job != "" { + taskName += ":" + tsk.Job + } + taskNames = append(taskNames, taskName) + if err := tm.producer.Send(tsk.Type, tsk.JSONBytes()); err != nil { return err } } + + // Store file message in database + if tm.taskCache != nil { + tm.taskCache.AddFileMessage(sts, taskIDs, taskNames) + } + if matches == 0 { return fmt.Errorf("no match found for %q", sts.Path) } diff --git a/apps/flowlord/files_test.go b/apps/flowlord/files_test.go index 6fc19361..62451817 100644 --- a/apps/flowlord/files_test.go +++ b/apps/flowlord/files_test.go @@ -128,7 +128,7 @@ func TestTaskMaster_MatchFile(t *testing.T) { Input: stat.Stats{Path: "gs://bucket/group/data.txt"}, Expected: []task.Task{ {Type: "basic", Meta: "file=gs://bucket/group/data.txt&filename=data.txt&workflow=basic.toml"}, - {Type: "data", Meta: "file=gs://bucket/group/data.txt&filename=data.txt&job=1&workflow=data.toml"}, + {Type: "data", Job: "1", Meta: "file=gs://bucket/group/data.txt&filename=data.txt&job=1&workflow=data.toml"}, }, }, } diff --git a/apps/flowlord/handler.go b/apps/flowlord/handler.go index b35ce859..85ad3c9e 100644 --- a/apps/flowlord/handler.go +++ b/apps/flowlord/handler.go @@ -1,34 +1,108 @@ package main import ( + "bytes" + "embed" "encoding/json" "errors" + "html/template" "io" + "io/fs" "log" "net/http" - "path" "path/filepath" "strconv" "strings" "time" - "github.com/jbsmith7741/uri" - - "github.com/pcelvng/task-tools/slack" - + "github.com/dustin/go-humanize" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" gtools "github.com/jbsmith7741/go-tools" "github.com/jbsmith7741/go-tools/appenderr" + "github.com/jbsmith7741/uri" "github.com/pcelvng/task" tools "github.com/pcelvng/task-tools" + "github.com/pcelvng/task-tools/apps/flowlord/sqlite" "github.com/pcelvng/task-tools/file" - "github.com/pcelvng/task-tools/workflow" + "github.com/pcelvng/task-tools/slack" ) +//go:embed handler/alert.tmpl +var AlertTemplate string + +//go:embed handler/files.tmpl +var FilesTemplate string + +//go:embed handler/task.tmpl +var TaskTemplate string + +//go:embed handler/workflow.tmpl +var WorkflowTemplate string + +//go:embed handler/header.tmpl +var HeaderTemplate string + +//go:embed handler/about.tmpl +var AboutTemplate string + +//go:embed handler/static/* +var StaticFiles embed.FS + +var isLocal = false + +// getBaseFuncMap returns a template.FuncMap with all common template functions +func getBaseFuncMap() template.FuncMap { + return template.FuncMap{ + // Time formatting functions + "formatFullDate": func(t time.Time) string { + return t.Format(time.RFC3339) + }, + "formatTimeHour": func(t time.Time) string { + return t.Format("2006-01-02T15") + }, + // Duration formatting + "formatDuration": gtools.PrintDuration, + // Size formatting + "formatBytes": func(bytes int64) string { + if bytes < 0 { + return "0 B" + } + return humanize.Bytes(uint64(bytes)) + }, + // String manipulation + "slice": func(s string, start, end int) string { + if start >= len(s) { + return "" + } + if end > len(s) { + end = len(s) + } + return s[start:end] + }, + // Math functions + "add": func(a, b int) int { + return a + b + }, + } +} + func (tm *taskMaster) StartHandler() { router := chi.NewRouter() - router.Get("/", tm.Info) + + // Enable gzip compression for all responses + router.Use(middleware.Compress(5)) + + // Static file serving - serve embedded static files + // Create a sub-filesystem that strips the "handler/" prefix + staticFS, err := fs.Sub(StaticFiles, "handler/static") + if err != nil { + log.Fatal("Failed to create static filesystem:", err) + } + router.Handle("/static/*", http.StripPrefix("/static/", http.FileServer(http.FS(staticFS)))) + + router.Get("/", tm.htmlAbout) router.Get("/info", tm.Info) router.Get("/refresh", tm.refreshHandler) router.Post("/backload", tm.Backloader) @@ -50,6 +124,11 @@ func (tm *taskMaster) StartHandler() { }) router.Get("/task/{id}", tm.taskHandler) router.Get("/recap", tm.recapHandler) + router.Get("/web/alert", tm.htmlAlert) + router.Get("/web/files", tm.htmlFiles) + router.Get("/web/task", tm.htmlTask) + router.Get("/web/workflow", tm.htmlWorkflow) + router.Get("/web/about", tm.htmlAbout) if tm.port == 0 { log.Println("flowlord router disabled") @@ -71,13 +150,18 @@ func (tm *taskMaster) Info(w http.ResponseWriter, r *http.Request) { } // create a copy of all workflows - wCache := make(map[string]map[string]workflow.Phase) // [file][task:job]Phase - for key, w := range tm.Cache.Workflows { - phases := make(map[string]workflow.Phase) - for _, j := range w.Phases { - phases[pName(j.Topic(), j.Job())] = j + wCache := make(map[string]map[string]sqlite.Phase) // [file][task:job]Phase + workflowFiles := tm.taskCache.GetWorkflowFiles() + for _, filePath := range workflowFiles { + phases, err := tm.taskCache.GetPhasesForWorkflow(filePath) + if err != nil { + continue } - wCache[key] = phases + phaseMap := make(map[string]sqlite.Phase) + for _, j := range phases { + phaseMap[pName(j.Phase.Topic(), j.Phase.Job())] = j.Phase + } + wCache[filePath] = phaseMap } entries := tm.cron.Entries() for i := 0; i < len(entries); i++ { @@ -150,27 +234,27 @@ func (tm *taskMaster) Info(w http.ResponseWriter, r *http.Request) { // Add non cron based tasks for f, w := range wCache { - for _, v := range w { - k := pName(v.Topic(), v.Job()) + for _, ph := range w { + k := pName(ph.Topic(), ph.Job()) // check for parents - for v.DependsOn != "" { - if t, found := wCache[f][v.DependsOn]; found { - k = v.DependsOn - v = t + for ph.DependsOn != "" { + if t, found := wCache[f][ph.DependsOn]; found { + k = ph.DependsOn + ph = t } else { break } } - children := tm.getAllChildren(v.Topic(), f, v.Job()) - // todo: remove children from Cache + children := tm.getAllChildren(ph.Topic(), f, ph.Job()) + // todo: remove children from SQLite if _, found := sts.Workflow[f]; !found { sts.Workflow[f] = make(map[string]cEntry) } - warning := validatePhase(v) - if v.DependsOn != "" { - warning += "parent task not found: " + v.DependsOn + warning := ph.Validate() + if ph.DependsOn != "" { + warning += "parent task not found: " + ph.DependsOn } sts.Workflow[f][k] = cEntry{ Schedule: make([]string, 0), @@ -206,7 +290,7 @@ func (tm *taskMaster) refreshHandler(w http.ResponseWriter, _ *http.Request) { func (tm *taskMaster) taskHandler(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") - v := tm.taskCache.Get(id) + v := tm.taskCache.GetTask(id) b, _ := json.Marshal(v) w.Header().Add("Content-Type", "application/json") w.Write(b) @@ -214,7 +298,7 @@ func (tm *taskMaster) taskHandler(w http.ResponseWriter, r *http.Request) { func (tm *taskMaster) recapHandler(w http.ResponseWriter, r *http.Request) { - data := tm.taskCache.Recap() + data := tm.taskCache.Recap(time.Now().UTC()) if r.Header.Get("Accept") == "application/json" { b, err := json.Marshal(data) @@ -242,13 +326,10 @@ func (tm *taskMaster) workflowFiles(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) return } - var pth string + pth := tm.path // support directory and single file for workflow path lookup. - if _, f := path.Split(tm.path); f == "" { - pth = tm.path + "/" + fName - } else { - // for single file show the file regardless of the file param - pth = tm.path + if tm.taskCache.IsDir() { + pth += "/" + fName } sts, err := file.Stat(pth, tm.fOpts) @@ -258,12 +339,12 @@ func (tm *taskMaster) workflowFiles(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "text/plain") if sts.IsDir { + w.WriteHeader(http.StatusOK) files, _ := file.List(pth, tm.fOpts) for _, f := range files { b, a, _ := strings.Cut(f.Path, tm.path) w.Write([]byte(b + a + "\n")) } - w.WriteHeader(http.StatusOK) return } reader, err := file.NewReader(pth, tm.fOpts) @@ -279,13 +360,387 @@ func (tm *taskMaster) workflowFiles(w http.ResponseWriter, r *http.Request) { case "json": w.Header().Set("Content-Type", "application/json") case "yaml", "yml": - w.Header().Set("Context-Type", "text/x-yaml") + w.Header().Set("Content-Type", "text/x-yaml") } - b, _ := io.ReadAll(reader) w.WriteHeader(http.StatusOK) + b, _ := io.ReadAll(reader) w.Write(b) } +func (tm *taskMaster) htmlAlert(w http.ResponseWriter, r *http.Request) { + + dt, _ := time.Parse("2006-01-02", r.URL.Query().Get("date")) + if dt.IsZero() { + dt = time.Now() + } + alerts, err := tm.taskCache.GetAlertsByDate(dt) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + // Get dates with alerts for calendar highlighting + datesWithData, _ := tm.taskCache.DatesByType("alerts") + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + w.Write(alertHTML(alerts, dt, datesWithData)) +} + +// htmlFiles handles GET /web/files - displays file messages for a specific date +func (tm *taskMaster) htmlFiles(w http.ResponseWriter, r *http.Request) { + dt, _ := time.Parse("2006-01-02", r.URL.Query().Get("date")) + if dt.IsZero() { + dt = time.Now() + } + + files, err := tm.taskCache.GetFileMessagesByDate(dt) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + // Get dates with file messages for calendar highlighting + datesWithData, _ := tm.taskCache.DatesByType("files") + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + w.Write(filesHTML(files, dt, datesWithData)) +} + +// htmlTask handles GET /web/task - displays task summary and table for a specific date +func (tm *taskMaster) htmlTask(w http.ResponseWriter, r *http.Request) { + dt, _ := time.Parse("2006-01-02", r.URL.Query().Get("date")) + if dt.IsZero() { + dt = time.Now() + } + + // Get filter parameters from query string + page := 1 + if pageStr := r.URL.Query().Get("page"); pageStr != "" { + if p, err := strconv.Atoi(pageStr); err == nil && p > 0 { + page = p + } + } + + filter := &sqlite.TaskFilter{ + ID: r.URL.Query().Get("id"), + Type: r.URL.Query().Get("type"), + Job: r.URL.Query().Get("job"), + Result: r.URL.Query().Get("result"), + Page: page, + Limit: sqlite.DefaultPageSize, + } + + // Get task summary statistics for the date + summaryStart := time.Now() + taskStats, err := tm.taskCache.GetTaskSummaryByDate(dt) + summaryTime := time.Since(summaryStart) + if err != nil { + log.Printf("Error getting task summary: %v", err) + taskStats = sqlite.TaskStats{} + } + + // Get filtered and paginated tasks + queryStart := time.Now() + tasks, totalCount, err := tm.taskCache.GetTasksByDate(dt, filter) + queryTime := time.Since(queryStart) + if err != nil { + log.Printf("Error getting tasks: %v", err) + tasks = []sqlite.TaskView{} + totalCount = 0 + } + + // Get dates with tasks for calendar highlighting + datesWithData, _ := tm.taskCache.DatesByType("tasks") + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + htmlBytes := taskHTML(tasks, taskStats, totalCount, dt, filter, datesWithData, summaryTime+queryTime) + w.Write(htmlBytes) +} + +// htmlWorkflow handles GET /web/workflow - displays workflow phases from database +func (tm *taskMaster) htmlWorkflow(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + w.Write(workflowHTML(tm.taskCache)) +} + +// htmlAbout handles GET /web/about - displays system information and cache statistics +func (tm *taskMaster) htmlAbout(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/html") + w.Write(tm.aboutHTML()) +} + +// filesHTML renders the file messages HTML page +func filesHTML(files []sqlite.FileMessage, date time.Time, datesWithData []string) []byte { + // Calculate statistics + totalFiles := len(files) + matchedFiles := 0 + totalTasks := 0 + + for _, file := range files { + if len(file.TaskNames) > 0 { + matchedFiles++ + totalTasks += len(file.TaskNames) + } + } + + unmatchedFiles := totalFiles - matchedFiles + + // Calculate navigation dates + prevDate := date.AddDate(0, 0, -1) + nextDate := date.AddDate(0, 0, 1) + + data := map[string]interface{}{ + "Date": date.Format("Monday, January 2, 2006"), + "DateValue": date.Format("2006-01-02"), + "PrevDate": prevDate.Format("2006-01-02"), + "NextDate": nextDate.Format("2006-01-02"), + "Files": files, + "TotalFiles": totalFiles, + "MatchedFiles": matchedFiles, + "UnmatchedFiles": unmatchedFiles, + "TotalTasks": totalTasks, + "CurrentPage": "files", + "PageTitle": "File Messages", + "isLocal": isLocal, + "DatesWithData": datesWithData, + } + + // Parse and execute template using the shared funcMap + tmpl, err := template.New("files").Funcs(getBaseFuncMap()).Parse(HeaderTemplate + FilesTemplate) + if err != nil { + return []byte(err.Error()) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return []byte(err.Error()) + } + + return buf.Bytes() +} + +// taskHTML renders the task summary and table HTML page +func taskHTML(tasks []sqlite.TaskView, taskStats sqlite.TaskStats, totalCount int, date time.Time, filter *sqlite.TaskFilter, datesWithData []string, queryTime time.Duration) []byte { + renderStart := time.Now() + + // Calculate navigation dates + prevDate := date.AddDate(0, 0, -1) + nextDate := date.AddDate(0, 0, 1) + + // Get aggregate counts from TaskStats + counts := taskStats.TotalCounts() + + // Get unique types and jobs from TaskStats for filter dropdowns + types := taskStats.UniqueTypes() + jobsByType := taskStats.JobsByType() + + // Calculate pagination info + totalPages := (totalCount + filter.Limit - 1) / filter.Limit + if totalPages == 0 { + totalPages = 1 + } + + // Calculate display indices + startIdx := (filter.Page-1)*filter.Limit + 1 + endIdx := startIdx + len(tasks) - 1 + if len(tasks) == 0 { + startIdx = 0 + endIdx = 0 + } + + data := map[string]interface{}{ + "Date": date.Format("Monday, January 2, 2006"), + "DateValue": date.Format("2006-01-02"), + "PrevDate": prevDate.Format("2006-01-02"), + "NextDate": nextDate.Format("2006-01-02"), + "Tasks": tasks, + "Counts": counts, + "Filter": filter, + "CurrentPage": "task", + "PageTitle": "Task Dashboard", + "isLocal": isLocal, + "DatesWithData": datesWithData, + "UniqueTypes": types, + "JobsByType": jobsByType, + // Pagination info + "Page": filter.Page, + "PageSize": filter.Limit, + "TotalPages": totalPages, + "StartIndex": startIdx, + "EndIndex": endIdx, + "FilteredCount": totalCount, + } + + // Parse and execute template using base funcMap + tmpl, err := template.New("task").Funcs(getBaseFuncMap()).Parse(HeaderTemplate + TaskTemplate) + if err != nil { + return []byte(err.Error()) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return []byte(err.Error()) + } + + htmlSize := buf.Len() + renderTime := time.Since(renderStart) + + // Single consolidated log with all metrics + log.Printf("Task page: date=%s filters=[id=%q type=%q job=%q result=%q] total=%d filtered=%d page=%d/%d query=%v render=%v size=%.2fMB", + date.Format("2006-01-02"), filter.ID, filter.Type, filter.Job, filter.Result, + counts.Total, totalCount, filter.Page, totalPages, + queryTime, renderTime, float64(htmlSize)/(1024*1024)) + + return buf.Bytes() +} + +// workflowHTML renders the workflow phases HTML page +func workflowHTML(tCache *sqlite.SQLite) []byte { + // Get all workflow files and their phases + workflowFiles := tCache.GetWorkflowFiles() + + workflowFileSummary := make(map[string]int) + allPhases := make([]sqlite.PhaseDB, 0) + + for _, filePath := range workflowFiles { + phases, err := tCache.GetPhasesForWorkflow(filePath) + if err != nil { + continue + } + + workflowFileSummary[filePath] = len(phases) + allPhases = append(allPhases, phases...) + } + + data := map[string]interface{}{ + "Phases": allPhases, + "WorkflowFileSummary": workflowFileSummary, + "CurrentPage": "workflow", + "PageTitle": "Workflow Dashboard", + "isLocal": isLocal, + "DatesWithData": []string{}, // Workflow page doesn't use date picker with highlights + } + + // Parse and execute template using the shared funcMap + tmpl, err := template.New("workflow").Funcs(getBaseFuncMap()).Parse(HeaderTemplate + WorkflowTemplate) + if err != nil { + return []byte("Error:" + err.Error()) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return []byte("Error:" + err.Error()) + } + + return buf.Bytes() +} + +// aboutHTML renders the about page HTML +func (tm *taskMaster) aboutHTML() []byte { + // Get basic system information + sts := stats{ + AppName: "flowlord", + Version: tools.Version, + RunTime: gtools.PrintDuration(time.Since(tm.initTime)), + NextUpdate: tm.nextUpdate.Format("2006-01-02T15:04:05"), + LastUpdate: tm.lastUpdate.Format("2006-01-02T15:04:05"), + } + + // Get database size information + dbSize, err := tm.taskCache.GetDBSize() + if err != nil { + return []byte("Error getting database size: " + err.Error()) + } + + // Get table statistics + tableStats, err := tm.taskCache.GetTableStats() + if err != nil { + return []byte("Error getting table statistics: " + err.Error()) + } + + // Create data structure for template + data := map[string]interface{}{ + "AppName": sts.AppName, + "Version": sts.Version, + "RunTime": sts.RunTime, + "LastUpdate": sts.LastUpdate, + "NextUpdate": sts.NextUpdate, + "TotalDBSize": dbSize.TotalSize, + "PageCount": dbSize.PageCount, + "PageSize": dbSize.PageSize, + "DBPath": dbSize.DBPath, + "TableStats": tableStats, + "SchemaVersion": tm.taskCache.GetSchemaVersion(), + "Retention": gtools.PrintDuration(tm.taskCache.Retention), + "TaskTTL": gtools.PrintDuration(tm.taskCache.TaskTTL), + "MinFrequency": gtools.PrintDuration(tm.slack.MinFrequency), + "MaxFrequency": gtools.PrintDuration(tm.slack.MaxFrequency), + "CurrentFrequency": gtools.PrintDuration(tm.slack.GetCurrentDuration()), + "CurrentPage": "about", + "DateValue": "", // About page doesn't need date + "PageTitle": "System Information", + "isLocal": isLocal, + "DatesWithData": []string{}, // About page doesn't use date picker with highlights + } + + // Parse and execute template using the shared funcMap + tmpl, err := template.New("about").Funcs(getBaseFuncMap()).Parse(HeaderTemplate + AboutTemplate) + if err != nil { + return []byte("Error parsing template: " + err.Error()) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return []byte("Error executing template: " + err.Error()) + } + + return buf.Bytes() +} + +// AlertData holds both the alerts and summary data for the template +type AlertData struct { + Alerts []sqlite.AlertRecord + Summary []sqlite.SummaryLine +} + +// alertHTML will take a list of task and display a html webpage that is easily to digest what is going on. +func alertHTML(tasks []sqlite.AlertRecord, date time.Time, datesWithData []string) []byte { + // Generate summary data using BuildCompactSummary + summary := sqlite.BuildCompactSummary(tasks) + + // Create data structure for template + data := map[string]interface{}{ + "Alerts": tasks, + "Summary": summary, + "CurrentPage": "alert", + "DateValue": date.Format("2006-01-02"), + "Date": date.Format("Monday, January 2, 2006"), + "PageTitle": "Task Alerts", + "isLocal": isLocal, + "DatesWithData": datesWithData, + } + + // Parse and execute template using the shared funcMap + tmpl, err := template.New("alert").Funcs(getBaseFuncMap()).Parse(HeaderTemplate + AlertTemplate) + if err != nil { + return []byte(err.Error()) + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return []byte(err.Error()) + } + + return buf.Bytes() +} + type request struct { From string // start To string // end @@ -370,12 +825,12 @@ func (tm *taskMaster) backload(req request) response { start = at end = at } - - workflowPath, phase := tm.Cache.Search(req.Task, req.Job) - if workflowPath != "" { - msg = append(msg, "phase found in "+workflowPath) + + phase := tm.taskCache.Search(req.Task, req.Job) + if phase.FilePath != "" { + msg = append(msg, "phase found in "+phase.FilePath) req.Template = phase.Template - req.Workflow = workflowPath + req.Workflow = phase.FilePath } if req.Template == "" { name := req.Task diff --git a/apps/flowlord/handler/about.tmpl b/apps/flowlord/handler/about.tmpl new file mode 100644 index 00000000..ae77ba28 --- /dev/null +++ b/apps/flowlord/handler/about.tmpl @@ -0,0 +1,138 @@ + + + + + + Flowlord: About + + + + + {{template "header" .}} +
+ +
+
+
+

Application

+
+ App Name + {{.AppName}} +
+
+ Version + {{.Version}} +
+
+ Runtime + {{.RunTime}} +
+
+ +
+

Cache Status

+
+ Last Update + {{.LastUpdate}} +
+
+ Next Update + {{.NextUpdate}} +
+
+ Database + {{.DBPath}} +
+
+ Schema Version + {{.SchemaVersion}} +
+
+ +
+

Database Size

+
+ Total Size + {{.TotalDBSize}} +
+
+ Page Count + {{.PageCount}} +
+
+ Page Size + {{.PageSize}} +
+
+ +
+

Cache Settings

+
+ Retention Period + {{.Retention}} +
+
+ Task TTL + {{.TaskTTL}} +
+
+ +
+

Notification Settings

+
+ Current Frequency + {{.CurrentFrequency}} +
+
+ Min Frequency + {{.MinFrequency}} +
+
+ Max Frequency + {{.MaxFrequency}} +
+
+
+ +
+

Table Breakdown

+
+ + + + + + + + + + + + + {{range .TableStats}} + + + + + + + + + {{end}} + +
Table NameRow CountTable SizeIndex SizeTotal SizePercentage
{{.Name}}{{.RowCount}}{{.TableHuman}}{{.IndexHuman}}{{.TotalHuman}}{{printf "%.1f" .Percentage}}%
+
+
+
+
+ + + + diff --git a/apps/flowlord/handler/alert.tmpl b/apps/flowlord/handler/alert.tmpl new file mode 100644 index 00000000..ae7f3ee5 --- /dev/null +++ b/apps/flowlord/handler/alert.tmpl @@ -0,0 +1,337 @@ + + + + + + Flowlord: Alerts + + + + + {{template "header" .}} +
+
+

Alert Summary

+
+ {{range .Summary}} +
+
+ {{.Key}} + {{.Count}} alerts +
+
{{.TimeRange}}
+
+ {{end}} +
+
+
+ + + + + + + + + + + + + {{range .Alerts}} + + + + + + + + + {{end}} + +
IDTask TypeJobMessageAlerted AtTask Time
+ {{.TaskID}} + {{.Type}}{{.Job}} + {{.Msg}} + {{.CreatedAt.Format "2006-01-02T15:04:05Z"}}{{if .TaskTime.IsZero}}N/A{{else}}{{.TaskTime.Format "2006-01-02T15"}}{{end}}
+
+
+ Total Alerts: {{len .Alerts}} +
+
+ + + + + \ No newline at end of file diff --git a/apps/flowlord/handler/files.tmpl b/apps/flowlord/handler/files.tmpl new file mode 100644 index 00000000..a4c00139 --- /dev/null +++ b/apps/flowlord/handler/files.tmpl @@ -0,0 +1,147 @@ + + + + + + Flowlord: Files + + + + + {{template "header" .}} +
+ + {{if .Files}} +
+
+
+
{{.TotalFiles}}
+
Total Files
+
+
+
{{.MatchedFiles}}
+
With Tasks
+
+
+
{{.UnmatchedFiles}}
+
No Matches
+
+
+
{{.TotalTasks}}
+
Tasks Created
+
+
+
+ +
+ {{range .Files}} +
+
{{.Path}}
+
+ Size: {{.Size | formatBytes}} + Received: {{.ReceivedAt | formatFullDate}} + {{if not .LastModified.IsZero}} + Last Modified: {{.LastModified | formatFullDate}} + {{end}} + {{if not .TaskTime.IsZero}} + Task Time: {{.TaskTime | formatTimeHour}} + {{end}} +
+ {{if .TaskIDs}} +
+
Task IDs: + {{range $index, $taskID := .TaskIDs}} + {{$taskID}} + {{end}} +
+
Tasks Names: + {{range .TaskNames}} + {{.}} + {{end}} +
+
+ {{else}} +
+
No matching patterns found
+
+ {{end}} +
+ {{end}} +
+ {{else}} +
+

No file messages found

+

No files were processed on {{.Date}}

+
+ {{end}} +
+ + + + diff --git a/apps/flowlord/handler/header.tmpl b/apps/flowlord/handler/header.tmpl new file mode 100644 index 00000000..27c90e26 --- /dev/null +++ b/apps/flowlord/handler/header.tmpl @@ -0,0 +1,78 @@ +{{define "header"}} +
+
+
+ {{if and (ne .CurrentPage "workflow") (ne .CurrentPage "about")}} +
+ + + +
+ {{end}} + {{if .ShowRefresh}} + + {{end}} +
+
+

{{if .PageTitle}}{{.PageTitle}}{{else}}Flowlord Dashboard{{end}}

+
+ +
+
+ + + + +{{end}} diff --git a/apps/flowlord/handler/static/calendar.js b/apps/flowlord/handler/static/calendar.js new file mode 100644 index 00000000..29a7d8c8 --- /dev/null +++ b/apps/flowlord/handler/static/calendar.js @@ -0,0 +1,163 @@ +// Calendar and Date Picker functionality +(function() { + 'use strict'; + + function initCalendar(datesWithData) { + const datePicker = document.getElementById('datePicker'); + const todayBtn = document.getElementById('todayBtn'); + + if (!datePicker) return; + + // Convert array to Set for faster lookup + const datesSet = new Set(datesWithData || []); + + // Set today's date as default if no date is provided + if (!datePicker.value) { + const today = new Date().toISOString().split('T')[0]; + datePicker.value = today; + } + + // Update data indicator + function updateDataIndicator(date) { + if (datesSet.has(date)) { + datePicker.classList.add('has-data'); + } else { + datePicker.classList.remove('has-data'); + } + } + + // Initial indicator update + updateDataIndicator(datePicker.value); + + // Create custom date picker dropdown + const dropdown = document.createElement('div'); + dropdown.id = 'dateDropdown'; + dropdown.className = 'date-dropdown'; + dropdown.style.display = 'none'; + + // Generate calendar for current month and surrounding dates + function generateCalendar() { + const currentDate = datePicker.value ? new Date(datePicker.value + 'T12:00:00') : new Date(); + const year = currentDate.getFullYear(); + const month = currentDate.getMonth(); + + let html = '
'; + html += ''; + html += '' + currentDate.toLocaleDateString('en-US', {month: 'long', year: 'numeric'}) + ''; + html += ''; + html += '
'; + + html += '
'; + html += '
Su
Mo
Tu
'; + html += '
We
Th
Fr
Sa
'; + + const firstDay = new Date(year, month, 1); + const lastDay = new Date(year, month + 1, 0); + const startDay = firstDay.getDay(); + + // Add empty cells for days before month starts + for (let i = 0; i < startDay; i++) { + html += '
'; + } + + // Add days of month + for (let day = 1; day <= lastDay.getDate(); day++) { + const dateStr = year + '-' + String(month + 1).padStart(2, '0') + '-' + String(day).padStart(2, '0'); + const hasData = datesSet.has(dateStr); + const isSelected = dateStr === datePicker.value; + + let classes = 'day-cell'; + if (hasData) classes += ' has-data'; + if (isSelected) classes += ' selected'; + + html += '
' + day + '
'; + } + + html += '
'; + dropdown.innerHTML = html; + } + + // Toggle dropdown + datePicker.addEventListener('click', function(e) { + e.stopPropagation(); + if (dropdown.style.display === 'none') { + generateCalendar(); + dropdown.style.display = 'block'; + + // Position dropdown + const rect = datePicker.getBoundingClientRect(); + dropdown.style.position = 'absolute'; + dropdown.style.top = (rect.bottom + 5) + 'px'; + dropdown.style.left = rect.left + 'px'; + } else { + dropdown.style.display = 'none'; + } + }); + + // Handle dropdown clicks + dropdown.addEventListener('click', function(e) { + e.stopPropagation(); + + if (e.target.classList.contains('day-cell') && !e.target.classList.contains('empty')) { + const selectedDate = e.target.getAttribute('data-date'); + datePicker.value = selectedDate; + updateDataIndicator(selectedDate); + dropdown.style.display = 'none'; + + // Navigate to date + navigateToDate(selectedDate); + } else if (e.target.classList.contains('month-nav')) { + const dir = parseInt(e.target.getAttribute('data-dir')); + const currentDate = new Date(datePicker.value + 'T12:00:00'); + currentDate.setMonth(currentDate.getMonth() + dir); + datePicker.value = currentDate.toISOString().split('T')[0]; + generateCalendar(); + } + }); + + // Close dropdown when clicking outside + document.addEventListener('click', function() { + dropdown.style.display = 'none'; + }); + + // Handle today button + if (todayBtn) { + todayBtn.addEventListener('click', function(e) { + e.stopPropagation(); + const today = new Date().toISOString().split('T')[0]; + datePicker.value = today; + updateDataIndicator(today); + dropdown.style.display = 'none'; + navigateToDate(today); + }); + } + + // Navigate to selected date + function navigateToDate(selectedDate) { + const currentUrl = new URL(window.location); + const currentPage = currentUrl.pathname; + + const newUrl = new URL(currentPage, window.location.origin); + newUrl.searchParams.set('date', selectedDate); + + // Preserve other query parameters + const otherParams = ['type', 'job', 'result', 'sort', 'direction']; + otherParams.forEach(param => { + if (currentUrl.searchParams.has(param)) { + newUrl.searchParams.set(param, currentUrl.searchParams.get(param)); + } + }); + + window.location.href = newUrl.toString(); + } + + // Append dropdown to body + document.body.appendChild(dropdown); + } + + // Export to global scope + window.FlowlordCalendar = { + init: initCalendar + }; +})(); + diff --git a/apps/flowlord/handler/static/favicon.svg b/apps/flowlord/handler/static/favicon.svg new file mode 100644 index 00000000..21e878e7 --- /dev/null +++ b/apps/flowlord/handler/static/favicon.svg @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/flowlord/handler/static/style.css b/apps/flowlord/handler/static/style.css new file mode 100644 index 00000000..930852aa --- /dev/null +++ b/apps/flowlord/handler/static/style.css @@ -0,0 +1,1290 @@ +/* Flowlord Dashboard Styles */ + +/* ===== UTILITY CLASSES ===== */ +/* Common font families */ +.font-mono { + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; +} + +/* ===== BASE STYLES ===== */ +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + margin: 0; + padding: 20px; + background-color: #f5f5f5; +} + +.container { + max-width: 1200px; + margin: 0 auto; + background: white; + border-radius: 8px; + box-shadow: 0 2px 10px rgba(0,0,0,0.1); + overflow: hidden; +} + +/* ===== HEADER STYLES - removed unused classes ===== */ + +/* ===== NAVIGATION STYLES ===== */ +.nav-header { + background: linear-gradient(135deg, #2c3e50 0%, #34495e 100%); + color: white; + padding: 0; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + border-radius: 12px 12px 0 0; + margin: 0; +} + +.nav-container { + width: 100%; + display: flex; + align-items: center; + padding: 0 15px; + flex-wrap: nowrap; + gap: 20px; + box-sizing: border-box; +} + +.nav-brand { + flex: 1; + text-align: center; + order: 2; +} + +.nav-brand h1 { + margin: 0; + font-size: 20px; + font-weight: 600; + padding: 15px 0; +} + +.nav-menu { + display: flex; + gap: 0; + order: 3; + flex-shrink: 0; + white-space: nowrap; +} + +.nav-link { + display: flex; + align-items: center; + gap: 8px; + padding: 15px 20px; + color: rgba(255, 255, 255, 0.8); + text-decoration: none; + font-weight: 500; + font-size: 14px; + transition: all 0.2s ease; + border-bottom: 3px solid transparent; +} + +.nav-link:hover { + color: white; + background: rgba(255, 255, 255, 0.1); +} + +.nav-link.active { + color: white; + background: rgba(255, 255, 255, 0.15); + border-bottom-color: #3498db; +} + +.nav-icon { + font-size: 16px; +} + +/* ===== DATE PICKER STYLES ===== */ +.nav-controls { + display: flex; + align-items: center; + gap: 15px; + order: 1; + margin: 0; + padding: 0; + flex-shrink: 0; +} + +.date-picker-container { + display: flex; + align-items: center; + gap: 8px; + background: rgba(255, 255, 255, 0.1); + padding: 8px 12px; + border-radius: 6px; + border: 1px solid rgba(255, 255, 255, 0.2); +} + +.date-picker-label { + color: rgba(255, 255, 255, 0.9); + font-size: 14px; + font-weight: 500; + margin: 0; + white-space: nowrap; +} + +.date-picker { + background: white; + border: 1px solid #ddd; + border-radius: 4px; + padding: 6px 8px; + font-size: 14px; + color: #333; + min-width: 140px; + cursor: pointer; +} + +.date-picker:focus { + outline: none; + border-color: #3498db; + box-shadow: 0 0 0 2px rgba(52, 152, 219, 0.2); +} + +.date-picker.has-data { + border-left: 3px solid #10b981; + background-color: #f0fdf4; +} + +.btn { + display: inline-block; + padding: 6px 12px; + font-size: 14px; + font-weight: 500; + text-align: center; + text-decoration: none; + border: 1px solid transparent; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s ease; + user-select: none; +} + +.btn-sm { + padding: 4px 8px; + font-size: 12px; +} + +.btn-outline { + color: rgba(255, 255, 255, 0.9); + background: transparent; + border-color: rgba(255, 255, 255, 0.3); +} + +.btn-outline:hover { + color: white; + background: rgba(255, 255, 255, 0.1); + border-color: rgba(255, 255, 255, 0.5); +} + +.btn-outline:active { + background: rgba(255, 255, 255, 0.2); +} + +/* Calendar dropdown */ +.date-dropdown { + background: white; + border: 1px solid #d1d5db; + border-radius: 8px; + box-shadow: 0 10px 25px rgba(0,0,0,0.15); + padding: 12px; + z-index: 1000; + min-width: 280px; +} + +.calendar-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 12px; + padding: 8px; +} + +.month-label { + font-weight: 600; + font-size: 0.95rem; + color: #1f2937; +} + +.month-nav { + background: #f3f4f6; + border: none; + border-radius: 4px; + padding: 4px 12px; + cursor: pointer; + font-size: 1rem; + transition: background 0.2s; +} + +.month-nav:hover { + background: #e5e7eb; +} + +.calendar-grid { + display: grid; + grid-template-columns: repeat(7, 1fr); + gap: 4px; +} + +.day-header { + text-align: center; + font-size: 0.75rem; + font-weight: 600; + color: #6b7280; + padding: 8px 4px; +} + +.day-cell { + text-align: center; + padding: 8px 4px; + cursor: pointer; + border-radius: 4px; + font-size: 0.875rem; + transition: all 0.2s; + position: relative; +} + +.day-cell:not(.empty):hover { + background: #f3f4f6; +} + +.day-cell.has-data { + background: #d1fae5; + font-weight: 600; + color: #065f46; +} + +.day-cell.has-data:hover { + background: #a7f3d0; +} + +.day-cell.selected { + background: #3b82f6; + color: white; + font-weight: 600; +} + +.day-cell.empty { + cursor: default; +} + +/* ===== DATE DISPLAY STYLES - removed unused classes ===== */ + +/* ===== CONTENT STYLES ===== */ +.content { + padding: 30px; +} + +.container { + padding-top: 20px; +} + +.full-width { + max-width: 100%; + margin: 0; + padding-left: 10px; + padding-right: 10px; +} + +/* ===== INFO GRID STYLES ===== */ +.info-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 20px; + margin-bottom: 30px; +} + +.info-card { + background: #f8f9fa; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 20px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); +} + +.info-card h3 { + margin: 0 0 15px 0; + color: #2c3e50; + font-size: 18px; + font-weight: 600; + border-bottom: 2px solid #3498db; + padding-bottom: 8px; +} + +.info-item { + display: flex; + justify-content: space-between; + align-items: center; + padding: 8px 0; + border-bottom: 1px solid #e9ecef; +} + +.info-item:last-child { + border-bottom: none; +} + +.info-label { + font-weight: 500; + margin-right: 10px; + color: #495057; +} + +.info-value { + color: #6c757d; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 14px; +} + +/* ===== CACHE SECTION STYLES ===== */ +.cache-section { + background: #f8f9fa; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 20px; + margin-top: 20px; +} + +.cache-section h3 { + margin: 0 0 15px 0; + color: #2c3e50; + font-size: 18px; + font-weight: 600; + border-bottom: 2px solid #e74c3c; + padding-bottom: 8px; +} + +/* ===== TABLE STYLES ===== */ +.table-container { + overflow-x: auto; + max-width: 100%; +} + +table { + border-collapse: collapse; + width: 100%; + margin: 0; + table-layout: fixed; + max-width: 100%; +} + +th, td { + border: 1px solid #e1e5e9; + padding: 12px 16px; + text-align: left; + vertical-align: top; +} + +th { + background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%); + font-weight: 600; + color: #495057; + cursor: pointer; + user-select: none; + position: relative; +} + +th:hover { + background: linear-gradient(135deg, #e9ecef 0%, #dee2e6 100%); +} + +th.sortable::after { + content: ' ↕'; + opacity: 0.5; + font-size: 12px; +} + +th.sort-asc::after { + content: ' ↑'; + opacity: 1; + color: #007bff; +} + +th.sort-desc::after { + content: ' ↓'; + opacity: 1; + color: #007bff; +} + +tr:nth-child(even) { + background-color: #f8f9fa; +} + +tr:hover { + background-color: #e3f2fd; +} + +/* ===== CELL STYLES ===== */ +.id-cell, .time-cell, .info-cell, .meta-cell, .size-cell { + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 13px; + color: #6c757d; +} + +/* Expandable cell styles for optimized truncation */ +.expandable { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + cursor: pointer; + user-select: text; + position: relative; +} + +.id-cell.expandable { + max-width: 100px; +} + +.message-cell.expandable, +.info-cell.expandable, +.meta-cell.expandable { + max-width: 180px; +} + +.expandable.expanded { + max-width: none !important; + white-space: normal; + word-break: break-word; + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 8px; + margin: 2px; + line-height: 1.4; +} + +.expandable:hover { + background-color: #f8f9fa; +} + + +.id-cell, .info-cell, .meta-cell { + cursor: pointer; + position: relative; + user-select: text; +} + +.id-cell:hover, .info-cell:hover, .meta-cell:hover { + background-color: #f8f9fa; +} + +.id-cell.truncated, .info-cell.truncated, .meta-cell.truncated, .rule-cell.truncated, .template-cell.truncated, .message-cell.truncated { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.id-cell.truncated { + max-width: 100px; +} + +.rule-cell.truncated, .template-cell.truncated { + max-width: 300px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.message-cell.truncated { + max-width: 400px; +} + +.id-cell.expanded, .info-cell.expanded, .meta-cell.expanded, .rule-cell.expanded, .template-cell.expanded, .message-cell.expanded { + max-width: none; + white-space: normal; + word-break: break-word; + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 8px; + margin: 2px; + line-height: 1.4; +} + +.info-cell.expanded, .meta-cell.expanded, .rule-cell.expanded, .template-cell.expanded, .message-cell.expanded { + word-wrap: break-word; +} + +.id-cell.copyable, .info-cell.copyable, .meta-cell.copyable, .rule-cell.copyable, .template-cell.copyable, .message-cell.copyable { + position: relative; + cursor: pointer; +} + +.rule-cell.copyable:hover, .template-cell.copyable:hover, .message-cell.copyable:hover { + background-color: #f8f9fa; +} + +.type-cell { + font-weight: 500; + color: #495057; +} + +.job-cell { + color: #6c757d; + font-style: italic; +} + +.result-cell { + font-weight: 500; +} + +.result-complete { color: #28a745; } +.result-error { color: #dc3545; } +.result-alert { color: #fd7e14; } +.result-warn { color: #ffc107; } +.result-running { color: #007bff; } + +.time-cell { + white-space: normal; + word-wrap: break-word; + word-break: break-word; + overflow-wrap: break-word; +} + +/* ===== UNIFIED TABLE COLUMN WIDTHS ===== */ +/* ID column - compact for IDs across all templates */ +th.id-column, td.id-column { + width: 7%; + max-width: 100px; +} + +/* Type and Job columns - standardized compact width */ +th.type-column, td.type-column, +th.job-column, td.job-column { + width: 6%; + max-width: 80px; +} + +/* Message column - takes remaining space with generous width */ +th.message-column, td.message-column { + width: auto; + min-width: 400px; +} + +/* Result/Status column - compact for status */ +th.result-column, td.result-column, +th.status-column, td.status-column { + width: 6%; + max-width: 80px; +} + +/* Info and Meta columns - medium space */ +th.info-column, td.info-column, +th.meta-column, td.meta-column { + width: 12%; + max-width: 180px; +} + +/* Timestamp columns - standard width for all time fields */ +th.created-column, td.created-column, +th.alerted-column, td.alerted-column, +th.tasktime-column, td.tasktime-column { + width: 10%; + max-width: 180px; + word-wrap: break-word; + word-break: break-word; + white-space: normal; + overflow-wrap: break-word; +} + +/* Duration columns - standardized for hh:mm:ss format */ +th.queue-column, td.queue-column, +th.process-column, td.process-column { + width: 5%; + max-width: 70px; +} + +/* ===== WORKFLOW-SPECIFIC COLUMN WIDTHS ===== */ +/* Workflow file column - compact for file names */ +th.workflow-file-column, td.workflow-file-column, +th.workflow-file-cell, td.workflow-file-cell { + width: 8%; + max-width: 120px; + word-break: break-all; +} + +/* Workflow task column - compact for task names */ +th.task-column, td.task-column, +th.task-cell, td.task-cell { + width: 8%; + max-width: 100px; + word-break: break-all; +} + +/* Workflow job column - compact for job names */ +th.job-column, td.job-column, +th.job-cell, td.job-cell { + width: 8%; + max-width: 100px; + word-break: break-all; +} + +/* Workflow rule and template columns - wide for content with better multi-line handling */ +th.rule-column, td.rule-column, +th.template-column, td.template-column, +th.rule-cell, td.rule-cell, +th.template-cell, td.template-cell { + width: 20%; + max-width: 300px; + word-break: break-word; + white-space: normal; + line-height: 1.4; +} + +/* Workflow depends-on column - medium width */ +th.depends-on-column, td.depends-on-column, +th.depends-on-cell, td.depends-on-cell { + width: 10%; + max-width: 120px; + word-break: break-all; +} + +/* Workflow retry column - very compact */ +th.retry-column, td.retry-column, +th.retry-cell, td.retry-cell { + width: 4%; + max-width: 60px; + text-align: center; +} + +/* Workflow status column - more space for status display */ +th.status-column, td.status-column, +th.status-cell, td.status-cell { + width: 8%; + max-width: 100px; + text-align: center; + font-weight: 500; +} + +/* Enhanced workflow table styling for better consistency */ +#workflowTable { + table-layout: fixed; + width: 100%; +} + +#workflowTable th, +#workflowTable td { + padding: 12px 8px; + vertical-align: top; +} + +/* Ensure workflow table doesn't overflow horizontally */ +.table-container { + overflow-x: auto; + max-width: 100%; + width: 100%; +} + +/* Better responsive handling for workflow table */ +@media (max-width: 1200px) { + th.rule-column, td.rule-column, + th.template-column, td.template-column { + width: 18%; + max-width: 250px; + } + + th.workflow-file-column, td.workflow-file-column { + width: 10%; + max-width: 100px; + } +} + +.message-cell { + max-width: none; + word-wrap: break-word; + line-height: 1.4; + cursor: pointer; + user-select: text; +} + +.message-cell:hover { + background-color: #f8f9fa; +} + +.message-cell.copyable { + position: relative; +} + +.percentage-cell { + font-weight: 500; + color: #495057; +} + +/* ===== BUTTON STYLES ===== */ +.refresh-btn, .btn { + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 14px; + transition: all 0.2s ease; +} + +.refresh-btn { + background: #3498db; + color: white; + padding: 10px 20px; + margin-bottom: 20px; +} + +.refresh-btn:hover { + background: #2980b9; +} + +.btn { + padding: 8px 16px; +} + +/* Removed unused btn-primary styles */ + +.btn-secondary { + background: #6c757d; + color: white; +} + +.btn-secondary:hover { + background: #545b62; +} + +/* ===== SUMMARY STYLES ===== */ +.summary-section { + padding: 20px; + background: #f8f9fa; + border-bottom: 1px solid #e1e5e9; +} + +/* ===== COLLAPSIBLE SECTION STYLES ===== */ +.collapsible-section { + margin-bottom: 20px; +} + +.collapsible-header { + display: flex; + align-items: baseline; + gap: 6px; + margin: 0 0 16px 0; + cursor: pointer; + transition: all 0.2s ease; + user-select: none; +} + +.collapsible-header:hover h3 { + color: #3498db; +} + +.collapsible-header h3 { + margin: 0; + font-size: 18px; + font-weight: 600; + color: #495057; + transition: color 0.2s ease; + line-height: 1; +} + +.collapsible-toggle { + font-size: 12px; + color: #6c757d; + transition: transform 0.2s ease; + line-height: 1; + display: inline-block; + vertical-align: baseline; + margin-top: 2px; +} + +.collapsible-toggle.expanded { + transform: rotate(180deg); +} + +.collapsible-content { + transition: all 0.3s ease; + overflow: hidden; +} + +.collapsible-content.collapsed { + max-height: 0; + opacity: 0; +} + +.summary-section h3 { + margin: 0 0 16px 0; + color: #495057; + font-size: 18px; + font-weight: 600; +} + +.summary { + background: #ecf0f1; + padding: 15px 20px; + border-bottom: 1px solid #bdc3c7; +} + +.summary-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 16px; +} + +.summary-card { + background: white; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 16px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); + transition: box-shadow 0.2s ease; +} + +.summary-card:hover { + box-shadow: 0 2px 8px rgba(0,0,0,0.15); +} + +.summary-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.summary-key { + font-weight: 600; + color: #495057; + font-size: 16px; +} + +.summary-count { + background: #007bff; + color: white; + padding: 4px 8px; + border-radius: 12px; + font-size: 12px; + font-weight: 500; +} + +.summary-time, .summary-details { + color: #6c757d; + font-size: 14px; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; +} + +.summary-stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; + margin-bottom: 20px; +} + +/* ===== TOPIC SUMMARY STYLES ===== */ +.topic-summary { + display: flex; + flex-direction: column; + gap: 20px; +} + +.topic-card { + background: white; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 20px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); + transition: box-shadow 0.2s ease; +} + +.topic-card:hover { + box-shadow: 0 2px 8px rgba(0,0,0,0.15); +} + +.topic-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 1px solid #e1e5e9; +} + +.topic-name { + margin: 0; + color: #2c3e50; + font-size: 20px; + font-weight: 600; +} + +.topic-total { + background: #007bff; + color: white; + padding: 6px 12px; + border-radius: 16px; + font-size: 14px; + font-weight: 500; +} + +.jobs-breakdown { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 16px; +} + +.job-card { + background: #f8f9fa; + border: 1px solid #e9ecef; + border-radius: 6px; + padding: 16px; + transition: all 0.2s ease; +} + +.job-card:hover { + background: #e9ecef; + border-color: #dee2e6; +} + +.job-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 12px; +} + +.job-name { + font-weight: 600; + color: #495057; + font-size: 16px; +} + +.job-total { + background: #6c757d; + color: white; + padding: 4px 8px; + border-radius: 12px; + font-size: 12px; + font-weight: 500; +} + +.job-metrics { + display: flex; + flex-direction: column; + gap: 8px; +} + +.metric-row { + display: flex; + justify-content: space-between; + align-items: center; + padding: 4px 0; +} + +.metric-label { + font-size: 14px; + color: #6c757d; + font-weight: 500; +} + +.metric-value { + font-size: 14px; + font-weight: 600; + padding: 2px 8px; + border-radius: 8px; + min-width: 30px; + text-align: center; +} + +.metric-value.complete { + background: #d4edda; + color: #155724; +} + +.metric-value.error { + background: #f8d7da; + color: #721c24; +} + +.metric-value.alert { + background: #fff3cd; + color: #856404; +} + +.metric-value.warn { + background: #ffeaa7; + color: #6c5ce7; +} + +.stat-card { + background: white; + border: 1px solid #e1e5e9; + border-radius: 8px; + padding: 16px; + text-align: center; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); + cursor: pointer; + transition: all 0.2s ease; +} + +.stat-card:hover { + background: #f8f9fa; + border-color: #3498db; + box-shadow: 0 2px 8px rgba(0,0,0,0.15); +} + +.stat-card.active { + background: #3498db; + border-color: #2980b9; + color: white; + box-shadow: 0 2px 8px rgba(52, 152, 219, 0.3); +} + +.stat-card.active .stat-number { + color: white; +} + +.stat-card.active .stat-label { + color: rgba(255, 255, 255, 0.9); +} + +.stat-number { + font-size: 24px; + font-weight: bold; + color: #2c3e50; + margin-bottom: 4px; +} + +.stat-label { + font-size: 12px; + color: #6c757d; + text-transform: uppercase; +} + +/* Removed unused .stat class - now using .stat-card */ + +/* ===== FILES LIST STYLES ===== */ +.files-list { + padding: 0; +} + +.file-item { + border-bottom: 1px solid #ecf0f1; + padding: 15px 20px; + transition: background-color 0.2s; +} + +.file-item:hover { + background-color: #f8f9fa; +} + +.file-item:last-child { + border-bottom: none; +} + +.file-path { + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 14px; + color: #2c3e50; + margin-bottom: 5px; + word-break: break-all; +} + +.file-meta { + display: flex; + gap: 20px; + font-size: 12px; + color: #7f8c8d; + margin-bottom: 8px; +} + +.file-meta span { + display: flex; + align-items: center; + gap: 4px; +} + +.tasks-section { + margin-top: 10px; +} + +.tasks-label { + font-size: 12px; + color: #7f8c8d; + margin-bottom: 5px; + line-height: 1.4; +} + +.task-id { + font-family: monospace; + background: #ecf0f1; + padding: 2px 6px; + border-radius: 3px; + font-size: 11px; + color: #2c3e50; +} + +.task-tag { + background: #3498db; + color: white; + padding: 2px 8px; + border-radius: 12px; + font-size: 11px; + font-weight: 500; +} + +.task-tag.error { + background: #e74c3c; +} + +.task-tag.success { + background: #27ae60; +} + +/* ===== FILTER STYLES ===== */ +.filters { + background: #f8f9fa; + padding: 20px; + border-bottom: 1px solid #e1e5e9; +} + +.filters h3 { + margin: 0 0 16px 0; + color: #495057; + font-size: 18px; +} + +.filter-row { + display: flex; + gap: 16px; + flex-wrap: wrap; + align-items: center; +} + +.filter-group { + display: flex; + flex-direction: column; + gap: 4px; +} + +.filter-group label { + font-size: 12px; + font-weight: 500; + color: #6c757d; + text-transform: uppercase; +} + +.filter-group input, .filter-group select { + padding: 8px 12px; + border: 1px solid #ced4da; + border-radius: 4px; + font-size: 14px; + min-width: 120px; +} + +.filter-group input:focus, .filter-group select:focus { + outline: none; + border-color: #007bff; + box-shadow: 0 0 0 2px rgba(0, 123, 255, 0.25); +} + +.filter-buttons { + display: flex; + gap: 8px; + margin-left: auto; +} + +/* ===== STATS STYLES ===== */ +.stats { + padding: 16px 20px; + background-color: #f8f9fa; + border-top: 1px solid #e1e5e9; + font-size: 14px; + color: #6c757d; +} + +/* ===== NO DATA STYLES ===== */ +.no-tasks, .no-files { + text-align: center; + padding: 40px 20px; + color: #7f8c8d; +} + +.no-tasks h3, .no-files h3 { + margin: 0 0 10px 0; + color: #95a5a6; +} + +.no-tasks { + color: #6c6c6c; + font-style: italic; + font-size: 12px; + background: linear-gradient(135deg, #fff9e6 0%, #f5f5f0 100%); + border: 1px solid #e8e4d0; + border-radius: 4px; + padding: 8px 16px; + display: inline-block; +} + +/* ===== CONTEXT MENU STYLES ===== */ +.context-menu { + position: fixed; + background: white; + border: 1px solid #dee2e6; + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + z-index: 1000; + min-width: 120px; + padding: 4px 0; +} + +.context-menu-item { + padding: 8px 16px; + cursor: pointer; + font-size: 14px; + color: #495057; + transition: background-color 0.2s ease; +} + +.context-menu-item:hover { + background-color: #f8f9fa; +} + +/* ===== COPY FEEDBACK STYLES ===== */ +.copy-feedback { + position: fixed; + background: #28a745; + color: white; + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + font-weight: 500; + z-index: 1001; + pointer-events: none; + animation: copyFeedbackFade 2s ease-out forwards; +} + +@keyframes copyFeedbackFade { + 0% { + opacity: 1; + transform: translateX(-50%) translateY(0); + } + 100% { + opacity: 0; + transform: translateX(-50%) translateY(-10px); + } +} + +/* Pagination Styles */ +.pagination { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem; + background: var(--card-bg); + border-radius: 8px; + margin: 1rem 0; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +.pagination-info { + color: var(--text-secondary); + font-size: 0.9rem; +} + +.pagination-controls { + display: flex; + gap: 0.5rem; +} + +.pagination-controls .btn { + padding: 0.5rem 1rem; + text-decoration: none; +} \ No newline at end of file diff --git a/apps/flowlord/handler/static/task.js b/apps/flowlord/handler/static/task.js new file mode 100644 index 00000000..43cef1cb --- /dev/null +++ b/apps/flowlord/handler/static/task.js @@ -0,0 +1,349 @@ +// Task page functionality +(function() { + 'use strict'; + + // Initialize task page with configuration + function initTaskPage(config) { + const table = document.getElementById('taskTable'); + if (!table) { + return; + } + + const tbody = table.querySelector('tbody'); + const headers = table.querySelectorAll('th.sortable'); + + let currentSort = { column: null, direction: 'asc' }; + + // Get URL parameters + function getUrlParams() { + const urlParams = new URLSearchParams(window.location.search); + return { + date: urlParams.get('date') || '', + type: urlParams.get('type') || '', + job: urlParams.get('job') || '', + result: urlParams.get('result') || '', + sort: urlParams.get('sort') || '', + direction: urlParams.get('direction') || 'asc' + }; + } + + // Update URL with new parameters + function updateUrl(date, type, job, result, sort, direction) { + const url = new URL(window.location); + + if (date) url.searchParams.set('date', date); + else url.searchParams.delete('date'); + + if (type) url.searchParams.set('type', type); + else url.searchParams.delete('type'); + + if (job) url.searchParams.set('job', job); + else url.searchParams.delete('job'); + + if (result) url.searchParams.set('result', result); + else url.searchParams.delete('result'); + + if (sort) { + url.searchParams.set('sort', sort); + url.searchParams.set('direction', direction); + } else { + url.searchParams.delete('sort'); + url.searchParams.delete('direction'); + } + + // Reload page with new URL + window.location.href = url.toString(); + } + + // Initialize sorting from URL + function initializeSorting() { + const params = getUrlParams(); + + if (params.sort) { + currentSort = { column: params.sort, direction: params.direction }; + updateSortIndicators(params.sort, params.direction); + } + } + + function sortTable(column, direction) { + const rows = Array.from(tbody.querySelectorAll('tr')); + const columnIndex = Array.from(headers).findIndex(th => th.dataset.sort === column); + + rows.sort((a, b) => { + const aVal = a.cells[columnIndex].textContent.trim(); + const bVal = b.cells[columnIndex].textContent.trim(); + + let comparison = 0; + + // Check if this is a datetime column + if (column === 'created' || column === 'started' || column === 'ended') { + const aDate = new Date(aVal); + const bDate = new Date(bVal); + + if (!isNaN(aDate.getTime()) && !isNaN(bDate.getTime())) { + comparison = aDate - bDate; + } else { + comparison = aVal.localeCompare(bVal); + } + } else if (column === 'duration') { + // Parse duration strings like "1h2m3s" or "N/A" + if (aVal === 'N/A' && bVal === 'N/A') comparison = 0; + else if (aVal === 'N/A') comparison = 1; + else if (bVal === 'N/A') comparison = -1; + else comparison = aVal.localeCompare(bVal); + } else { + // Try to parse as numbers first + const aNum = parseFloat(aVal); + const bNum = parseFloat(bVal); + + if (!isNaN(aNum) && !isNaN(bNum)) { + comparison = aNum - bNum; + } else { + comparison = aVal.localeCompare(bVal); + } + } + + return direction === 'asc' ? comparison : -comparison; + }); + + // Clear tbody and re-append sorted rows + tbody.innerHTML = ''; + rows.forEach(row => tbody.appendChild(row)); + } + + function updateSortIndicators(activeColumn, direction) { + headers.forEach(th => { + th.classList.remove('sort-asc', 'sort-desc'); + if (th.dataset.sort === activeColumn) { + th.classList.add(direction === 'asc' ? 'sort-asc' : 'sort-desc'); + } + }); + } + + // Column sorting event listeners + headers.forEach(header => { + header.addEventListener('click', function() { + const column = this.dataset.sort; + let direction = 'asc'; + + if (currentSort.column === column) { + direction = currentSort.direction === 'asc' ? 'desc' : 'asc'; + } + + currentSort = { column, direction }; + const params = getUrlParams(); + updateUrl(params.date, params.type, params.job, params.result, column, direction); + }); + }); + + // Initialize filters + initializeFilters(config); + + // Event delegation for expand/collapse on click + if (tbody) { + tbody.addEventListener('click', function(e) { + const cell = e.target.closest('.expandable'); + if (cell) { + e.stopPropagation(); + cell.classList.toggle('expanded'); + } + }); + } + + // Event delegation for copy on double-click + if (tbody) { + tbody.addEventListener('dblclick', function(e) { + const cell = e.target.closest('.expandable'); + if (cell) { + e.stopPropagation(); + e.preventDefault(); + window.FlowlordUtils.copyToClipboard(cell.textContent.trim()); + } + }); + } + + // Event delegation for context menu + if (tbody) { + tbody.addEventListener('contextmenu', function(e) { + const cell = e.target.closest('.expandable'); + if (cell) { + e.preventDefault(); + e.stopPropagation(); + window.FlowlordUtils.showContextMenu(e, cell.textContent.trim()); + } + }); + } + + // Initialize the page + initializeSorting(); + } + + // Initialize responsive filters + function initializeFilters(config) { + const typeFilter = document.getElementById('typeFilter'); + const jobFilter = document.getElementById('jobFilter'); + const table = document.getElementById('taskTable'); + + if (!table || !config) return; + + const taskTypes = config.taskTypes || []; + const jobMap = new Map(config.jobsByType || []); + const currentType = config.currentType || ""; + const currentJob = config.currentJob || ""; + + // Populate task type dropdown from server data + taskTypes.forEach(type => { + const option = document.createElement('option'); + option.value = type; + option.textContent = type; + typeFilter.appendChild(option); + }); + + // Populate job dropdown based on current type selection + if (currentType && jobMap.has(currentType)) { + const jobs = jobMap.get(currentType); + jobs.forEach(job => { + const option = document.createElement('option'); + option.value = job; + option.textContent = job; + if (job === currentJob) { + option.selected = true; + } + jobFilter.appendChild(option); + }); + } + + // Set current filter values from URL + if (currentType) { + typeFilter.value = currentType; + } + + // Handle task type change - update job dropdown and apply filter + typeFilter.addEventListener('change', function() { + const selectedType = this.value; + const jobOptions = jobFilter.querySelectorAll('option:not([value=""])'); + jobOptions.forEach(option => option.remove()); + jobFilter.value = ''; // Clear job selection + + if (selectedType && jobMap.has(selectedType)) { + const jobs = jobMap.get(selectedType); + jobs.forEach(job => { + const option = document.createElement('option'); + option.value = job; + option.textContent = job; + jobFilter.appendChild(option); + }); + } + + // Apply filter by reloading page + applyFiltersWithResultReset(); + }); + + // Handle job change - reload page with filter + jobFilter.addEventListener('change', function() { + applyFilters(); + }); + } + + // Apply filters by reloading page with query parameters + function applyFilters() { + const typeFilter = document.getElementById('typeFilter'); + const jobFilter = document.getElementById('jobFilter'); + + const url = new URL(window.location); + url.searchParams.delete('page'); // Reset to page 1 when filtering + + const selectedType = typeFilter ? typeFilter.value : ''; + const selectedJob = jobFilter ? jobFilter.value : ''; + + if (selectedType) { + url.searchParams.set('type', selectedType); + } else { + url.searchParams.delete('type'); + } + + if (selectedJob) { + url.searchParams.set('job', selectedJob); + } else { + url.searchParams.delete('job'); + } + + window.location.href = url.toString(); + } + + // Apply filters and reset result filter (for task type changes) + function applyFiltersWithResultReset() { + const typeFilter = document.getElementById('typeFilter'); + const jobFilter = document.getElementById('jobFilter'); + + const url = new URL(window.location); + url.searchParams.delete('page'); // Reset to page 1 when filtering + url.searchParams.delete('result'); // Reset result filter to show all results + + const selectedType = typeFilter ? typeFilter.value : ''; + const selectedJob = jobFilter ? jobFilter.value : ''; + + if (selectedType) { + url.searchParams.set('type', selectedType); + } else { + url.searchParams.delete('type'); + } + + if (selectedJob) { + url.searchParams.set('job', selectedJob); + } else { + url.searchParams.delete('job'); + } + + window.location.href = url.toString(); + } + + // Clear all filters + window.clearFilters = function() { + const url = new URL(window.location); + url.searchParams.delete('id'); + url.searchParams.delete('type'); + url.searchParams.delete('job'); + url.searchParams.delete('result'); + url.searchParams.delete('page'); + window.location.href = url.toString(); + }; + + // Filter by result type using stat-cards + window.filterByResult = function(resultType) { + const url = new URL(window.location); + url.searchParams.delete('page'); // Reset to page 1 + + // Reset task type and job filters when clicking on stat cards + url.searchParams.delete('type'); + url.searchParams.delete('job'); + + if (resultType === 'all') { + url.searchParams.delete('result'); + } else { + url.searchParams.set('result', resultType); + } + + window.location.href = url.toString(); + }; + + // Toggle collapsible section + window.toggleCollapsible = function(sectionId) { + const content = document.getElementById(sectionId + '-content'); + const toggle = document.getElementById(sectionId + '-toggle'); + + if (content.classList.contains('collapsed')) { + content.classList.remove('collapsed'); + toggle.classList.add('expanded'); + } else { + content.classList.add('collapsed'); + toggle.classList.remove('expanded'); + } + }; + + // Export to global scope + window.FlowlordTask = { + init: initTaskPage + }; +})(); + diff --git a/apps/flowlord/handler/static/utils.js b/apps/flowlord/handler/static/utils.js new file mode 100644 index 00000000..f12ddbd2 --- /dev/null +++ b/apps/flowlord/handler/static/utils.js @@ -0,0 +1,130 @@ +// Common utility functions used across Flowlord pages +(function() { + 'use strict'; + + // Context menu functionality + function showContextMenu(event, text) { + // Remove any existing context menu + const existingMenu = document.querySelector('.context-menu'); + if (existingMenu) { + existingMenu.remove(); + } + + // Create context menu + const contextMenu = document.createElement('div'); + contextMenu.className = 'context-menu'; + contextMenu.innerHTML = ` +
+ 📋 Copy +
+ `; + + // Position the context menu + contextMenu.style.left = event.pageX + 'px'; + contextMenu.style.top = event.pageY + 'px'; + + document.body.appendChild(contextMenu); + + // Close context menu when clicking elsewhere + const closeMenu = (e) => { + if (!contextMenu.contains(e.target)) { + contextMenu.remove(); + document.removeEventListener('click', closeMenu); + } + }; + + setTimeout(() => { + document.addEventListener('click', closeMenu); + }, 100); + } + + // Escape HTML for safe insertion + function escapeHtml(text) { + return text.replace(/'/g, "\\'").replace(/"/g, '\\"'); + } + + // Copy to clipboard functionality with enhanced feedback + function copyToClipboard(text) { + const targetElement = event ? event.target : document.activeElement; + + navigator.clipboard.writeText(text).then(function() { + showCopyFeedback(targetElement, 'Copied!'); + }).catch(function(err) { + console.error('Could not copy text: ', err); + // Fallback for older browsers + const textArea = document.createElement('textarea'); + textArea.value = text; + textArea.style.position = 'fixed'; + textArea.style.left = '-999999px'; + textArea.style.top = '-999999px'; + document.body.appendChild(textArea); + textArea.focus(); + textArea.select(); + try { + document.execCommand('copy'); + showCopyFeedback(targetElement, 'Copied!'); + } catch (err) { + console.error('Fallback copy failed: ', err); + showCopyFeedback(targetElement, 'Copy failed!', true); + } + document.body.removeChild(textArea); + }); + } + + // Show copy feedback with animation + function showCopyFeedback(element, message, isError = false) { + // Remove any existing feedback + const existingFeedback = element.querySelector('.copy-feedback'); + if (existingFeedback) { + existingFeedback.remove(); + } + + // Create feedback element + const feedback = document.createElement('div'); + feedback.className = 'copy-feedback'; + feedback.textContent = message; + feedback.style.backgroundColor = isError ? '#dc3545' : '#28a745'; + + // Position feedback relative to the element + const rect = element.getBoundingClientRect(); + feedback.style.position = 'fixed'; + feedback.style.left = (rect.left + rect.width / 2) + 'px'; + feedback.style.top = (rect.top - 10) + 'px'; + feedback.style.transform = 'translateX(-50%)'; + + element.appendChild(feedback); + + // Remove feedback after animation + setTimeout(() => { + if (feedback.parentNode) { + feedback.remove(); + } + }, 2000); + } + + // Toggle field expansion + function toggleField(element, fullText) { + if (element.classList.contains('truncated')) { + element.classList.remove('truncated'); + element.classList.add('expanded'); + element.textContent = fullText; + } else { + element.classList.add('truncated'); + element.classList.remove('expanded'); + // Reset to truncated text if available in data attribute + const truncatedText = element.getAttribute('data-truncated-text'); + if (truncatedText) { + element.textContent = truncatedText; + } + } + } + + // Export to global scope + window.FlowlordUtils = { + showContextMenu: showContextMenu, + copyToClipboard: copyToClipboard, + showCopyFeedback: showCopyFeedback, + toggleField: toggleField + }; +})(); + diff --git a/apps/flowlord/handler/task.tmpl b/apps/flowlord/handler/task.tmpl new file mode 100644 index 00000000..2c203291 --- /dev/null +++ b/apps/flowlord/handler/task.tmpl @@ -0,0 +1,190 @@ + + + + + + Flowlord: Tasks + + + + + {{template "header" .}} +
+ +
+

Task Summary

+
+
+
{{.Counts.Total}}
+
Total Tasks
+
+
+
{{.Counts.Completed}}
+
Completed
+
+
+
{{.Counts.Error}}
+
Errors
+
+
+
{{.Counts.Alert}}
+
Alerts
+
+
+
{{.Counts.Warn}}
+
Warnings
+
+
+
{{.Counts.Running}}
+
Running
+
+
+
+ + {{if gt .TotalPages 1}} + + {{end}} + +
+

Filters

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + {{if .Tasks}} +
+ + + + + + + + + + + + + + + + + {{range .Tasks}} + + + + + + + + + + + + + {{end}} + +
IDTypeJobMessageResultInfoMetaCreatedQueueProcess
{{.Type}}{{.Job}} + {{if .Result}}{{.Result}}{{else}}Running{{end}} + {{if .Created}}{{.Created}}{{else}}N/A{{end}}{{if .QueueTime}}{{.QueueTime}}{{else}}N/A{{end}}{{if .TaskTime}}{{.TaskTime}}{{else}}N/A{{end}}
+
+
+ {{if or .Filter.ID .Filter.Type .Filter.Job .Filter.Result}} + Showing: {{len .Tasks}} | Filtered Tasks: {{.FilteredCount}} | Total Tasks: {{.Counts.Total}} + {{else}} + Showing: {{len .Tasks}} | Total Tasks: {{.Counts.Total}} + {{end}} +
+ + {{if gt .TotalPages 1}} + + {{end}} + {{else}} +
+

No tasks found

+

No tasks were found for {{.Date}} with the current filters.

+
+ {{end}} +
+ + + + + diff --git a/apps/flowlord/handler/task_summary.tmpl b/apps/flowlord/handler/task_summary.tmpl new file mode 100644 index 00000000..13589a27 --- /dev/null +++ b/apps/flowlord/handler/task_summary.tmpl @@ -0,0 +1,51 @@ +
+
+

Task Topic Summary

+ ▼ +
+
+
+ {{range $key, $stats := .Summary}} +
+
+

{{$key}}

+ 1 job +
+
+
+
+ {{$key}} + {{add $stats.CompletedCount $stats.ErrorCount}} tasks +
+
+
+ Complete: + {{$stats.CompletedCount}} +
+
+ Error: + {{$stats.ErrorCount}} +
+
+ Alert: + {{getAlertCount $key $key}} +
+
+ Warning: + {{getWarnCount $key $key}} +
+ {{if gt $stats.CompletedCount 0}} +
+ Exec Times: + min: {{formatDuration $stats.ExecTimes.Min}} max: {{formatDuration $stats.ExecTimes.Max}} avg: {{formatDuration $stats.ExecTimes.Average}} +
+ {{end}} +
+
+
+
+ {{end}} +
+
+
+ diff --git a/apps/flowlord/handler/workflow.tmpl b/apps/flowlord/handler/workflow.tmpl new file mode 100644 index 00000000..013db21c --- /dev/null +++ b/apps/flowlord/handler/workflow.tmpl @@ -0,0 +1,529 @@ + + + + + + Flowlord: Workflows + + + + + {{template "header" .}} +
+ +
+

Workflow File Summary

+
+ {{range $filePath, $count := .WorkflowFileSummary}} +
+
+ {{$filePath}} + {{$count}} phases +
+
Workflow file with {{$count}} phase{{if ne $count 1}}s{{end}}
+
+ {{end}} +
+
+ +
+

Filters

+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+ + {{if .Phases}} +
+ + + + + + + + + + + + + + + {{range .Phases}} + + + + + + + + + + + {{end}} + +
WorkflowTopicJobRuleDepends OnRetryTemplateStatus
{{.FilePath}}{{.Topic}}{{.Job}} + {{if ge (len .Rule) 120}}{{slice .Rule 0 120}}...{{else}}{{.Rule}}{{end}} + {{.DependsOn}}{{.Retry}} + {{if ge (len .Template) 120}}{{slice .Template 0 120}}...{{else}}{{.Template}}{{end}} + + + {{if .Status}}{{.Status}}{{else}}OK{{end}} + +
+
+
+ Total Phases: {{len .Phases}} +
+ {{else}} +
+

No workflow phases found

+

No workflow phases were found in the database.

+
+ {{end}} +
+ + + + diff --git a/apps/flowlord/handler_test.go b/apps/flowlord/handler_test.go index 12c4bb7a..5b46de52 100644 --- a/apps/flowlord/handler_test.go +++ b/apps/flowlord/handler_test.go @@ -1,27 +1,53 @@ package main import ( + "encoding/json" "errors" + "os" + "strings" "testing" "time" "github.com/hydronica/trial" "github.com/pcelvng/task" - "github.com/pcelvng/task-tools/workflow" + "github.com/pcelvng/task-tools/apps/flowlord/sqlite" ) const testPath = "../../internal/test" +func TestMain(t *testing.M) { + isLocal = true + t.Run() + +} + +// loadTaskViewData loads TaskView data from a JSON file +func loadTaskViewData(filename string) ([]sqlite.TaskView, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + + var tasks []sqlite.TaskView + err = json.Unmarshal(data, &tasks) + if err != nil { + return nil, err + } + + return tasks, nil +} + func TestBackloader(t *testing.T) { - cache, err := workflow.New(testPath+"/workflow/f3.toml", nil) + sqlDB := &sqlite.SQLite{LocalPath: ":memory:"} + err := sqlDB.Open(testPath+"/workflow/f3.toml", nil) today := time.Now().Format("2006-01-02") toHour := time.Now().Format(DateHour) if err != nil { t.Fatal(err) } tm := &taskMaster{ - Cache: cache, + taskCache: sqlDB, } fn := func(req request) (response, error) { @@ -330,3 +356,313 @@ func TestMeta_UnmarshalJSON(t *testing.T) { } trial.New(fn, cases).SubTest(t) } + +// TestWebAlertPreview generates an HTML preview of the alert template for visual inspection +// this provides an html file +func TestAlertHTML(t *testing.T) { + + // Create sample alert data to showcase the templating + sampleAlerts := []sqlite.AlertRecord{ + { + TaskID: "task-001", + TaskTime: trial.TimeHour("2024-01-15T11"), + Type: "data-validation", + Job: "quality-check", + Msg: "Validation failed: missing required field 'email'", + CreatedAt: trial.Time(time.RFC3339, "2024-01-15T11:15:00Z"), + }, + { + TaskID: "task-002", + TaskTime: trial.TimeHour("2024-01-15T12"), + Type: "data-validation", + Job: "quality-check", + Msg: "Validation failed: missing required field 'email'", + CreatedAt: trial.Time(time.RFC3339, "2024-01-15T12:15:00Z"), + }, + { + TaskID: "task-003-really-long-id12345", + TaskTime: trial.TimeHour("2024-01-15T11"), + Type: "file-transfer", + Job: "backup", + Msg: "File transfer completed: 1.2GB transferred", + CreatedAt: trial.Time(time.RFC3339, "2024-01-15T12:00:00Z"), + }, + { + TaskID: "task-004", + TaskTime: trial.TimeHour("2024-01-15T13"), + Type: "database-sync", + Job: "replication", + Msg: "This is a really long message that needs to be shorten. The quick brown fox jumped over the lazy dog. Peter Pipper picked a peck of pickled peppers. ", + CreatedAt: trial.Time(time.RFC3339, "2024-01-15T13:30:00Z"), + }, + { + TaskID: "task-005", + TaskTime: trial.TimeHour("2024-01-15T13"), + Type: "notification", + Job: "email-alert", + Msg: "Email notification sent to 150 users", + CreatedAt: trial.Time(time.RFC3339, "2024-01-15T14:00:00Z"), + }, + } + + // Generate HTML using the alertHTML function + // Pass sample dates with data for calendar highlighting + datesWithData := []string{"2024-01-14", "2024-01-15", "2024-01-16"} + htmlContent := alertHTML(sampleAlerts, trial.TimeDay("2024-01-15"), datesWithData) + + // Validate HTML using the new function + if err := validateHTML(htmlContent); err != nil { + t.Errorf("HTML validation failed: %v", err) + } + + // Write HTML to a file for easy viewing + outputFile := "handler/alert_preview.html" + err := os.WriteFile(outputFile, htmlContent, 0644) + if err != nil { + t.Fatalf("Failed to write HTML file: %v", err) + } + + t.Logf("Alert preview generated and saved to: ./%s", outputFile) + +} + +// TestFilesHTML generate a html file based on the files.tmpl it is used for vision examination of the files +func TestFilesHTML(t *testing.T) { + // Create sample file messages + files := []sqlite.FileMessage{ + { + ID: 1, + Path: "gs://bucket/data/2024-01-15/file1.json", + Size: 1024, + LastModified: time.Now().Add(-1 * time.Hour), + ReceivedAt: time.Now().Add(-30 * time.Minute), + TaskTime: time.Now().Add(-1 * time.Hour), + TaskIDs: []string{"task-1", "task-2"}, + TaskNames: []string{"data-load:import", "transform:clean"}, + }, + { + ID: 2, + Path: "gs://bucket/data/2024-01-15/file2.csv", + Size: 2048, + LastModified: time.Now().Add(-2 * time.Hour), + ReceivedAt: time.Now().Add(-15 * time.Minute), + TaskTime: time.Now().Add(-2 * time.Hour), + TaskIDs: []string{}, + TaskNames: []string{}, + }, + } + + date := time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC) + // Pass sample dates with data for calendar highlighting + datesWithData := []string{ "2024-01-15"} + html := filesHTML(files, date, datesWithData) + + // Validate HTML using the new function + if err := validateHTML(html); err != nil { + t.Errorf("HTML validation failed: %v", err) + } + + // Write HTML to a file for easy viewing + outputFile := "handler/files_preview.html" + err := os.WriteFile(outputFile, html, 0644) + if err != nil { + t.Fatalf("Failed to write HTML file: %v", err) + } + + t.Logf("Files preview generated and saved to: ./%s", outputFile) + +} + +// validateHTML performs general HTML validation and returns an error if invalid +func validateHTML(html []byte) error { + htmlStr := string(html) + + // Check for empty HTML + if len(html) == 0 { + return errors.New("HTML output is empty") + } + + // Check for valid HTML structure + if !strings.Contains(htmlStr, "") { + return errors.New("HTML missing DOCTYPE declaration") + } + if !strings.Contains(htmlStr, "") { + return errors.New("HTML missing closing html tag") + } + + // Check for template execution errors (any Go template error messages) + if strings.Contains(htmlStr, "template:") && strings.Contains(htmlStr, "error") { + return errors.New("template execution error detected in HTML output") + } + + // Check for function not defined errors + if strings.Contains(htmlStr, "not defined") { + return errors.New("template function not defined error detected in HTML output") + } + + // Check for any other common template errors + if strings.Contains(htmlStr, "executing") && strings.Contains(htmlStr, "error") { + return errors.New("template execution error detected") + } + + return nil +} + +func TestTaskHTML(t *testing.T) { + // Load TaskView data from JSON file + testTasks, err := loadTaskViewData("test/tasks.json") + if err != nil { + t.Fatalf("Failed to load task data: %v", err) + } + + // Set test date + date := trial.TimeDay("2024-01-15") + + // Create empty task stats and filter for the test + taskStats := generateSummary(testTasks) + filter := &sqlite.TaskFilter{Page: 1, Limit: 100} + + + // Test with no filters - summary will be generated from tasks data + // Pass sample dates with data for calendar highlighting + datesWithData := []string{"2024-01-15"} + html := taskHTML(testTasks, taskStats, len(testTasks), date, filter, datesWithData, 0) + + // Validate HTML using the new function + if err := validateHTML(html); err != nil { + t.Errorf("HTML validation failed: %v", err) + } + + // Write HTML to a file for easy viewing + outputFile := "handler/task_preview.html" + err = os.WriteFile(outputFile, html, 0644) + if err != nil { + t.Fatalf("Failed to write HTML file: %v", err) + } + + t.Logf("Task preview generated and saved to: ./%s", outputFile) +} + +func generateSummary(tasks []sqlite.TaskView) sqlite.TaskStats { + data := make(map[string]*sqlite.Stats) + + for _, tv := range tasks { + // Convert TaskView to task.Task + t := task.Task{ + ID: tv.ID, + Type: tv.Type, + Job: tv.Job, + Info: tv.Info, + Result: task.Result(tv.Result), + Meta: tv.Meta, + Msg: tv.Msg, + Created: tv.Created, + Started: tv.Started, + Ended: tv.Ended, + } + + // Create key from type:job (same logic as Recap) + key := strings.TrimRight(t.Type+":"+t.Job, ":") + + // Get or create stats for this key + stat, found := data[key] + if !found { + stat = &sqlite.Stats{ + CompletedTimes: make([]time.Time, 0), + ErrorTimes: make([]time.Time, 0), + AlertTimes: make([]time.Time, 0), + WarnTimes: make([]time.Time, 0), + RunningTimes: make([]time.Time, 0), + ExecTimes: &sqlite.DurationStats{}, + } + data[key] = stat + } + + // Add task to stats + stat.Add(t) + } + + return sqlite.TaskStats(data) +} + +func TestWorkflowHTML(t *testing.T) { + // Load workflow files + taskCache := &sqlite.SQLite{LocalPath: ":memory:"} + if err := taskCache.Open(testPath+"/workflow/", nil); err != nil { + t.Fatalf("Failed to create test cache: %v", err) + } + + // Test with no filters - summary will be generated from tasks data + html := workflowHTML(taskCache) + + // Validate HTML using the new function + if err := validateHTML(html); err != nil { + t.Errorf("HTML validation failed: %v", err) + } + + // Write HTML to a file for easy viewing + outputFile := "handler/workflow_preview.html" + err := os.WriteFile(outputFile, html, 0644) + if err != nil { + t.Fatalf("Failed to write HTML file: %v", err) + } + + t.Logf("Workflow preview generated and saved to: ./%s", outputFile) + +} + +func TestAboutHTML(t *testing.T) { + // Create a real SQLite cache for testing + taskCache := &sqlite.SQLite{LocalPath: ":memory:"} + if err := taskCache.Open(testPath+"/workflow/", nil); err != nil { + t.Fatalf("Failed to create test cache: %v", err) + } + + // Create a mock Notification for slack + notification := &Notification{ + MinFrequency: 5 * time.Minute, + MaxFrequency: 30 * time.Minute, + } + notification.currentDuration.Store(int64(10 * time.Minute)) + + // Create a mock taskMaster with test data + tm := &taskMaster{ + initTime: time.Now().Add(-2 * time.Hour), // 2 hours ago + nextUpdate: time.Now().Add(30 * time.Minute), // 30 minutes from now + lastUpdate: time.Now().Add(-15 * time.Minute), // 15 minutes ago + taskCache: taskCache, + slack: notification, + } + + // Generate HTML using the aboutHTML method + html := tm.aboutHTML() + + // Validate HTML using the new function + if err := validateHTML(html); err != nil { + t.Errorf("HTML validation failed: %v", err) + } + + // Write HTML to a file for easy viewing + outputFile := "handler/about_preview.html" + err := os.WriteFile(outputFile, html, 0644) + if err != nil { + t.Fatalf("Failed to write HTML file: %v", err) + } + + t.Logf("About preview generated and saved to: ./%s", outputFile) + + // Check that key content is present + htmlStr := string(html) + if !strings.Contains(htmlStr, "flowlord") { + t.Error("Expected 'flowlord' in HTML output") + } + if !strings.Contains(htmlStr, "System Information") { + t.Error("Expected 'System Information' in HTML output") + } + if !strings.Contains(htmlStr, "Table Breakdown") { + t.Error("Expected 'Table Breakdown' in HTML output") + } +} diff --git a/apps/flowlord/job.go b/apps/flowlord/job.go index 7d4a9419..2ea47135 100644 --- a/apps/flowlord/job.go +++ b/apps/flowlord/job.go @@ -2,9 +2,9 @@ package main import ( "errors" + "fmt" "log" "net/url" - "strings" "time" "github.com/jbsmith7741/uri" @@ -74,10 +74,8 @@ func (tm *taskMaster) NewJob(ph workflow.Phase, path string) (cron.Job, error) { return nil, err } - if fields := strings.Fields(bJob.Schedule); len(fields) == 5 { - bJob.Schedule = "0 " + bJob.Schedule - } else if len(fields) > 6 || len(fields) < 5 { - return nil, errors.New("invalid schedule must be of pattern [second] minute day_of_month month day_of_week") + if _, err := cronParser.Parse(bJob.Schedule); err != nil { + return nil, fmt.Errorf("cron: %w", err) } // return Cronjob if not batch params diff --git a/apps/flowlord/main.go b/apps/flowlord/main.go index 1d913d21..3aecd203 100644 --- a/apps/flowlord/main.go +++ b/apps/flowlord/main.go @@ -13,6 +13,7 @@ import ( "github.com/pcelvng/task/bus" tools "github.com/pcelvng/task-tools" + "github.com/pcelvng/task-tools/apps/flowlord/sqlite" "github.com/pcelvng/task-tools/file" ) @@ -34,25 +35,32 @@ Field | Field name | Allowed values | Allowed special characters type options struct { Workflow string `toml:"workflow" comment:"path to workflow file or directory"` Refresh time.Duration `toml:"refresh" comment:"the workflow changes refresh duration value default is 15 min"` - TaskTTL time.Duration `toml:"task-ttl" comment:"time that tasks are expected to have completed in. This values tells the cache how long to keep track of items and alerts if items haven't completed when the cache is cleared"` DoneTopic string `toml:"done_topic" comment:"default is done"` FileTopic string `toml:"file_topic" comment:"file topic for file watching"` FailedTopic string `toml:"failed_topic" comment:"all retry failures published to this topic default is retry-failed, disable with '-'"` Port int `toml:"status_port"` + Host string `toml:"host" comment:"host address of server "` Slack *Notification `toml:"slack"` Bus bus.Options `toml:"bus"` File *file.Options `toml:"file"` + + DB *sqlite.SQLite `toml:"sqlite"` } func main() { log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime) + opts := &options{ Refresh: time.Minute * 15, - TaskTTL: 4 * time.Hour, DoneTopic: "done", + Host: "localhost", FailedTopic: "retry-failed", File: file.NewOptions(), Slack: &Notification{}, + DB: &sqlite.SQLite{ + TaskTTL: 4 * time.Hour, + LocalPath: "./tasks.db", + }, } config.New(opts).Version(tools.String()).Description(description).LoadOrDie() diff --git a/apps/flowlord/sqlite/alerts.go b/apps/flowlord/sqlite/alerts.go new file mode 100644 index 00000000..f24f6753 --- /dev/null +++ b/apps/flowlord/sqlite/alerts.go @@ -0,0 +1,184 @@ +package sqlite + +import ( + "sort" + "time" + + "github.com/pcelvng/task" + + "github.com/pcelvng/task-tools/tmpl" +) + +// AlertRecord represents an alert stored in the database +type AlertRecord struct { + ID int64 `json:"id"` + TaskID string `json:"task_id"` + TaskTime time.Time `json:"task_time"` + Type string `json:"type"` + Job string `json:"job"` + Msg string `json:"msg"` + CreatedAt time.Time `json:"created_at"` +} + +// SummaryLine represents a grouped alert summary for dashboard display +type SummaryLine struct { + Key string `json:"key"` // "task.type:job" + Count int `json:"count"` // number of alerts + TimeRange string `json:"time_range"` // formatted time range +} + +// summaryGroup is used internally for building compact summaries +type summaryGroup struct { + Key string + Count int + TaskTimes []time.Time +} + +// AddAlert stores an alert record in the database +func (s *SQLite) AddAlert(t task.Task, message string) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Allow empty task ID for job send failures - store as "unknown" + taskID := t.ID + if taskID == "" { + taskID = "unknown" + } + + // Extract job using helper function + job := extractJobFromTask(t) + + // Get task time using tmpl.TaskTime function + taskTime := tmpl.TaskTime(t) + + _, err := s.db.Exec(` + INSERT INTO alert_records (task_id, task_time, task_type, job, msg) + VALUES (?, ?, ?, ?, ?) + `, taskID, taskTime, t.Type, job, message) + + if err != nil { + return err + } + + // Update date index - use current time for alert created_at + s.updateDateIndex(time.Now().Format(time.RFC3339), "alerts") + + return nil +} + +// GetAlertsByDate retrieves all alerts for a specific date +func (s *SQLite) GetAlertsByDate(date time.Time) ([]AlertRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + dateStr := date.Format("2006-01-02") + + query := `SELECT id, task_id, task_time, task_type, job, msg, created_at + FROM alert_records + WHERE DATE(created_at) = ? + ORDER BY created_at DESC` + + rows, err := s.db.Query(query, dateStr) + if err != nil { + return nil, err + } + defer rows.Close() + + var alerts []AlertRecord + for rows.Next() { + var alert AlertRecord + err := rows.Scan( + &alert.ID, &alert.TaskID, &alert.TaskTime, &alert.Type, + &alert.Job, &alert.Msg, &alert.CreatedAt, + ) + if err != nil { + continue + } + alerts = append(alerts, alert) + } + + return alerts, nil +} + +// GetAlertsAfterTime retrieves all alerts created after a specific time +func (s *SQLite) GetAlertsAfterTime(afterTime time.Time) ([]AlertRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + query := `SELECT id, task_id, task_time, task_type, job, msg, created_at + FROM alert_records + WHERE created_at > ? + ORDER BY created_at ASC` + + rows, err := s.db.Query(query, afterTime.Format("2006-01-02 15:04:05")) + if err != nil { + return nil, err + } + defer rows.Close() + + var alerts []AlertRecord + for rows.Next() { + var alert AlertRecord + err := rows.Scan( + &alert.ID, &alert.TaskID, &alert.TaskTime, &alert.Type, + &alert.Job, &alert.Msg, &alert.CreatedAt, + ) + if err != nil { + continue + } + alerts = append(alerts, alert) + } + + return alerts, nil +} + +// BuildCompactSummary processes alerts in memory to create compact summary +// Groups by TaskType:Job and collects task times for proper date formatting +func BuildCompactSummary(alerts []AlertRecord) []SummaryLine { + groups := make(map[string]*summaryGroup) + + for _, alert := range alerts { + key := alert.Type + if alert.Job != "" { + key += ":" + alert.Job + } + + // Extract TaskTime from alert meta (not TaskCreated) + + if summary, exists := groups[key]; exists { + summary.Count++ + summary.TaskTimes = append(summary.TaskTimes, alert.TaskTime) + } else { + groups[key] = &summaryGroup{ + Key: key, + Count: 1, + TaskTimes: []time.Time{alert.TaskTime}, + } + } + } + + // Convert map to slice and format time ranges using tmpl.PrintDates + var result []SummaryLine + for _, summary := range groups { + // Use tmpl.PrintDates for consistent formatting with existing Slack notifications + timeRange := tmpl.PrintDates(summary.TaskTimes) + + result = append(result, SummaryLine{ + Key: summary.Key, + Count: summary.Count, + TimeRange: timeRange, + }) + } + + // Use proper sorting (can be replaced with slices.Sort in Go 1.21+) + sort.Slice(result, func(i, j int) bool { + if result[i].Count != result[j].Count { + return result[i].Count > result[j].Count // Sort by count descending + } + return result[i].Key < result[j].Key // Then by key ascending + }) + + return result +} + + diff --git a/apps/flowlord/sqlite/dates.go b/apps/flowlord/sqlite/dates.go new file mode 100644 index 00000000..acdf3f2e --- /dev/null +++ b/apps/flowlord/sqlite/dates.go @@ -0,0 +1,226 @@ +package sqlite + +import ( + "fmt" + "log" + "time" +) + +// updateDateIndex updates the date_index table for a given timestamp and data type +// This method should be called within an existing lock (s.mu.Lock) +func (s *SQLite) updateDateIndex(timestamp, dataType string) { + // Parse timestamp to extract date + t, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + // Try other formats if RFC3339 fails + t, err = time.Parse("2006-01-02 15:04:05", timestamp) + if err != nil { + return // Skip if we can't parse the timestamp + } + } + + dateStr := t.Format("2006-01-02") + + // Determine which column to update + var column string + switch dataType { + case "tasks": + column = "has_tasks" + case "alerts": + column = "has_alerts" + case "files": + column = "has_files" + default: + return + } + + // First try to insert the date, if it already exists, update the column + _, err = s.db.Exec("INSERT OR IGNORE INTO date_index (date) VALUES (?)", dateStr) + if err != nil { + log.Printf("WARNING: Failed to insert date into date_index for %s on %s: %v", dataType, dateStr, err) + return + } + + // Now update the specific column + query := fmt.Sprintf("UPDATE date_index SET %s = 1 WHERE DATE = ?", column) + _, err = s.db.Exec(query, dateStr) + if err != nil { + log.Printf("WARNING: Failed to update date_index for %s on %s: %v", dataType, dateStr, err) + } +} + +// GetDatesWithData returns a list of dates (YYYY-MM-DD format) that have any data +// for tasks, alerts, or files within the retention period +func (s *SQLite) GetDatesWithData() ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + query := ` + SELECT DISTINCT date_val FROM ( + SELECT DISTINCT DATE(created) AS date_val FROM task_records + WHERE created >= datetime('now', '-' || ? || ' days') + UNION + SELECT DISTINCT DATE(created_at) AS date_val FROM alert_records + WHERE created_at >= datetime('now', '-' || ? || ' days') + UNION + SELECT DISTINCT DATE(received_at) AS date_val FROM file_messages + WHERE received_at >= datetime('now', '-' || ? || ' days') + ) + ORDER BY date_val DESC + ` + + // Use retention period in days (default 90) + retentionDays := int(s.Retention.Hours() / 24) + if retentionDays == 0 { + retentionDays = 90 + } + + rows, err := s.db.Query(query, retentionDays, retentionDays, retentionDays) + if err != nil { + return nil, err + } + defer rows.Close() + + var dates []string + for rows.Next() { + var date string + if err := rows.Scan(&date); err != nil { + continue + } + dates = append(dates, date) + } + + return dates, nil +} + +// DatesByType returns a list of dates (YYYY-MM-DD format) that have data for the specified type +// dataType can be "tasks", "alerts", or "files" +// This uses the date_index table for instant lookups +func (s *SQLite) DatesByType(dataType string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var column string + switch dataType { + case "tasks": + column = "has_tasks" + case "alerts": + column = "has_alerts" + case "files": + column = "has_files" + default: + return nil, fmt.Errorf("invalid data type: %s (must be 'tasks', 'alerts', or 'files')", dataType) + } + + query := fmt.Sprintf(` + SELECT DATE + FROM date_index + WHERE %s = 1 + ORDER BY DATE DESC + `, column) + + rows, err := s.db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var dates []string + for rows.Next() { + var date string + if err := rows.Scan(&date); err != nil { + continue + } + dates = append(dates, date) + } + + return dates, nil +} + +// GetDatesWithTasks returns a list of dates (YYYY-MM-DD format) that have task records +func (s *SQLite) GetDatesWithTasks() ([]string, error) { + return s.DatesByType("tasks") +} + +// GetDatesWithAlerts returns a list of dates (YYYY-MM-DD format) that have alert records +func (s *SQLite) GetDatesWithAlerts() ([]string, error) { + return s.DatesByType("alerts") +} + +// GetDatesWithFiles returns a list of dates (YYYY-MM-DD format) that have file message records +func (s *SQLite) GetDatesWithFiles() ([]string, error) { + return s.DatesByType("files") +} + +// RebuildDateIndex scans all tables and rebuilds the date_index table +// This should be called once during migration or can be exposed as an admin endpoint +func (s *SQLite) RebuildDateIndex() error { + s.mu.Lock() + defer s.mu.Unlock() + + log.Println("Starting date_index rebuild...") + + // Clear existing index + _, err := s.db.Exec("DELETE FROM date_index") + if err != nil { + return fmt.Errorf("failed to clear date_index: %w", err) + } + + // Populate from task_records + // First insert the dates, then update the has_tasks flag + _, err = s.db.Exec(` + INSERT OR IGNORE INTO date_index (DATE, has_tasks) + SELECT DISTINCT DATE(created), 1 + FROM task_records + `) + if err != nil { + return fmt.Errorf("failed to populate date_index from tasks: %w", err) + } + + // Populate from alert_records + // Insert new dates and update has_alerts for existing dates + _, err = s.db.Exec(` + INSERT OR IGNORE INTO date_index (DATE) + SELECT DISTINCT DATE(created_at) + FROM alert_records + WHERE DATE(created_at) NOT IN (SELECT DATE FROM date_index) + `) + if err != nil { + return fmt.Errorf("failed to insert new dates from alerts: %w", err) + } + + _, err = s.db.Exec(` + UPDATE date_index + SET has_alerts = 1 + WHERE DATE IN (SELECT DISTINCT DATE(created_at) FROM alert_records) + `) + if err != nil { + return fmt.Errorf("failed to update date_index from alerts: %w", err) + } + + // Populate from file_messages + // Insert new dates and update has_files for existing dates + _, err = s.db.Exec(` + INSERT OR IGNORE INTO date_index (DATE) + SELECT DISTINCT DATE(received_at) + FROM file_messages + WHERE DATE(received_at) NOT IN (SELECT DATE FROM date_index) + `) + if err != nil { + return fmt.Errorf("failed to insert new dates from files: %w", err) + } + + _, err = s.db.Exec(` + UPDATE date_index + SET has_files = 1 + WHERE DATE IN (SELECT DISTINCT DATE(received_at) FROM file_messages) + `) + if err != nil { + return fmt.Errorf("failed to update date_index from files: %w", err) + } + + log.Println("Successfully rebuilt date_index") + return nil +} + + diff --git a/apps/flowlord/sqlite/dbstats.go b/apps/flowlord/sqlite/dbstats.go new file mode 100644 index 00000000..73fbfc76 --- /dev/null +++ b/apps/flowlord/sqlite/dbstats.go @@ -0,0 +1,192 @@ +package sqlite + +import ( + "fmt" +) + +// DBSizeInfo contains database size information +type DBSizeInfo struct { + TotalSize string `json:"total_size"` + PageCount int64 `json:"page_count"` + PageSize int64 `json:"page_size"` + DBPath string `json:"db_path"` +} + +// TableStat contains information about a database table +type TableStat struct { + Name string `json:"name"` + RowCount int64 `json:"row_count"` + TableBytes int64 `json:"table_bytes"` + TableHuman string `json:"table_human"` + IndexBytes int64 `json:"index_bytes"` + IndexHuman string `json:"index_human"` + TotalBytes int64 `json:"total_bytes"` + TotalHuman string `json:"total_human"` + Percentage float64 `json:"percentage"` +} + +// GetDBSize returns database size information +func (s *SQLite) GetDBSize() (*DBSizeInfo, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Get page count and page size + var pageCount, pageSize int64 + err := s.db.QueryRow("PRAGMA page_count").Scan(&pageCount) + if err != nil { + return nil, err + } + + err = s.db.QueryRow("PRAGMA page_size").Scan(&pageSize) + if err != nil { + return nil, err + } + + dbPath := s.LocalPath + if s.BackupPath != "" { + dbPath = s.BackupPath + } + + totalSize := pageCount * pageSize + totalSizeStr := formatBytes(totalSize) + + return &DBSizeInfo{ + TotalSize: totalSizeStr, + PageCount: pageCount, + PageSize: pageSize, + DBPath: dbPath, + }, nil +} + +// GetTableStats returns statistics for all tables in the database +func (s *SQLite) GetTableStats() ([]TableStat, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Get total database size first + var totalSize int64 + err := s.db.QueryRow("SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()").Scan(&totalSize) + if err != nil { + return nil, err + } + + // Get list of tables + rows, err := s.db.Query(` + SELECT name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + ORDER BY name + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var tables []string + for rows.Next() { + var tableName string + if err := rows.Scan(&tableName); err != nil { + return nil, err + } + tables = append(tables, tableName) + } + + var stats []TableStat + var totalTableSize int64 + + for _, tableName := range tables { + // Get row count + var rowCount int64 + err := s.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)).Scan(&rowCount) + if err != nil { + continue // Skip tables we can't read + } + + // Calculate more accurate table size + var tableBytes int64 + if rowCount > 0 { + // Try to get actual table size using dbstat if available + var actualSize int64 + err := s.db.QueryRow(fmt.Sprintf(` + SELECT SUM(pgsize) FROM dbstat WHERE name = '%s' AND aggregate = 1 + `, tableName)).Scan(&actualSize) + + if err == nil && actualSize > 0 { + // Use actual size from dbstat + tableBytes = actualSize + } + } + + // Get index sizes for this table + indexBytes := s.getIndexSize(tableName) + totalBytes := tableBytes + indexBytes + totalTableSize += totalBytes + + percentage := float64(0) + if totalSize > 0 { + percentage = float64(totalBytes) / float64(totalSize) * 100 + } + + stats = append(stats, TableStat{ + Name: tableName, + RowCount: rowCount, + TableBytes: tableBytes, + TableHuman: formatBytes(tableBytes), + IndexBytes: indexBytes, + IndexHuman: formatBytes(indexBytes), + TotalBytes: totalBytes, + TotalHuman: formatBytes(totalBytes), + Percentage: percentage, + }) + } + + return stats, nil +} + +// getIndexSize calculates the total size of all indexes for a table +func (s *SQLite) getIndexSize(tableName string) int64 { + // Get all indexes for this table + rows, err := s.db.Query(fmt.Sprintf(` + SELECT name FROM sqlite_master + WHERE type='index' AND tbl_name='%s' AND name NOT LIKE 'sqlite_%%' + `, tableName)) + if err != nil { + return 0 + } + defer rows.Close() + + var totalIndexSize int64 + for rows.Next() { + var indexName string + if err := rows.Scan(&indexName); err != nil { + continue + } + + // Try to get actual index size using dbstat + var indexSize int64 + err := s.db.QueryRow(fmt.Sprintf(` + SELECT SUM(pgsize) FROM dbstat WHERE name = '%s' AND aggregate = 1 + `, indexName)).Scan(&indexSize) + + if err == nil && indexSize > 0 { + totalIndexSize += indexSize + } + } + + return totalIndexSize +} + +// formatBytes converts bytes to human readable format +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} + + diff --git a/apps/flowlord/sqlite/files.go b/apps/flowlord/sqlite/files.go new file mode 100644 index 00000000..d48344ba --- /dev/null +++ b/apps/flowlord/sqlite/files.go @@ -0,0 +1,232 @@ +package sqlite + +import ( + "database/sql" + "encoding/json" + "time" + + "github.com/pcelvng/task-tools/file/stat" + "github.com/pcelvng/task-tools/tmpl" +) + +// FileMessage represents a file message record +type FileMessage struct { + ID int `json:"id"` + Path string `json:"path"` + Size int64 `json:"size"` + LastModified time.Time `json:"last_modified"` + ReceivedAt time.Time `json:"received_at"` + TaskTime time.Time `json:"task_time"` + TaskIDs []string `json:"task_ids"` + TaskNames []string `json:"task_names"` +} + +// FileMessageWithTasks represents a file message with associated task details +type FileMessageWithTasks struct { + FileID int `json:"file_id"` + Path string `json:"path"` + TaskTime time.Time `json:"task_time"` + ReceivedAt time.Time `json:"received_at"` + TaskID string `json:"task_id"` + TaskType string `json:"task_type"` + TaskJob string `json:"task_job"` + TaskResult string `json:"task_result"` + TaskCreated time.Time `json:"task_created"` + TaskStarted time.Time `json:"task_started"` + TaskEnded time.Time `json:"task_ended"` +} + +// AddFileMessage stores a file message in the database +func (s *SQLite) AddFileMessage(sts stat.Stats, taskIDs []string, taskNames []string) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Parse last modified time from the created field + var lastModified time.Time + if sts.Created != "" { + lastModified, _ = time.Parse(time.RFC3339, sts.Created) + } + + // Extract task time from path + taskTime := tmpl.PathTime(sts.Path) + + // Convert slices to JSON arrays + var taskIDsJSON, taskNamesJSON sql.NullString + if len(taskIDs) > 0 { + if jsonBytes, err := json.Marshal(taskIDs); err == nil { + taskIDsJSON = sql.NullString{String: string(jsonBytes), Valid: true} + } + } + if len(taskNames) > 0 { + if jsonBytes, err := json.Marshal(taskNames); err == nil { + taskNamesJSON = sql.NullString{String: string(jsonBytes), Valid: true} + } + } + + _, err := s.db.Exec(` + INSERT INTO file_messages (path, size, last_modified, task_time, task_ids, task_names) + VALUES (?, ?, ?, ?, ?, ?) + `, sts.Path, sts.Size, lastModified, taskTime, taskIDsJSON, taskNamesJSON) + + if err != nil { + return err + } + + // Update date index - use current time for received_at + s.updateDateIndex(time.Now().Format(time.RFC3339), "files") + + return nil +} + +// GetFileMessages retrieves file messages with optional filtering +func (s *SQLite) GetFileMessages(limit int, offset int) ([]FileMessage, error) { + s.mu.Lock() + defer s.mu.Unlock() + + query := ` + SELECT id, path, SIZE, last_modified, received_at, task_time, task_ids, task_names + FROM file_messages + ORDER BY received_at DESC + LIMIT ? OFFSET ? + ` + + rows, err := s.db.Query(query, limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + var messages []FileMessage + for rows.Next() { + var msg FileMessage + var taskIDsJSON, taskNamesJSON sql.NullString + + err := rows.Scan( + &msg.ID, &msg.Path, &msg.Size, &msg.LastModified, &msg.ReceivedAt, + &msg.TaskTime, &taskIDsJSON, &taskNamesJSON, + ) + if err != nil { + continue + } + + // Parse JSON arrays + if taskIDsJSON.Valid { + json.Unmarshal([]byte(taskIDsJSON.String), &msg.TaskIDs) + } + if taskNamesJSON.Valid { + json.Unmarshal([]byte(taskNamesJSON.String), &msg.TaskNames) + } + + messages = append(messages, msg) + } + + return messages, nil +} + +// GetFileMessagesByDate retrieves file messages for a specific date +func (s *SQLite) GetFileMessagesByDate(date time.Time) ([]FileMessage, error) { + s.mu.Lock() + defer s.mu.Unlock() + + dateStr := date.Format("2006-01-02") + query := ` + SELECT id, path, SIZE, last_modified, received_at, task_time, task_ids, task_names + FROM file_messages + WHERE DATE(received_at) = ? + ORDER BY received_at DESC + ` + + rows, err := s.db.Query(query, dateStr) + if err != nil { + return nil, err + } + defer rows.Close() + + var messages []FileMessage + for rows.Next() { + var msg FileMessage + var taskIDsJSON, taskNamesJSON sql.NullString + + err := rows.Scan( + &msg.ID, &msg.Path, &msg.Size, &msg.LastModified, &msg.ReceivedAt, + &msg.TaskTime, &taskIDsJSON, &taskNamesJSON, + ) + if err != nil { + continue + } + + // Parse JSON arrays + if taskIDsJSON.Valid { + json.Unmarshal([]byte(taskIDsJSON.String), &msg.TaskIDs) + } + if taskNamesJSON.Valid { + json.Unmarshal([]byte(taskNamesJSON.String), &msg.TaskNames) + } + + messages = append(messages, msg) + } + + return messages, nil +} + +// GetFileMessagesWithTasks retrieves file messages with their associated task details +func (s *SQLite) GetFileMessagesWithTasks(limit int, offset int) ([]FileMessageWithTasks, error) { + s.mu.Lock() + defer s.mu.Unlock() + + query := ` + SELECT + fm.id, fm.path, fm.task_time, fm.received_at, + json_extract(t.value, '$') AS task_id, + tl.type AS task_type, + tl.job AS task_job, + tl.result AS task_result, + tl.created AS task_created, + tl.started AS task_started, + tl.ended AS task_ended + FROM file_messages fm, + json_each(fm.task_ids) AS t + JOIN task_log tl ON json_extract(t.value, '$') = tl.id + WHERE fm.task_ids IS NOT NULL + ORDER BY fm.received_at DESC, fm.id + LIMIT ? OFFSET ? + ` + + rows, err := s.db.Query(query, limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []FileMessageWithTasks + for rows.Next() { + var result FileMessageWithTasks + var taskCreated, taskStarted, taskEnded sql.NullString + + err := rows.Scan( + &result.FileID, &result.Path, &result.TaskTime, &result.ReceivedAt, + &result.TaskID, &result.TaskType, &result.TaskJob, &result.TaskResult, + &taskCreated, &taskStarted, &taskEnded, + ) + if err != nil { + continue + } + + // Parse timestamps + if taskCreated.Valid { + result.TaskCreated, _ = time.Parse(time.RFC3339, taskCreated.String) + } + if taskStarted.Valid { + result.TaskStarted, _ = time.Parse(time.RFC3339, taskStarted.String) + } + if taskEnded.Valid { + result.TaskEnded, _ = time.Parse(time.RFC3339, taskEnded.String) + } + + results = append(results, result) + } + + return results, nil +} + + diff --git a/apps/flowlord/sqlite/schema.sql b/apps/flowlord/sqlite/schema.sql new file mode 100644 index 00000000..a5f606c1 --- /dev/null +++ b/apps/flowlord/sqlite/schema.sql @@ -0,0 +1,138 @@ +-- Schema version tracking +CREATE TABLE IF NOT EXISTS schema_version ( + id INTEGER PRIMARY KEY DEFAULT 1, + version INTEGER NOT NULL +); + +-- SQL schema for the task cache +-- Single table for all task records with composite primary key +CREATE TABLE IF NOT EXISTS task_records ( + id TEXT, + type TEXT, + job TEXT, + info TEXT, + result TEXT, -- NULL if not completed + meta TEXT, + msg TEXT, -- NULL if not completed + created TIMESTAMP, + started TIMESTAMP, -- NULL if not started + ended TIMESTAMP, -- NULL if not completed + PRIMARY KEY (type, job, id, created) +); + +CREATE INDEX IF NOT EXISTS idx_task_records_type ON task_records (type); +CREATE INDEX IF NOT EXISTS idx_task_records_job ON task_records (job); +CREATE INDEX IF NOT EXISTS idx_task_records_created ON task_records (created); +CREATE INDEX IF NOT EXISTS idx_task_records_type_job ON task_records (type, job); +CREATE INDEX IF NOT EXISTS idx_task_records_date_range ON task_records (created, ended); + +-- Create a view that calculates task and queue times +DROP VIEW IF EXISTS tasks; +CREATE VIEW IF NOT EXISTS tasks AS +SELECT + task_records.id, + task_records.type, + task_records.job, + task_records.info, + -- SQLite doesn't have parse_url function, we'll need to handle this in Go + task_records.meta, + -- SQLite doesn't have parse_param function, we'll need to handle this in Go + task_records.msg, + task_records.result, + -- Calculate task duration in seconds + CASE + WHEN task_records.ended IS NULL OR task_records.started IS NULL OR task_records.ended = '' OR task_records.started = '' THEN 0 + ELSE ROUND((julianday(task_records.ended) - julianday(task_records.started)) * 24 * 60 * 60) + END as task_seconds, + -- Format task duration as HH:MM:SS + CASE + WHEN task_records.ended IS NULL OR task_records.started IS NULL OR task_records.ended = '' OR task_records.started = '' THEN 'N/A' + ELSE + printf('%02d:%02d:%02d', + ROUND((julianday(task_records.ended) - julianday(task_records.started)) * 24 * 60 * 60) / 3600, + ROUND((julianday(task_records.ended) - julianday(task_records.started)) * 24 * 60 * 60) % 3600 / 60, + ROUND((julianday(task_records.ended) - julianday(task_records.started)) * 24 * 60 * 60) % 60 + ) + END as task_time, + -- Calculate queue time in seconds + CASE + WHEN task_records.started IS NULL OR task_records.created IS NULL OR task_records.started = '' OR task_records.created = '' THEN 0 + ELSE ROUND((julianday(task_records.started) - julianday(task_records.created)) * 24 * 60 * 60) + END as queue_seconds, + -- Format queue duration as HH:MM:SS + CASE + WHEN task_records.started IS NULL OR task_records.created IS NULL OR task_records.started = '' OR task_records.created = '' THEN 'N/A' + ELSE + printf('%02d:%02d:%02d', + ROUND((julianday(task_records.started) - julianday(task_records.created)) * 24 * 60 * 60) / 3600, + ROUND((julianday(task_records.started) - julianday(task_records.created)) * 24 * 60 * 60) % 3600 / 60, + ROUND((julianday(task_records.started) - julianday(task_records.created)) * 24 * 60 * 60) % 60 + ) + END as queue_time, + task_records.created, + task_records.started, + task_records.ended +FROM task_records; + +-- Alert records table for storing alert events +CREATE TABLE IF NOT EXISTS alert_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT, -- task ID (can be empty for job send failures) + task_time TIMESTAMP, -- task time (can be empty) + task_type TEXT NOT NULL, -- task type for quick filtering + job TEXT, -- task job for quick filtering + msg TEXT NOT NULL, -- alert message (contains alert context) + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_alert_records_created_at ON alert_records (created_at); + +-- File message history table for tracking file processing +CREATE TABLE IF NOT EXISTS file_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, -- File path (e.g., "gs://bucket/path/file.json") + size INTEGER, -- File size in bytes + last_modified TIMESTAMP, -- When file was modified (from file system/GCS metadata) + received_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- When the record was received (time.Now()) + task_time TIMESTAMP, -- Time extracted from path using tmpl.PathTime(sts.Path) + task_ids TEXT, -- JSON array of task IDs (null if no matches) + task_names TEXT -- JSON array of task names (type:job format, null if no matches) +); + +-- Indexes for efficient querying +CREATE INDEX IF NOT EXISTS idx_file_messages_path ON file_messages (path); +CREATE INDEX IF NOT EXISTS idx_file_messages_received ON file_messages (received_at); + +-- Workflow file tracking (replaces in-memory workflow file cache) +CREATE TABLE IF NOT EXISTS workflow_files ( + file_path TEXT PRIMARY KEY, + file_hash TEXT NOT NULL, + loaded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_modified TIMESTAMP +); + +-- Workflow phases (matches Phase struct exactly) +CREATE TABLE IF NOT EXISTS workflow_phases ( + file_path TEXT NOT NULL, + task TEXT NOT NULL, -- topic:job format (e.g., "data-load:hourly") + depends_on TEXT, + rule TEXT, -- URI query parameters (e.g., "cron=0 0 * * *&offset=1h") + template TEXT, + retry integer DEFAULT 0 -- number of retries (default 0) +); + +-- Indexes for performance +CREATE INDEX IF NOT EXISTS idx_workflow_phases_task ON workflow_phases (task); +CREATE INDEX IF NOT EXISTS idx_workflow_phases_depends_on ON workflow_phases (depends_on); + +-- Date index table for fast date lookups +-- Tracks which dates have data in each table +CREATE TABLE IF NOT EXISTS date_index ( + date TEXT PRIMARY KEY, + -- YYYY-MM-DD format + has_tasks BOOLEAN DEFAULT 0, + -- 1 if task_records has data for this date + has_alerts BOOLEAN DEFAULT 0, + -- 1 if alert_records has data for this date + has_files BOOLEAN DEFAULT 0 -- 1 if file_messages has data for this date +); \ No newline at end of file diff --git a/apps/flowlord/sqlite/sqlite.go b/apps/flowlord/sqlite/sqlite.go new file mode 100644 index 00000000..73450e11 --- /dev/null +++ b/apps/flowlord/sqlite/sqlite.go @@ -0,0 +1,270 @@ +package sqlite + +import ( + "database/sql" + _ "embed" + "fmt" + "io" + "log" + "sync" + "time" + + _ "modernc.org/sqlite" + + "github.com/pcelvng/task-tools/file" +) + +//go:embed schema.sql +var schema string + +const ( + // DefaultPageSize is the default number of items per page for paginated queries + DefaultPageSize = 100 +) + +// currentSchemaVersion is the current version of the database schema. +// Increment this when making schema changes that require migration. +// Version 1: Initial schema +// Version 2: Added date_index table for performance optimization +const currentSchemaVersion = 2 + +type SQLite struct { + LocalPath string + BackupPath string + + TaskTTL time.Duration `toml:"task-ttl" comment:"time that tasks are expected to have completed in. This values tells the cache how long to keep track of items and alerts if items haven't completed when the cache is cleared"` + Retention time.Duration // 90 days + + db *sql.DB + fOpts *file.Options + mu sync.Mutex + + // Workflow-specific fields + workflowPath string + isDir bool +} + +// Open the sqlite DB. If localPath doesn't exist then check if BackupPath exists and copy it to localPath +// Also initializes workflow path and determines if it's a directory +func (o *SQLite) Open(workflowPath string, fOpts *file.Options) error { + o.fOpts = fOpts + o.workflowPath = workflowPath + if o.TaskTTL < time.Hour { + o.TaskTTL = time.Hour + } + if o.db == nil { + if err := o.initDB(); err != nil { + return err + } + } + + // Determine if workflow path is a directory + sts, err := file.Stat(workflowPath, fOpts) + if err != nil { + return fmt.Errorf("problem with workflow path %s %w", workflowPath, err) + } + o.isDir = sts.IsDir + _, err = o.Refresh() + + return err +} + +func (o *SQLite) initDB() error { + backupSts, _ := file.Stat(o.BackupPath, o.fOpts) + localSts, _ := file.Stat(o.LocalPath, o.fOpts) + + if localSts.Size == 0 && backupSts.Size > 0 { + log.Printf("Restoring local DB from backup %s", o.BackupPath) + // no local file but backup exists so copy it down + if err := copyFiles(o.BackupPath, o.LocalPath, o.fOpts); err != nil { + log.Println(err) // TODO: should this be fatal? + } + } + + // Open the database + db, err := sql.Open("sqlite", o.LocalPath) + if err != nil { + return err + } + + // Enable WAL mode for safer concurrent reads and writes + // WAL mode allows multiple readers and one writer simultaneously + _, err = db.Exec("PRAGMA journal_mode = WAL;") + if err != nil { + return fmt.Errorf("failed to set WAL mode: %w", err) + } + + // Set busy timeout to handle concurrent access gracefully + // This prevents immediate lock failures by retrying for up to 30 seconds + _, err = db.Exec("PRAGMA busy_timeout = 30000;") + if err != nil { + return fmt.Errorf("failed to set busy timeout: %w", err) + } + + // Set a smaller page size to reduce DB file size + _, err = db.Exec("PRAGMA page_size = 4096;") + if err != nil { + return fmt.Errorf("failed to set page size: %w", err) + } + + // Enable auto vacuum to reclaim space when records are deleted + _, err = db.Exec("PRAGMA auto_vacuum = INCREMENTAL;") + if err != nil { + return fmt.Errorf("failed to set auto vacuum: %w", err) + } + + o.db = db + + // Check and migrate schema if needed + // This will handle initial schema creation for new databases (version 0) + // and apply incremental migrations for existing databases + if err := o.migrateIfNeeded(); err != nil { + return fmt.Errorf("schema migration failed: %w", err) + } + + return nil +} + +func copyFiles(src, dst string, fOpts *file.Options) error { + r, err := file.NewReader(src, fOpts) + if err != nil { + return fmt.Errorf("init reader err: %w", err) + } + w, err := file.NewWriter(dst, fOpts) + if err != nil { + return fmt.Errorf("init writer err: %w", err) + } + _, err = io.Copy(w, r) + if err != nil { + return fmt.Errorf("copy err: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("close writer err: %w", err) + } + return r.Close() +} + +// migrateIfNeeded checks the current schema version and applies migrations if needed +func (o *SQLite) migrateIfNeeded() error { + currentVersion := o.GetSchemaVersion() + + if currentVersion < currentSchemaVersion { + log.Printf("Migrating database schema from version %d to %d", currentVersion, currentSchemaVersion) + if err := o.migrateSchema(currentVersion); err != nil { + return fmt.Errorf("migration failed: %w", err) + } + if err := o.setVersion(currentSchemaVersion); err != nil { + return fmt.Errorf("failed to update schema version: %w", err) + } + log.Printf("Schema migration completed successfully") + } + + return nil +} + +// GetSchemaVersion returns the current schema version from the database. +// Returns 0 if the schema_version table doesn't exist or is empty (new database). +func (o *SQLite) GetSchemaVersion() int { + var version int + err := o.db.QueryRow("SELECT version FROM schema_version LIMIT 1").Scan(&version) + if err != nil { + // Table doesn't exist or other error - treat as version 0 (new database) + // schema.sql will create the table when applied + return 0 + } + + return version +} + +// setVersion updates the schema version in the database +func (o *SQLite) setVersion(version int) error { + // Delete all existing records to ensure we only have one row + _, err := o.db.Exec("DELETE FROM schema_version") + if err != nil { + return fmt.Errorf("failed to clear schema_version: %w", err) + } + + // Insert the new version + _, err = o.db.Exec("INSERT INTO schema_version (version) VALUES (?)", version) + if err != nil { + return fmt.Errorf("failed to insert schema version: %w", err) + } + + return nil +} + +// migrateSchema applies version-specific migrations based on the current version +func (o *SQLite) migrateSchema(currentVersion int) error { + // Version 0 → 1: Initial schema creation for new databases + if currentVersion < 1 { + log.Println("Creating initial schema (version 1)") + + // Drop workflow tables if they exist (they may have been created before versioning) + _, err := o.db.Exec(` + DROP TABLE IF EXISTS workflow_files; + DROP TABLE IF EXISTS workflow_phases; + `) + if err != nil { + return fmt.Errorf("failed to drop old workflow tables: %w", err) + } + + _, err = o.db.Exec(schema) + if err != nil { + return fmt.Errorf("failed to create initial schema: %w", err) + } + } + + // Version 1 → 2: Add date_index table for performance optimization + if currentVersion < 2 { + log.Println("Migrating schema from version 1 to 2 (adding date_index table)") + + // Re-apply schema.sql - it has IF NOT EXISTS so it's safe and will add new tables + _, err := o.db.Exec(schema) + if err != nil { + return fmt.Errorf("failed to apply schema for version 2: %w", err) + } + + // Populate the date_index from existing data + if err := o.RebuildDateIndex(); err != nil { + return fmt.Errorf("failed to populate date_index: %w", err) + } + + log.Println("Successfully migrated to schema version 2") + } + + // Add future migrations here as needed: + // Example: + // if currentVersion < 3 { + // db := o.db + // // Drop column by recreating table (since data loss is OK) + // db.Exec("DROP TABLE IF EXISTS task_records") + // // schema.sql will recreate it with correct structure + // } + + return nil +} + +// Close the DB connection and copy the current file to the backup location +func (o *SQLite) Close() error { + var errs []error + if err := o.db.Close(); err != nil { + errs = append(errs, err) + } + if o.BackupPath != "" { + log.Printf("Backing up DB to %s", o.BackupPath) + if err := o.Sync(); err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return fmt.Errorf("close errors: %v", errs) + } + return nil +} + +// Sync the local DB to the backup location +func (o *SQLite) Sync() error { + return copyFiles(o.LocalPath, o.BackupPath, o.fOpts) +} + + diff --git a/apps/flowlord/sqlite/sqlite_test.go b/apps/flowlord/sqlite/sqlite_test.go new file mode 100644 index 00000000..aeddce65 --- /dev/null +++ b/apps/flowlord/sqlite/sqlite_test.go @@ -0,0 +1,112 @@ +package sqlite + +import ( + "testing" + + "github.com/pcelvng/task" + "github.com/pcelvng/task-tools/file/stat" +) + +// TestDatesByType tests the unified date query method +func TestDatesByType(t *testing.T) { + // Create in-memory database + db := &SQLite{LocalPath: ":memory:"} + if err := db.initDB(); err != nil { + t.Fatalf("Failed to initialize database: %v", err) + } + defer db.Close() + + // Add sample task records + db.Add(task.Task{ + ID: "task1", + Type: "data-load", + Job: "import", + Created: "2024-01-15T10:00:00Z", + Result: task.CompleteResult, + Ended: "2024-01-15T10:05:00Z", + }) + db.Add(task.Task{ + ID: "task2", + Type: "data-load", + Job: "import", + Created: "2024-01-16T10:00:00Z", + Result: task.CompleteResult, + Ended: "2024-01-16T10:05:00Z", + }) + + // Add sample alert records with specific created_at times + _, err := db.db.Exec(` + INSERT INTO alert_records (task_id, task_time, task_type, job, msg, created_at) + VALUES (?, ?, ?, ?, ?, ?), + (?, ?, ?, ?, ?, ?) + `, "alert1", "2024-01-15T11:00:00Z", "data-validation", "check", "Validation error", "2024-01-15T11:00:00Z", + "alert2", "2024-01-17T11:00:00Z", "data-validation", "check", "Validation error", "2024-01-17T11:00:00Z") + if err != nil { + t.Fatalf("Failed to insert alerts: %v", err) + } + + // Add sample file messages + fileMsg1 := stat.Stats{ + Path: "gs://bucket/file1.json", + Size: 1024, + } + db.AddFileMessage(fileMsg1, []string{}, []string{}) + + // Rebuild the date index to capture the directly-inserted alerts + // (In production, all inserts go through Add/AddAlert/AddFileMessage which maintain the index) + if err := db.RebuildDateIndex(); err != nil { + t.Fatalf("Failed to rebuild date index: %v", err) + } + + // Test "tasks" type + taskDates, err := db.DatesByType("tasks") + if err != nil { + t.Errorf("DatesByType('tasks') error: %v", err) + } + if len(taskDates) != 2 { + t.Errorf("Expected 2 task dates, got %d", len(taskDates)) + } + if len(taskDates) > 0 && taskDates[0] != "2024-01-16" { + t.Errorf("Expected first task date '2024-01-16', got '%s'", taskDates[0]) + } + + // Test "alerts" type + alertDates, err := db.DatesByType("alerts") + if err != nil { + t.Errorf("DatesByType('alerts') error: %v", err) + } + if len(alertDates) != 2 { + t.Errorf("Expected 2 alert dates, got %v", alertDates) + } + + // Test "files" type + fileDates, err := db.DatesByType("files") + if err != nil { + t.Errorf("DatesByType('files') error: %v", err) + } + if len(fileDates) == 0 { + t.Error("Expected at least 1 file date") + } + + // Test invalid type + _, err = db.DatesByType("invalid") + if err == nil { + t.Error("Expected error for invalid data type, got nil") + } + + // Test backward compatibility methods + taskDates2, _ := db.GetDatesWithTasks() + if len(taskDates2) != len(taskDates) { + t.Error("GetDatesWithTasks() should return same results as DatesByType('tasks')") + } + + alertDates2, _ := db.GetDatesWithAlerts() + if len(alertDates2) != len(alertDates) { + t.Error("GetDatesWithAlerts() should return same results as DatesByType('alerts')") + } + + fileDates2, _ := db.GetDatesWithFiles() + if len(fileDates2) != len(fileDates) { + t.Error("GetDatesWithFiles() should return same results as DatesByType('files')") + } +} \ No newline at end of file diff --git a/apps/flowlord/sqlite/stats.go b/apps/flowlord/sqlite/stats.go new file mode 100644 index 00000000..8c69656b --- /dev/null +++ b/apps/flowlord/sqlite/stats.go @@ -0,0 +1,230 @@ +package sqlite + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + gtools "github.com/jbsmith7741/go-tools" + "github.com/pcelvng/task" + + "github.com/pcelvng/task-tools/tmpl" +) + +const ( + precision = 10 * time.Millisecond +) + +type Stats struct { + CompletedCount int + CompletedTimes []time.Time + + ErrorCount int + ErrorTimes []time.Time + + AlertCount int + AlertTimes []time.Time + + WarnCount int + WarnTimes []time.Time + + RunningCount int + RunningTimes []time.Time + + ExecTimes *DurationStats +} + +func (s *Stats) MarshalJSON() ([]byte, error) { + type count struct { + Count int + Times string + } + + v := struct { + Min string `json:"min"` + Max string `json:"max"` + Average string `json:"avg"` + Complete count `json:"complete"` + Error count `json:"error"` + }{ + Min: gtools.PrintDuration(s.ExecTimes.Min), + Max: gtools.PrintDuration(s.ExecTimes.Max), + Average: gtools.PrintDuration(s.ExecTimes.Average()), + Complete: count{ + Count: s.CompletedCount, + Times: tmpl.PrintDates(s.CompletedTimes), + }, + Error: count{ + Count: s.ErrorCount, + Times: tmpl.PrintDates(s.ErrorTimes), + }, + } + return json.Marshal(v) +} + +func (s Stats) String() string { + r := s.ExecTimes.String() + if s.CompletedCount > 0 { + r += fmt.Sprintf("\n\tComplete: %d %v", s.CompletedCount, tmpl.PrintDates(s.CompletedTimes)) + } + if s.ErrorCount > 0 { + r += fmt.Sprintf("\n\tError: %d %v", s.ErrorCount, tmpl.PrintDates(s.ErrorTimes)) + } + + return r + "\n" +} + +type DurationStats struct { + Min time.Duration + Max time.Duration + sum int64 + count int64 +} + +func (s *DurationStats) Add(d time.Duration) { + if s.count == 0 { + s.Min = d + s.Max = d + } + + if d > s.Max { + s.Max = d + } else if d < s.Min { + s.Min = d + } + // truncate times to milliseconds to preserve space + s.sum += int64(d / precision) + s.count++ +} + +func (s *DurationStats) Average() time.Duration { + if s.count == 0 { + return 0 + } + return time.Duration(s.sum/s.count) * precision +} + +func (s *DurationStats) String() string { + return fmt.Sprintf("min: %v max: %v avg: %v", + s.Min, s.Max, s.Average()) +} + +func (stats *Stats) Add(tsk task.Task) { + tm := tmpl.TaskTime(tsk) + + // Handle different result types + switch tsk.Result { + case task.ErrResult: + stats.ErrorCount++ + stats.ErrorTimes = append(stats.ErrorTimes, tm) + return + case "alert": + stats.AlertCount++ + stats.AlertTimes = append(stats.AlertTimes, tm) + return + case "warn": + stats.WarnCount++ + stats.WarnTimes = append(stats.WarnTimes, tm) + return + case "": + // Empty result means task is running + stats.RunningCount++ + stats.RunningTimes = append(stats.RunningTimes, tm) + return + default: + // Assume "complete" or any other result is a completion + stats.CompletedCount++ + stats.CompletedTimes = append(stats.CompletedTimes, tm) + } + + // Track execution time for completed tasks + if tsk.Ended != "" && tsk.Started != "" { + end, _ := time.Parse(time.RFC3339, tsk.Ended) + start, _ := time.Parse(time.RFC3339, tsk.Started) + stats.ExecTimes.Add(end.Sub(start)) + } +} + +type pathTime time.Time + +func (p *pathTime) UnmarshalText(b []byte) error { + t := tmpl.PathTime(string(b)) + *p = pathTime(t) + return nil +} + +// TaskCounts represents aggregate counts of tasks by result status +type TaskCounts struct { + Total int + Completed int + Error int + Alert int + Warn int + Running int +} + +// TaskStats is a map of task keys (type:job) to their statistics +type TaskStats map[string]*Stats + +// UniqueTypes returns a sorted list of unique task types +func (ts TaskStats) UniqueTypes() []string { + typeSet := make(map[string]struct{}) + for key := range ts { + // Split the key to get type (everything before the first colon) + if idx := strings.Index(key, ":"); idx > 0 { + typeSet[key[:idx]] = struct{}{} + } else { + // No colon means the entire key is the type + typeSet[key] = struct{}{} + } + } + + types := make([]string, 0, len(typeSet)) + for t := range typeSet { + types = append(types, t) + } + sort.Strings(types) + return types +} + +// JobsByType returns jobs organized by type +func (ts TaskStats) JobsByType() map[string][]string { + jobsByType := make(map[string][]string) + + for key := range ts { + // Split key into type and job + parts := strings.SplitN(key, ":", 2) + if len(parts) == 2 { + typ := parts[0] + job := parts[1] + if job != "" { + jobsByType[typ] = append(jobsByType[typ], job) + } + } + } + + // Sort jobs for each type + for typ := range jobsByType { + sort.Strings(jobsByType[typ]) + } + + return jobsByType +} + +// TotalCounts returns aggregate result counts across all tasks +func (ts TaskStats) TotalCounts() TaskCounts { + var counts TaskCounts + + for _, stats := range ts { + counts.Total += stats.CompletedCount + stats.ErrorCount + stats.AlertCount + stats.WarnCount + stats.RunningCount + counts.Completed += stats.CompletedCount + counts.Error += stats.ErrorCount + counts.Alert += stats.AlertCount + counts.Warn += stats.WarnCount + counts.Running += stats.RunningCount + } + + return counts +} diff --git a/apps/flowlord/sqlite/tasks.go b/apps/flowlord/sqlite/tasks.go new file mode 100644 index 00000000..2fe43f2a --- /dev/null +++ b/apps/flowlord/sqlite/tasks.go @@ -0,0 +1,467 @@ +package sqlite + +import ( + "fmt" + "log" + "net/url" + "strings" + "time" + + "github.com/pcelvng/task" + "github.com/pcelvng/task/bus" + + "github.com/pcelvng/task-tools/tmpl" +) + +// TaskJob describes info about completed tasks that are within the cache +type TaskJob struct { + LastUpdate time.Time // time since the last event with id + Completed bool + count int + Events []task.Task +} + +// TaskFilter contains options for filtering and paginating task queries. +// Empty string fields are ignored in the query. +type TaskFilter struct { + ID string // Filter by task ID (resets other filters) + Type string // Filter by task type + Job string // Filter by job name + Result string // Filter by result status (complete, error, alert, warn, or "running" for empty) + Page int // Page number (1-based, default: 1) + Limit int // Number of results per page (default: 100) +} + +// TaskView represents a task with calculated times from the tasks view +type TaskView struct { + ID string `json:"id"` + Type string `json:"type"` + Job string `json:"job"` + Info string `json:"info"` + Result string `json:"result"` + Meta string `json:"meta"` + Msg string `json:"msg"` + TaskSeconds int `json:"task_seconds"` + TaskTime string `json:"task_time"` + QueueSeconds int `json:"queue_seconds"` + QueueTime string `json:"queue_time"` + Created string `json:"created"` + Started string `json:"started"` + Ended string `json:"ended"` +} + +func (s *SQLite) Add(t task.Task) { + if t.ID == "" { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + // Use UPSERT to handle both new tasks and updates + result, err := s.db.Exec(` + INSERT INTO task_records (id, type, job, info, result, meta, msg, created, started, ended) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (TYPE, job, id, created) + DO UPDATE SET + result = excluded.result, + meta = excluded.meta, + msg = excluded.msg, + started = excluded.started, + ended = excluded.ended + `, + t.ID, t.Type, t.Job, t.Info, t.Result, t.Meta, t.Msg, + t.Created, t.Started, t.Ended, + ) + + if err != nil { + log.Printf("ERROR: Failed to insert task record: %v", err) + return + } + + // Update date index for this task's date + if t.Created != "" { + s.updateDateIndex(t.Created, "tasks") + } + + // Check if this was an update (conflict) rather than insert + rowsAffected, _ := result.RowsAffected() + if rowsAffected == 0 { + // This indicates a conflict occurred and the record was updated + // Log this as it's unexpected for new task creation + log.Printf("WARNING: Task creation conflict detected - task %s:%s:%s at %s was updated instead of inserted", + t.Type, t.Job, t.ID, t.Created) + } +} + +func (s *SQLite) GetTask(id string) TaskJob { + s.mu.Lock() + defer s.mu.Unlock() + + var tj TaskJob + + // Get all task records for this ID, ordered by created time + rows, err := s.db.Query(` + SELECT id, type, job, info, result, meta, msg, + created, started, ended + FROM task_records + WHERE id = ? + ORDER BY created + `, id) + if err != nil { + return tj + } + defer rows.Close() + + var events []task.Task + var lastUpdate time.Time + var completed bool + + for rows.Next() { + var t task.Task + err := rows.Scan( + &t.ID, &t.Type, &t.Job, &t.Info, &t.Result, &t.Meta, &t.Msg, + &t.Created, &t.Started, &t.Ended, + ) + if err != nil { + continue + } + events = append(events, t) + + // Track completion status and last update time + if t.Result != "" { + completed = true + if ended, err := time.Parse(time.RFC3339, t.Ended); err == nil { + if ended.After(lastUpdate) { + lastUpdate = ended + } + } + } else { + if created, err := time.Parse(time.RFC3339, t.Created); err == nil { + if created.After(lastUpdate) { + lastUpdate = created + } + } + } + } + + tj = TaskJob{ + LastUpdate: lastUpdate, + Completed: completed, + Events: events, + count: len(events), + } + + return tj +} + +// Recycle cleans up any records older than day in the DB tables: files, alerts and tasks. +func (s *SQLite) Recycle(t time.Time) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + day := t.Format("2006-01-02") + totalDeleted := 0 + + // Delete old task records + result, err := s.db.Exec("DELETE FROM task_records WHERE created < ?", day) + if err != nil { + return totalDeleted, fmt.Errorf("error deleting old task records: %w", err) + } + rowsAffected, _ := result.RowsAffected() + totalDeleted += int(rowsAffected) + + // Delete old alert records + result, err = s.db.Exec("DELETE FROM alert_records WHERE created_at < ?", day) + if err != nil { + return totalDeleted, fmt.Errorf("error deleting old alert records: %w", err) + } + rowsAffected, _ = result.RowsAffected() + totalDeleted += int(rowsAffected) + + // Delete old file messages + result, err = s.db.Exec("DELETE FROM file_messages WHERE received_at < ?", day) + if err != nil { + return totalDeleted, fmt.Errorf("error deleting old file messages: %w", err) + } + rowsAffected, _ = result.RowsAffected() + totalDeleted += int(rowsAffected) + + return totalDeleted, nil +} + +// CheckIncompleteTasks checks for tasks that have not completed within the TTL period +// and adds them to the alerts table with deduplication. Returns count of incomplete tasks found. +// Uses a JOIN query to efficiently find incomplete tasks without existing alerts. +func (s *SQLite) CheckIncompleteTasks() int { + s.mu.Lock() + defer s.mu.Unlock() + + tasks := make([]task.Task, 0) + t := time.Now() + + // Use LEFT JOIN to find incomplete tasks that don't have existing alerts + // This eliminates the need for separate deduplication queries + rows, err := s.db.Query(` + SELECT tr.id, tr.type, tr.job, tr.info, tr.result, tr.meta, tr.msg, + tr.created, tr.started, tr.ended + FROM task_records tr + LEFT JOIN alert_records ar ON ( + tr.id = ar.task_id AND + tr.type = ar.task_type AND + tr.job = ar.job AND + ar.msg LIKE 'INCOMPLETE:%' AND + ar.created_at > datetime('now', '-1 day') + ) + WHERE tr.created < ? + AND tr.result = '' + AND ar.id IS NULL + `, t.Add(-s.TaskTTL)) + if err != nil { + return 0 + } + defer rows.Close() + + // Process incomplete records that don't have alerts + for rows.Next() { + var task task.Task + err := rows.Scan( + &task.ID, &task.Type, &task.Job, &task.Info, &task.Result, + &task.Meta, &task.Msg, &task.Created, &task.Started, &task.Ended, + ) + if err != nil { + continue + } + + // Add incomplete task to alert list + tasks = append(tasks, task) + + // Add alert directly (no need to check for duplicates since JOIN already filtered them) + taskID := task.ID + if taskID == "" { + taskID = "unknown" + } + + taskTime := tmpl.TaskTime(task) + + _, err = s.db.Exec(` + INSERT INTO alert_records (task_id, task_time, task_type, job, msg) + VALUES (?, ?, ?, ?, ?) + `, taskID, taskTime, task.Type, task.Job, "INCOMPLETE: unfinished task detected") + + if err != nil { + // Continue processing even if alert insertion fails + continue + } + } + + return len(tasks) +} + +// Recap returns a summary of task statistics for a given day +func (s *SQLite) Recap(day time.Time) TaskStats { + s.mu.Lock() + defer s.mu.Unlock() + + data := make(map[string]*Stats) + rows, err := s.db.Query(` + SELECT id, type, job, info, result, meta, msg, + created, started, ended + FROM task_records + WHERE created >= ? AND created < ? + `, day.Format("2006-01-02"), day.Add(24*time.Hour).Format("2006-01-02")) + if err != nil { + return data + } + defer rows.Close() + + for rows.Next() { + var t task.Task + err := rows.Scan( + &t.ID, &t.Type, &t.Job, &t.Info, &t.Result, &t.Meta, &t.Msg, + &t.Created, &t.Started, &t.Ended, + ) + if err != nil { + continue + } + key := strings.TrimRight(t.Type+":"+t.Job, ":") + stat, found := data[key] + if !found { + stat = &Stats{ + CompletedTimes: make([]time.Time, 0), + ErrorTimes: make([]time.Time, 0), + AlertTimes: make([]time.Time, 0), + WarnTimes: make([]time.Time, 0), + RunningTimes: make([]time.Time, 0), + ExecTimes: &DurationStats{}, + } + data[key] = stat + } + stat.Add(t) + } + + return TaskStats(data) +} + +func (s *SQLite) SendFunc(p bus.Producer) func(string, *task.Task) error { + return func(topic string, tsk *task.Task) error { + s.Add(*tsk) + return p.Send(topic, tsk.JSONBytes()) + } +} + +// GetTasksByDate retrieves tasks for a specific date with optional filtering and pagination +func (s *SQLite) GetTasksByDate(date time.Time, filter *TaskFilter) ([]TaskView, int, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Handle nil filter or set defaults + if filter == nil { + filter = &TaskFilter{} + } + if filter.Limit <= 0 { + filter.Limit = DefaultPageSize + } + if filter.Page <= 0 { + filter.Page = 1 // default to first page + } + + dateStr := date.Format("2006-01-02") + + // Build WHERE clause with filters + whereClause := "WHERE DATE(created) = ?" + args := []interface{}{dateStr} + + // If ID is specified, only filter by ID (ignores other filters) + if filter.ID != "" { + whereClause += " AND id = ?" + args = append(args, filter.ID) + } else { + // Apply other filters only when ID is not specified + if filter.Type != "" { + whereClause += " AND type = ?" + args = append(args, filter.Type) + } + + if filter.Job != "" { + whereClause += " AND job = ?" + args = append(args, filter.Job) + } + + if filter.Result != "" { + // Handle "running" as empty result + if filter.Result == "running" { + whereClause += " AND result = ''" + } else { + whereClause += " AND result = ?" + args = append(args, filter.Result) + } + } + } + + // Get total count of filtered results + countQuery := "SELECT COUNT(*) FROM tasks " + whereClause + var totalCount int + err := s.db.QueryRow(countQuery, args...).Scan(&totalCount) + if err != nil { + return nil, 0, err + } + + // Build main query with pagination + query := `SELECT id, type, job, info, result, meta, msg, task_seconds, task_time, queue_seconds, queue_time, created, started, ended + FROM tasks ` + whereClause + ` + ORDER BY created DESC + LIMIT ? OFFSET ?` + + // Calculate offset from page number + offset := (filter.Page - 1) * filter.Limit + args = append(args, filter.Limit, offset) + + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, 0, err + } + defer rows.Close() + + var tasks []TaskView + for rows.Next() { + var t TaskView + err := rows.Scan( + &t.ID, &t.Type, &t.Job, &t.Info, &t.Result, &t.Meta, &t.Msg, + &t.TaskSeconds, &t.TaskTime, &t.QueueSeconds, &t.QueueTime, + &t.Created, &t.Started, &t.Ended, + ) + if err != nil { + t.Result = string(task.ErrResult) + t.Msg = err.Error() + } + tasks = append(tasks, t) + } + + return tasks, totalCount, nil +} + +// GetTaskSummaryByDate creates a summary of tasks for a specific date +func (s *SQLite) GetTaskSummaryByDate(date time.Time) (TaskStats, error) { + s.mu.Lock() + defer s.mu.Unlock() + + dateStr := date.Format("2006-01-02") + + query := `SELECT id, type, job, info, result, meta, msg, created, started, ended + FROM task_records + WHERE DATE(created) = ? + ORDER BY created` + + rows, err := s.db.Query(query, dateStr) + if err != nil { + return nil, err + } + defer rows.Close() + + data := make(map[string]*Stats) + for rows.Next() { + var t task.Task + err := rows.Scan( + &t.ID, &t.Type, &t.Job, &t.Info, &t.Result, &t.Meta, &t.Msg, + &t.Created, &t.Started, &t.Ended, + ) + if err != nil { + continue + } + + job := t.Job + if job == "" { + v, _ := url.ParseQuery(t.Meta) + job = v.Get("job") + } + key := strings.TrimRight(t.Type+":"+job, ":") + stat, found := data[key] + if !found { + stat = &Stats{ + CompletedTimes: make([]time.Time, 0), + ErrorTimes: make([]time.Time, 0), + AlertTimes: make([]time.Time, 0), + WarnTimes: make([]time.Time, 0), + RunningTimes: make([]time.Time, 0), + ExecTimes: &DurationStats{}, + } + data[key] = stat + } + stat.Add(t) + } + + return TaskStats(data), nil +} + +// extractJobFromTask is a helper function to get job from task +func extractJobFromTask(t task.Task) string { + job := t.Job + if job == "" { + if meta, err := url.ParseQuery(t.Meta); err == nil { + job = meta.Get("job") + } + } + return job +} + + diff --git a/apps/flowlord/sqlite/workflow.go b/apps/flowlord/sqlite/workflow.go new file mode 100644 index 00000000..14230283 --- /dev/null +++ b/apps/flowlord/sqlite/workflow.go @@ -0,0 +1,492 @@ +package sqlite + +import ( + "fmt" + "io" + "net/url" + "path/filepath" + "strings" + + "github.com/hydronica/toml" + "github.com/jbsmith7741/go-tools/appenderr" + "github.com/pcelvng/task" + "github.com/robfig/cron/v3" + + "github.com/pcelvng/task-tools/file" + "github.com/pcelvng/task-tools/workflow" +) + +// Phase represents a workflow phase (same as workflow.Phase) +type Phase struct { + Task string // Should use Topic() and Job() for access + Rule string + DependsOn string // Task that the previous workflow depends on + Retry int + Template string // template used to create the task +} + +type PhaseDB struct { + Phase + FilePath string // workflow file path + Status string // status of the phase (e.g. valid, invalid, warning) +} + +func (p PhaseDB) Topic() string { + return p.Phase.Topic() +} +func (p PhaseDB) Job() string { + return p.Phase.Job() +} + +func (ph Phase) IsEmpty() bool { + return ph.Task == "" && ph.Rule == "" && ph.DependsOn == "" && ph.Template == "" +} + +// Job portion of the Task +func (ph Phase) Job() string { + s := strings.Split(ph.Task, ":") + if len(s) > 1 { + return s[1] + } + + r, _ := url.ParseQuery(ph.Rule) + if j := r.Get("job"); j != "" { + return j + } + return "" +} + +// Topic portion of the Task +func (ph Phase) Topic() string { + s := strings.Split(ph.Task, ":") + return s[0] +} + +// Deprecated: +// ToWorkflowPhase converts cache.Phase to workflow.Phase +func (ph Phase) ToWorkflowPhase() workflow.Phase { + return workflow.Phase{ + Task: ph.Task, + Rule: ph.Rule, + DependsOn: ph.DependsOn, + Retry: ph.Retry, + Template: ph.Template, + } +} + +// Workflow represents a workflow file with phases +type Workflow struct { + Checksum string // md5 hash for the file to check for changes + Phases []Phase `toml:"phase"` +} + +// Workflow Cache Methods - implementing workflow.Cache interface + +// IsDir returns true if the original workflow path is a folder rather than a file +func (s *SQLite) IsDir() bool { + return s.isDir +} + +// Search the all workflows within the cache and return the first +// matching phase with the specific task and job (optional) +func (s *SQLite) Search(taskType, job string) PhaseDB { + return s.Get(task.Task{Type: taskType, Job: job}) +} + +// Get the Phase associated with the task +// looks for matching phases within a workflow defined in meta +// that matches the task Type and job. +func (s *SQLite) Get(t task.Task) PhaseDB { + s.mu.Lock() + defer s.mu.Unlock() + + values, _ := url.ParseQuery(t.Meta) + //key := values.Get("workflow") + job := t.Job + if job == "" { + job = values.Get("job") + } + key := t.Type + if job != "" { + key += ":" + job + } + + query := ` + SELECT file_path, task, depends_on, rule, template, retry + FROM workflow_phases + WHERE task = ? + ORDER BY file_path, task + LIMIT 1 + ` + rows, err := s.db.Query(query, key) + if err != nil { + return PhaseDB{Status: err.Error()} + } + defer rows.Close() + + if rows.Next() { + ph := PhaseDB{} + + err := rows.Scan(&ph.FilePath, &ph.Task, &ph.DependsOn, &ph.Rule, &ph.Template, &ph.Retry) + if err != nil { + return PhaseDB{Status: err.Error()} + } + // Compute validation status on read instead of storing it + ph.Status = ph.Phase.Validate() + return ph + } + return PhaseDB{Status: "not found"} +} + +// Children of the given task t, a child phase is one that dependsOn another task +// Empty slice will be returned if no children are found. +// A task without a type or metadata containing the workflow info +// will result in an error +func (s *SQLite) Children(t task.Task) []Phase { + s.mu.Lock() + defer s.mu.Unlock() + + if t.Type == "" { + return nil + } + + values, _ := url.ParseQuery(t.Meta) + key := values.Get("workflow") + job := t.Job + if job == "" { + job = values.Get("job") + } + + if key == "" { + return nil + } + + // Find phases that depend on this task + query := ` + SELECT task, depends_on, rule, template, retry + FROM workflow_phases + WHERE file_path = ? AND (depends_on LIKE ? OR depends_on = ?) + ORDER BY task + ` + + rows, err := s.db.Query(query, key, t.Type+":%", t.Type) + if err != nil { + return nil + } + defer rows.Close() + + var result []Phase + for rows.Next() { + var taskStr, dependsOn, rule, template string + var retry int + + err := rows.Scan(&taskStr, &dependsOn, &rule, &template, &retry) + if err != nil { + continue + } + + // Parse depends_on to check if it matches the task + v := strings.Split(dependsOn, ":") + depends := v[0] + var j string + if len(v) > 1 { + j = v[1] + } + + if depends == t.Type { + if j == "" || j == job { + result = append(result, Phase{ + Task: taskStr, + Rule: rule, + DependsOn: dependsOn, + Retry: retry, + Template: template, + }) + } + } + } + + return result +} + +// Refresh checks the cache and reloads any files if the checksum has changed. +func (s *SQLite) Refresh() (changedFiles []string, err error) { + if !s.isDir { + f, err := s.loadFile(s.workflowPath, s.fOpts) + if len(f) > 0 { + changedFiles = append(changedFiles, f) + } + return changedFiles, err + } + + // List and read all files + allFiles, err := listAllFiles(s.workflowPath, s.fOpts) + if err != nil { + return changedFiles, err + } + + errs := appenderr.New() + for _, filePath := range allFiles { + f, err := s.loadFile(filePath, s.fOpts) + if err != nil { + errs.Add(err) + } + if len(f) > 0 { + changedFiles = append(changedFiles, f) + } + } + + // Remove deleted workflows from database + for _, key := range s.GetWorkflowFiles() { + found := false + for _, v := range allFiles { + f := s.filePath(v) + if f == key { + found = true + break + } + } + if !found { + errs.Add(s.removeWorkflow(key)) + changedFiles = append(changedFiles, "-"+key) + } + } + + return changedFiles, errs.ErrOrNil() +} + +// Helper methods for workflow operations + +// listAllFiles recursively lists all files in a folder and sub-folders +func listAllFiles(p string, opts *file.Options) ([]string, error) { + files := make([]string, 0) + sts, err := file.List(p, opts) + if err != nil { + return nil, err + } + for _, f := range sts { + if f.IsDir { + s, err := listAllFiles(f.Path, opts) + if err != nil { + return nil, err + } + files = append(files, s...) + continue + } + files = append(files, f.Path) + } + return files, nil +} + +// loadFile checks a files checksum and updates database if required +// loaded file name is returned +func (s *SQLite) loadFile(path string, opts *file.Options) (f string, err error) { + f = s.filePath(path) + sts, err := file.Stat(path, opts) + // permission issues + if err != nil { + return "", fmt.Errorf("stats %s %w", path, err) + } + // We can't process a directory here + if sts.IsDir { + return "", fmt.Errorf("can not read directory %s", path) + } + + // Check if file has changed by comparing checksum + existingHash := s.getFileHash(f) + if existingHash == sts.Checksum { + return "", nil // No changes + } + + // Read and parse the workflow file + r, err := file.NewReader(path, opts) + if err != nil { + return "", fmt.Errorf("new reader %s %w", path, err) + } + b, err := io.ReadAll(r) + if err != nil { + return "", fmt.Errorf("read-all: %s %w", path, err) + } + + var workflow Workflow + if _, err := toml.Decode(string(b), &workflow); err != nil { + return "", fmt.Errorf("decode: %s %w", string(b), err) + } + + // Update database with new workflow data + err = s.updateWorkflowInDB(f, sts.Checksum, workflow.Phases) + if err != nil { + return "", fmt.Errorf("update workflow in db: %w", err) + } + + return f, nil +} + +// filePath returns a filePath consist of all unique part +// after the path set in the cache +func (s *SQLite) filePath(p string) (filePath string) { + path := strings.TrimLeft(s.workflowPath, ".") + if i := strings.LastIndex(p, path); i != -1 { + filePath = strings.TrimLeft(p[i+len(path):], "/") + } + if filePath == "" { + _, filePath = filepath.Split(p) + } + return filePath +} + +// getFileHash retrieves the current hash for a workflow file +func (s *SQLite) getFileHash(filePath string) string { + path := s.filePath(filePath) + var hash string + err := s.db.QueryRow("SELECT file_hash FROM workflow_files WHERE file_path = ?", path).Scan(&hash) + if err != nil { + return "" + } + return hash +} + +// GetWorkflowFiles returns a map of all workflow files in the database +func (s *SQLite) GetWorkflowFiles() []string { + files := make([]string, 0) + rows, err := s.db.Query("SELECT file_path FROM workflow_files") + if err != nil { + return files + } + defer rows.Close() + + for rows.Next() { + var filePath string + if err := rows.Scan(&filePath); err == nil { + files = append(files, filePath) + } + } + return files +} + +// GetPhasesForWorkflow returns all phases for a specific workflow file +func (s *SQLite) GetPhasesForWorkflow(filePath string) ([]PhaseDB, error) { + rows, err := s.db.Query(` + SELECT file_path, task, depends_on, rule, template, retry + FROM workflow_phases + WHERE file_path = ? + ORDER BY task + `, filePath) + if err != nil { + return nil, err + } + defer rows.Close() + + var phases []PhaseDB + for rows.Next() { + ph := PhaseDB{} + + err := rows.Scan(&ph.FilePath, &ph.Task, &ph.DependsOn, &ph.Rule, &ph.Template, &ph.Retry) + if err != nil { + continue + } + + // Compute validation status on read instead of storing it + ph.Status = ph.Phase.Validate() + + phases = append(phases, ph) + } + + return phases, nil +} + +// updateWorkflowInDB updates the workflow data in the database +func (s *SQLite) updateWorkflowInDB(filePath, checksum string, phases []Phase) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Update or insert workflow file record + _, err := s.db.Exec(` + INSERT INTO workflow_files (file_path, file_hash, loaded_at, last_modified) + VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (file_path) DO UPDATE SET + file_hash = excluded.file_hash, + loaded_at = CURRENT_TIMESTAMP, + last_modified = CURRENT_TIMESTAMP + `, filePath, checksum) + if err != nil { + return err + } + + // Remove existing phases for this workflow + _, err = s.db.Exec("DELETE FROM workflow_phases WHERE file_path = ?", filePath) + if err != nil { + return err + } + + // Insert new phases + for _, phase := range phases { + task := phase.Task + if !strings.Contains(task, ":") && phase.Job() != "" { + task = task + ":" + phase.Job() + } + phase.Task = task + + _, err = s.db.Exec(` + INSERT INTO workflow_phases (file_path, task, depends_on, rule, template, retry) + VALUES (?, ?, ?, ?, ?, ?) + `, filePath, phase.Task, phase.DependsOn, phase.Rule, phase.Template, phase.Retry) + if err != nil { + return err + } + } + + return nil +} + +// removeWorkflow removes a workflow and its phases from the database +func (s *SQLite) removeWorkflow(filePath string) error { + // Start transaction + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + // Remove phases first + _, err = tx.Exec("DELETE FROM workflow_phases WHERE file_path = ?", filePath) + if err != nil { + return err + } + + // Remove workflow file record + _, err = tx.Exec("DELETE FROM workflow_files WHERE file_path = ?", filePath) + if err != nil { + return err + } + + return tx.Commit() +} + +// Validate a phase and returns status message +func (ph Phase) Validate() string { + + values, err := url.ParseQuery(ph.Rule) + if err != nil { + return fmt.Sprintf("invalid rule format: %s", ph.Rule) + } + + // Basic validation logic + if ph.DependsOn == "" && values.Get("cron") == "" && values.Get("files") == "" { + return "non-scheduled phase: use depends_on, cron or files" + } + + // Check for valid cron rule + + if c := values.Get("cron"); c != "" { + + if _, err := cron.NewParser( + cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, + ).Parse(c); err != nil { + return fmt.Sprintf("invalid cron: %s %v", c, err) + } + + } + + return "" // No issues +} diff --git a/apps/flowlord/sqlite/workflow_test.go b/apps/flowlord/sqlite/workflow_test.go new file mode 100644 index 00000000..0ff290b7 --- /dev/null +++ b/apps/flowlord/sqlite/workflow_test.go @@ -0,0 +1,333 @@ +package sqlite + +import ( + "errors" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/hydronica/toml" + "github.com/hydronica/trial" + "github.com/pcelvng/task" +) + +func MockSQLite() *SQLite { + s := &SQLite{LocalPath: ":memory:"} + if err := s.initDB(); err != nil { + panic(err) + } + return s +} + +const testPath = "../../../internal/test/" + +func TestLoadFile(t *testing.T) { + fn := func(in string) (string, error) { + cache := MockSQLite() + _, err := cache.loadFile(in, nil) + return cache.getFileHash(in), err + } + cases := trial.Cases[string, string]{ + "read file": { + Input: testPath + "workflow/f1.toml", + Expected: "4422274d9c9f7e987c609687a7702651", // checksum of test file + }, + "stat error": { + Input: "nop://stat_err", + ExpectedErr: errors.New("nop stat error"), + }, + "dir error": { + Input: "nop://stat_dir", + ExpectedErr: errors.New("can not read directory"), + }, + "read err": { + Input: "nop://init_err", + ExpectedErr: errors.New("new reader"), + }, + "decode error": { + Input: testPath + "invalid.toml", + ExpectedErr: errors.New("decode"), + }, + } + trial.New(fn, cases).SubTest(t) +} + +// TestLoadPhase is used to validated that a phase is correctly loaded into the DB +func TestValidatePhase(t *testing.T) { + fn := func(ph Phase) (string, error) { + s := ph.Validate() + if s != "" { + return "", errors.New(s) + } + return "", nil + } + cases := trial.Cases[Phase, string]{ + "Ok": { + Input: Phase{Rule: "files=s3:/bucket/path/*/*.txt"}, + }, + "empty phase": { + Input: Phase{}, + ExpectedErr: errors.New("non-scheduled phase"), + }, + "invalid cron": { + Input: Phase{Rule: "cron=abcdefg"}, + ExpectedErr: errors.New("invalid cron"), + }, + "cron 5": { + Input: Phase{Rule: "cron=1 2 3 4 5"}, + }, + "cron 6": { + Input: Phase{Rule: "cron=1 2 3 4 5 6"}, + }, + "cron complex": + { + Input: Phase{Rule: "cron=20 */6 * * SUN"}, + }, + "parse_err": { + Input: Phase{Rule: "a;lskdfj?%$`?\"^"}, + ExpectedErr: errors.New("invalid rule format"), + }, + } + trial.New(fn, cases).SubTest(t) +} + +func TestToml(t *testing.T) { + v := ` +[[phase]] +task = "task1" +rule = "cron=0 * * * *&offset=-4h&job=t2&retry_delay=10ms" +retry = 3 +template = "?date={yyyy}-{mm}-{dd}T{hh}" + +[[phase]] +task = "task2" +dependsOn = "task1" +rule = "" +retry = 3 +template = "{meta:file}?time={yyyy}-{mm}-{dd}" +` + w := &Workflow{} + + if _, err := toml.Decode(v, w); err != nil { + t.Fatalf(err.Error()) + } + if len(w.Phases) != 2 { + t.Errorf("Expected 2 phases got %d", len(w.Phases)) + t.Log(spew.Sdump(w.Phases)) + } + +} + +func TestRefresh(t *testing.T) { + type Cache struct { + path string + isDir bool + Workflows map[string]Workflow + } + + fn := func(c *Cache) (int, error) { + sqlite := MockSQLite() + for path, workflow := range c.Workflows { + if err := sqlite.updateWorkflowInDB(path, workflow.Checksum, workflow.Phases); err != nil { + return 0, err + } + } + sqlite.workflowPath = c.path + sqlite.isDir = c.isDir + _, err := sqlite.Refresh() + return len(sqlite.GetWorkflowFiles()), err + } + cases := trial.Cases[*Cache, int]{ + "single file": { + Input: &Cache{path: testPath + "workflow/f1.toml"}, + Expected: 1, // load 1 file + }, + "folder": { + Input: &Cache{path: testPath + "workflow", isDir: true}, + Expected: 4, // load folder with 2 files + }, + "sub-folder": { + Input: &Cache{path: testPath + "parent", isDir: true}, + Expected: 2, // load folder with 1 files and sub-folder with 1 file + }, + "error case": { + Input: &Cache{path: "nop://err", isDir: true}, + ShouldErr: true, + }, + "file removed": { + Input: &Cache{ + path: testPath + "workflow", + isDir: true, + Workflows: map[string]Workflow{ + "missing.toml": {}, + "f1.toml": {}, + "f2.toml": {}, + "f3.toml": {}, + }, + }, + Expected: 4, + }, + "keep loaded": { + Input: &Cache{ + path: testPath + "workflow", + isDir: true, + Workflows: map[string]Workflow{ + "f1.toml": { + Checksum: "34cf5142fbd029fa778ee657592d03ce", + }, + "f2.toml": { + Checksum: "eac7716a13d9dea0d630c5d8b1e6c6b1", + }, + }, + }, + Expected: 4, + }, + } + trial.New(fn, cases).SubTest(t) +} + +func TestGet(t *testing.T) { + cache := MockSQLite() + err := cache.updateWorkflowInDB("workflow.toml", "NA", []Phase{ + {Task: "task1"}, + {Task: "dup"}, + {Task: "task2", DependsOn: "task1"}, + {Task: "task3", DependsOn: "task2"}, + {Task: "task4", DependsOn: "task2"}, + }) + if err != nil { + t.Fatal(err) + } + err = cache.updateWorkflowInDB("w2job.toml", "NA", []Phase{ + {Task: "dup"}, + {Task: "t2", Rule: "job=j1"}, + {Task: "t2", Rule: "job=j2"}, + {Task: "t2:j3", Rule: ""}, + }) + if err != nil { + t.Fatal(err) + } + fn := func(t task.Task) (Phase, error) { + return cache.Get(t).Phase, nil + } + cases := trial.Cases[task.Task, Phase]{ + "no meta": { + Input: task.Task{Type: "task1"}, + Expected: Phase{Task: "task1"}, + }, + "blank task": { + Input: task.Task{Meta: "workflow=workflow.toml"}, + Expected: Phase{}, + }, + "not found": { + Input: task.Task{Type: "missing", Meta: "workflow=workflow.toml"}, + Expected: Phase{}, + }, + "task_job": { + Input: task.Task{Type: "t2", Job: "j2", Meta: "workflow=*"}, + Expected: Phase{Rule: "job=j2", Task: "t2:j2"}, + }, + "task2": { + Input: task.Task{Type: "task2", Meta: "workflow=workflow.toml"}, + Expected: Phase{Task: "task2", DependsOn: "task1"}, + }, + "task=t2 with job=j1": { + Input: task.Task{Type: "t2", Meta: "workflow=w2job.toml&job=j1"}, + Expected: Phase{Task: "t2:j1", Rule: "job=j1"}, + }, + "job does not exist": { + Input: task.Task{Type: "t2", Meta: "workflow=w2job.toml&job=invalid"}, + Expected: Phase{}, + }, + "wildcard search": { + Input: task.Task{Type: "t2", Meta: "workflow=*&job=j3"}, + Expected: Phase{Task: "t2:j3"}, + }, + "wildcard with same task in different files": { // picks first match, results will vary + Input: task.Task{Type: "dup", Meta: "workflow=*"}, + Expected: Phase{Task: "dup"}, + }, + } + trial.New(fn, cases).SubTest(t) +} + +func TestChildren(t *testing.T) { + cache := MockSQLite() + err := cache.updateWorkflowInDB("workflow.toml", "NA", []Phase{ + {Task: "task1"}, + {Task: "task2", DependsOn: "task1"}, + {Task: "task3", DependsOn: "task2"}, + {Task: "task4", DependsOn: "task2"}, + {Task: "task5", DependsOn: "task1:j4"}, + }) + if err != nil { + t.Fatal(err) + } + fn := func(t task.Task) ([]Phase, error) { + return cache.Children(t), nil + } + cases := trial.Cases[task.Task, []Phase]{ + "no meta": { + Input: task.Task{Type: "task1"}, + Expected: []Phase(nil), + }, + "blank task": { + Input: task.Task{Meta: "workflow=workflow.toml"}, + Expected: []Phase(nil), + }, + "task1": { + Input: task.Task{Type: "task1", Meta: "workflow=workflow.toml"}, + Expected: []Phase{{Task: "task2", DependsOn: "task1"}}, + }, + "task2": { + Input: task.Task{Type: "task2", Meta: "workflow=workflow.toml"}, + Expected: []Phase{{Task: "task3", DependsOn: "task2"}, {Task: "task4", DependsOn: "task2"}}, + }, + "task3": { + Input: task.Task{Type: "task3", Meta: "workflow=workflow.toml"}, + Expected: []Phase{}, + }, + "task4": { + Input: task.Task{Type: "task4", Meta: "workflow=workflow.toml"}, + Expected: []Phase{}, + }, + "task1:j4": { + Input: task.Task{Type: "task1", Meta: "workflow=workflow.toml&job=j4"}, + Expected: []Phase{ + {Task: "task2", DependsOn: "task1"}, + {Task: "task5", DependsOn: "task1:j4"}, + }, + }, + } + trial.New(fn, cases).SubTest(t) +} + +func TestCache_FilePath(t *testing.T) { + // TODO: remove receive struct as it is unneeded + fn := func(v trial.Input) (string, error) { + c := &SQLite{workflowPath: v.Slice(0).String()} + return c.filePath(v.Slice(1).String()), nil + } + cases := trial.Cases[trial.Input, string]{ + "single file": { + Input: trial.Args("./path", "./path/file.toml"), + Expected: "file.toml", + }, + "same name": { + Input: trial.Args("./path/file.toml", "./path/file.toml"), + Expected: "file.toml", + }, + "sub directory": { + Input: trial.Args("./path", "./path/sub/file.toml"), + Expected: "sub/file.toml", + }, + "embedded": { + Input: trial.Args("./path", "root/folder/path/file.toml"), + Expected: "file.toml", + }, + "embedded sub": { + Input: trial.Args("./path", "root/path/sub/file.toml"), + Expected: "sub/file.toml", + }, + } + trial.New(fn, cases).SubTest(t) +} diff --git a/apps/flowlord/taskmaster.go b/apps/flowlord/taskmaster.go index b64163a1..a43bc57c 100644 --- a/apps/flowlord/taskmaster.go +++ b/apps/flowlord/taskmaster.go @@ -11,6 +11,7 @@ import ( "regexp" "strconv" "strings" + "sync/atomic" "time" gtools "github.com/jbsmith7741/go-tools" @@ -19,13 +20,14 @@ import ( "github.com/pcelvng/task/bus" "github.com/robfig/cron/v3" - "github.com/pcelvng/task-tools/apps/flowlord/cache" + "github.com/pcelvng/task-tools/apps/flowlord/sqlite" "github.com/pcelvng/task-tools/file" "github.com/pcelvng/task-tools/slack" "github.com/pcelvng/task-tools/tmpl" - "github.com/pcelvng/task-tools/workflow" ) +var cronParser = cron.NewParser(cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor) + type taskMaster struct { // options @@ -40,25 +42,36 @@ type taskMaster struct { fOpts *file.Options doneTopic string failedTopic string - taskCache *cache.Memory - *workflow.Cache - port int - cron *cron.Cron - slack *Notification - files []fileRule + taskCache *sqlite.SQLite + HostName string + port int + cron *cron.Cron + slack *Notification + files []fileRule alerts chan task.Task } type Notification struct { slack.Slack - ReportPath string - MinFrequency time.Duration - MaxFrequency time.Duration + //ReportPath string + MinFrequency time.Duration + MaxFrequency time.Duration + currentDuration atomic.Int64 // Current notification duration (atomically updated) file *file.Options } +// GetCurrentDuration returns the current notification duration +func (n *Notification) GetCurrentDuration() time.Duration { + return time.Duration(n.currentDuration.Load()) +} + +// setCurrentDuration atomically sets the current notification duration +func (n *Notification) setCurrentDuration(d time.Duration) { + n.currentDuration.Store(int64(d)) +} + type stats struct { AppName string `json:"app_name"` Version string `json:"version"` @@ -97,11 +110,16 @@ func New(opts *options) *taskMaster { if opts.Slack.MaxFrequency <= opts.Slack.MinFrequency { opts.Slack.MaxFrequency = 16 * opts.Slack.MinFrequency } + if err := opts.DB.Open(opts.Workflow, opts.File); err != nil { + log.Fatal("db init", err) + } opts.Slack.file = opts.File + // Initialize current duration to MinFrequency + opts.Slack.setCurrentDuration(opts.Slack.MinFrequency) tm := &taskMaster{ initTime: time.Now(), - taskCache: cache.NewMemory(opts.TaskTTL), + taskCache: opts.DB, path: opts.Workflow, doneTopic: opts.DoneTopic, failedTopic: opts.FailedTopic, @@ -109,7 +127,8 @@ func New(opts *options) *taskMaster { producer: producer, doneConsumer: consumer, port: opts.Port, - cron: cron.New(cron.WithSeconds()), + HostName: opts.Host, + cron: cron.New(cron.WithParser(cronParser)), dur: opts.Refresh, slack: opts.Slack, alerts: make(chan task.Task, 20), @@ -137,7 +156,7 @@ func pName(topic, job string) string { } func (tm *taskMaster) getAllChildren(topic, workflow, job string) (s []string) { - for _, c := range tm.Children(task.Task{Type: topic, Meta: "workflow=" + workflow + "&job=" + job}) { + for _, c := range tm.taskCache.Children(task.Task{Type: topic, Meta: "workflow=" + workflow + "&job=" + job}) { job := strings.Trim(c.Topic()+":"+c.Job(), ":") if children := tm.getAllChildren(c.Task, workflow, c.Job()); len(children) > 0 { job += " ➞ " + strings.Join(children, " ➞ ") @@ -148,17 +167,8 @@ func (tm *taskMaster) getAllChildren(topic, workflow, job string) (s []string) { } func (tm *taskMaster) refreshCache() ([]string, error) { - stat := tm.taskCache.Recycle() - if stat.Removed > 0 { - log.Printf("task-cache: size %d removed %d time: %v", stat.Count, stat.Removed, stat.ProcessTime) - for _, t := range stat.Unfinished { - // add unfinished tasks to alerts channel - t.Msg += "unfinished task detected" - tm.alerts <- t - } - } - - files, err := tm.Cache.Refresh() + // Reload workflow files + files, err := tm.taskCache.Refresh() if err != nil { return nil, fmt.Errorf("error reloading workflow: %w", err) } @@ -180,20 +190,28 @@ func (tm *taskMaster) refreshCache() ([]string, error) { } func (tm *taskMaster) Run(ctx context.Context) (err error) { - if tm.Cache, err = workflow.New(tm.path, tm.fOpts); err != nil { - return fmt.Errorf("workflow setup %w", err) - } + // The SQLite struct now implements the workflow.Cache interface directly - // refresh the workflow if the file(s) have been changed + // check for alerts from today on startup // refresh the workflow if the file(s) have been changed _, err = tm.refreshCache() if err != nil { log.Fatal(err) } go func() { // auto refresh cache after set duration - tick := time.NewTicker(tm.dur) - for range tick.C { - if _, err := tm.refreshCache(); err != nil { - log.Println(err) + workflowTick := time.NewTicker(tm.dur) + DBTick := time.NewTicker(24 * time.Hour) + for { + select { + case <-DBTick.C: + if i, err := tm.taskCache.Recycle(time.Now().Add(-tm.taskCache.Retention)); err != nil { + log.Println("task cache recycle:", err) + } else { + log.Printf("task cache recycled %d old records", i) + } + case <-workflowTick.C: + if _, err := tm.refreshCache(); err != nil { + log.Println(err) + } } } }() @@ -206,50 +224,39 @@ func (tm *taskMaster) Run(ctx context.Context) (err error) { go tm.readFiles(ctx) go tm.StartHandler() - go tm.slack.handleNotifications(tm.alerts, ctx) + go tm.handleNotifications(tm.alerts, ctx) <-ctx.Done() log.Println("shutting down") - return nil - -} - -func validatePhase(p workflow.Phase) string { - if p.DependsOn == "" { - if p.Rule == "" { - return "invalid phase: rule and dependsOn are blank" - } - // verify at least one valid rule is there - rules, _ := url.ParseQuery(p.Rule) - if rules.Get("cron") == "" { - return fmt.Sprintf("no valid rule found: %v", p.Rule) - } - - return "" - - } - // DependsOn != "" - if p.Rule != "" { - return fmt.Sprintf("ignored rule: %v", p.Rule) - } - - return "" + return tm.taskCache.Close() } // schedule the tasks and refresh the schedule when updated func (tm *taskMaster) schedule() (err error) { errs := make([]error, 0) - if len(tm.Workflows) == 0 { + + // Get all workflow files from database + workflowFiles := tm.taskCache.GetWorkflowFiles() + + if len(workflowFiles) == 0 { return fmt.Errorf("no workflows found check path %s", tm.path) } - for path, workflow := range tm.Workflows { - for _, w := range workflow.Phases { + + // Get all phases for each workflow file + for _, filePath := range workflowFiles { + phases, err := tm.taskCache.GetPhasesForWorkflow(filePath) + if err != nil { + errs = append(errs, fmt.Errorf("error getting phases for %s: %w", filePath, err)) + continue + } + + for _, w := range phases { rules, _ := url.ParseQuery(w.Rule) cronSchedule := rules.Get("cron") if f := rules.Get("files"); f != "" { r := fileRule{ SrcPattern: f, - workflowFile: path, - Phase: w, + workflowFile: filePath, + Phase: w.ToWorkflowPhase(), CronCheck: cronSchedule, } r.CountCheck, _ = strconv.Atoi(rules.Get("count")) @@ -260,17 +267,18 @@ func (tm *taskMaster) schedule() (err error) { } if cronSchedule == "" { - log.Printf("no cron: task:%s, rule:%s", w.Task, w.Rule) + //log.Printf("no cron: task:%s, rule:%s", w.Task, w.Rule) + // this should already be in the status field continue } - j, err := tm.NewJob(w, path) + j, err := tm.NewJob(w.ToWorkflowPhase(), filePath) if err != nil { errs = append(errs, fmt.Errorf("issue with %s %w", w.Task, err)) } if _, err = tm.cron.AddJob(cronSchedule, j); err != nil { - errs = append(errs, fmt.Errorf("invalid rule for %s:%s %s %w", path, w.Task, w.Rule, err)) + errs = append(errs, fmt.Errorf("invalid rule for %s:%s %s %w", filePath, w.Task, w.Rule, err)) } } } @@ -286,8 +294,13 @@ func (tm *taskMaster) Process(t *task.Task) error { meta, _ := url.ParseQuery(t.Meta) tm.taskCache.Add(*t) // attempt to retry - if t.Result == task.ErrResult { - p := tm.Get(*t) + switch t.Result { + case task.WarnResult: + return nil // do nothing + case task.AlertResult: + tm.alerts <- *t + case task.ErrResult: + p := tm.taskCache.Get(*t) rules, _ := url.ParseQuery(p.Rule) r := meta.Get("retry") @@ -318,37 +331,30 @@ func (tm *taskMaster) Process(t *task.Task) error { } }() return nil - } else { // send to the retry failed topic if retries > p.Retry - meta.Set("retry", "failed") - meta.Set("retried", strconv.Itoa(p.Retry)) - t.Meta = meta.Encode() - if tm.failedTopic != "-" && tm.failedTopic != "" { - tm.taskCache.Add(*t) - if err := tm.producer.Send(tm.failedTopic, t.JSONBytes()); err != nil { - return err - } - } - - // don't alert if slack isn't enabled or disabled in phase - if tm.slack == nil || rules.Get("no_alert") != "" { - return nil + } + // send to the retry failed topic if retries > p.Retry + meta.Set("retry", "failed") + meta.Set("retried", strconv.Itoa(p.Retry)) + t.Meta = meta.Encode() + if tm.failedTopic != "-" && tm.failedTopic != "" { + tm.taskCache.Add(*t) + if err := tm.producer.Send(tm.failedTopic, t.JSONBytes()); err != nil { + return err } - tm.alerts <- *t } - return nil - } - - if t.Result == task.AlertResult && tm.slack != nil { - if tm.slack != nil { - tm.alerts <- *t + // don't alert if slack isn't enabled or disabled in phase + if rules.Get("no_alert") != "" { + return nil } - } - // start off any children tasks - if t.Result == task.CompleteResult { + tm.alerts <- *t + + return nil + case task.CompleteResult: + // start off any children tasks taskTime := tmpl.TaskTime(*t) - phases := tm.Children(*t) + phases := tm.taskCache.Children(*t) for _, p := range phases { if !isReady(p.Rule, t.Meta) { @@ -401,6 +407,7 @@ func (tm *taskMaster) Process(t *task.Task) error { log.Printf("no matches found for %v:%v", t.Type, t.Job) } return nil + } return fmt.Errorf("unknown result %q %s", t.Result, t.JSONString()) } @@ -465,28 +472,49 @@ func (tm *taskMaster) readFiles(ctx context.Context) { } } -// handleNotifications gathers all 'failed' tasks and +// handleNotifications gathers all 'failed' tasks and incomplete tasks // sends a summary message every X minutes // It uses an exponential backoff to limit the number of messages // ie, (min) 5 -> 10 -> 20 -> 40 -> 80 -> 160 (max) // The backoff is cleared after no failed tasks occur within the window -func (n *Notification) handleNotifications(taskChan chan task.Task, ctx context.Context) { +func (tm *taskMaster) handleNotifications(taskChan chan task.Task, ctx context.Context) { sendChan := make(chan struct{}) - tasks := make([]task.Task, 0) + var alerts []sqlite.AlertRecord + + // Initialize lastAlertTime to today at 00:00:00 (zero hour) + now := time.Now() + lastAlertTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + go func() { - dur := n.MinFrequency - for { - if len(tasks) > 0 { + dur := tm.slack.MinFrequency + for ; ; time.Sleep(dur) { + var err error + + // Check for incomplete tasks and add them to alerts + tm.taskCache.CheckIncompleteTasks() + + // Get NEW alerts only - those after the last time we sent + alerts, err = tm.taskCache.GetAlertsAfterTime(lastAlertTime) + if err != nil { + log.Printf("failed to retrieve alerts: %v", err) + continue + } + + if len(alerts) > 0 { sendChan <- struct{}{} - if dur *= 2; dur > n.MaxFrequency { - dur = n.MaxFrequency + // Update lastAlertTime to now (before we send, so we don't miss any) + lastAlertTime = time.Now() + if dur *= 2; dur > tm.slack.MaxFrequency { + dur = tm.slack.MaxFrequency } + tm.slack.setCurrentDuration(dur) // Update current duration atomically log.Println("wait time ", dur) - } else if dur != n.MinFrequency { - dur = n.MinFrequency + } else if dur != tm.slack.MinFrequency { + // No NEW alerts - reset to minimum frequency + dur = tm.slack.MinFrequency + tm.slack.setCurrentDuration(dur) // Update current duration atomically log.Println("Reset ", dur) } - time.Sleep(dur) } }() for { @@ -495,47 +523,19 @@ func (n *Notification) handleNotifications(taskChan chan task.Task, ctx context. // if the task result is an alert result, send a slack notification now if tsk.Result == task.AlertResult { b, _ := json.MarshalIndent(tsk, "", " ") - if err := n.Slack.Notify(string(b), slack.Critical); err != nil { + if err := tm.slack.Slack.Notify(string(b), slack.Critical); err != nil { log.Println(err) } } else { // if the task result is not an alert result add to the tasks list summary - tasks = append(tasks, tsk) + if err := tm.taskCache.AddAlert(tsk, tsk.Msg); err != nil { + log.Printf("failed to store alert: %v", err) + } } case <-sendChan: // prepare message - m := make(map[string]*alertStat) // [task:job]message - fPath := tmpl.Parse(n.ReportPath, time.Now()) - writer, err := file.NewWriter(fPath, n.file) - if err != nil { + if err := tm.sendAlertSummary(alerts); err != nil { log.Println(err) } - for _, tsk := range tasks { - writer.WriteLine(tsk.JSONBytes()) - - meta, _ := url.ParseQuery(tsk.Meta) - key := tsk.Type + ":" + meta.Get("job") - v, found := m[key] - if !found { - v = &alertStat{key: key, times: make([]time.Time, 0)} - m[key] = v - } - v.count++ - v.times = append(v.times, tmpl.TaskTime(tsk)) - } - - var s string - for k, v := range m { - s += fmt.Sprintf("%-35s%5d %v\n", k, v.count, tmpl.PrintDates(v.times)) - } - if err := writer.Close(); err == nil && fPath != "" { - s += "see report at " + fPath - } - fmt.Println(s) - if err := n.Slack.Notify(s, slack.Critical); err != nil { - log.Println(err) - } - - tasks = tasks[0:0] // reset slice case <-ctx.Done(): return } @@ -543,10 +543,34 @@ func (n *Notification) handleNotifications(taskChan chan task.Task, ctx context. } -type alertStat struct { - key string - count int - times []time.Time +// sendAlertSummary sends a formatted alert summary to Slack +// This can be reused by backup alert system and other components +func (tm *taskMaster) sendAlertSummary(alerts []sqlite.AlertRecord) error { + if len(alerts) == 0 { + return nil + } + + // build compact summary using existing logic + summary := sqlite.BuildCompactSummary(alerts) + + // format message similar to current Slack format + var message strings.Builder + message.WriteString(fmt.Sprintf("see report at %v:%d/web/alert?date=%s\n", tm.HostName, tm.port, time.Now().Format("2006-01-02"))) + + for _, line := range summary { + message.WriteString(fmt.Sprintf("%-35s%5d %s\n", + line.Key+":", line.Count, line.TimeRange)) + } + + // send to Slack if configured + log.Println(message.String()) + if tm.slack != nil { + if err := tm.slack.Notify(message.String(), slack.Critical); err != nil { + return fmt.Errorf("failed to send alert summary to Slack: %w", err) + } + } + + return nil } // jitterPercent will return a time.Duration representing extra diff --git a/apps/flowlord/taskmaster_test.go b/apps/flowlord/taskmaster_test.go index 50db832a..5895ce1b 100644 --- a/apps/flowlord/taskmaster_test.go +++ b/apps/flowlord/taskmaster_test.go @@ -15,6 +15,7 @@ import ( "github.com/pcelvng/task/bus/nop" "github.com/robfig/cron/v3" + "github.com/pcelvng/task-tools/apps/flowlord/sqlite" "github.com/pcelvng/task-tools/workflow" ) @@ -22,7 +23,9 @@ const base_test_path string = "../../internal/test/" func TestTaskMaster_Process(t *testing.T) { delayRegex := regexp.MustCompile(`delayed=(\d+.\d+)`) - cache, fatalErr := workflow.New(base_test_path+"workflow", nil) + // Initialize taskCache for the test + taskCache := &sqlite.SQLite{LocalPath: ":memory:"} + fatalErr := taskCache.Open(base_test_path+"workflow", nil) if fatalErr != nil { t.Fatal("cache init", fatalErr) } @@ -32,7 +35,8 @@ func TestTaskMaster_Process(t *testing.T) { } fn := func(tsk task.Task) ([]task.Task, error) { var alerts int64 - tm := taskMaster{doneConsumer: consumer, Cache: cache, failedTopic: "failed-topic", alerts: make(chan task.Task), slack: &Notification{}} + + tm := taskMaster{doneConsumer: consumer, taskCache: taskCache, failedTopic: "failed-topic", alerts: make(chan task.Task), slack: &Notification{}} producer, _ := nop.NewProducer("") tm.producer = producer nop.FakeMsg = tsk.JSONBytes() @@ -79,6 +83,7 @@ func TestTaskMaster_Process(t *testing.T) { "task1 attempt 0": { Input: task.Task{ Type: "task1", + Job: "t2", Info: "?date=2019-12-12", Result: task.ErrResult, Started: "now", @@ -97,6 +102,7 @@ func TestTaskMaster_Process(t *testing.T) { "task1 attempt 2": { Input: task.Task{ Type: "task1", + Job: "t2", Info: "?date=2019-12-12", Result: task.ErrResult, ID: "UUID_task1_attempt2", @@ -271,12 +277,12 @@ func TestTaskMaster_Schedule(t *testing.T) { Files []fileRule } fn := func(in string) (expected, error) { - cache, err := workflow.New(base_test_path+in, nil) - if err != nil { + tm := taskMaster{cron: cron.New(cron.WithParser(cronParser))} + tm.taskCache = &sqlite.SQLite{LocalPath: ":memory:"} + if err := tm.taskCache.Open(base_test_path+in, nil); err != nil { return expected{}, err } - tm := taskMaster{Cache: cache, cron: cron.New()} - err = tm.schedule() + err := tm.schedule() exp := expected{ Jobs: make([]Cronjob, 0), Files: tm.files, @@ -296,7 +302,7 @@ func TestTaskMaster_Schedule(t *testing.T) { Name: "t2", Workflow: "f1.toml", Topic: "task1", - Schedule: "0 0 * * * *", + Schedule: "0 * * * *", Offset: -4 * time.Hour, Template: "?date={yyyy}-{mm}-{dd}T{hh}", }, @@ -304,7 +310,7 @@ func TestTaskMaster_Schedule(t *testing.T) { Name: "t4", Workflow: "f1.toml", Topic: "task1", - Schedule: "0 0 * * * *", + Schedule: "0 * * * *", Offset: -4 * time.Hour, Template: "?date={yyyy}-{mm}-{dd}T{hh}", }, @@ -318,7 +324,7 @@ func TestTaskMaster_Schedule(t *testing.T) { { Workflow: "f3.toml", Topic: "task1", - Schedule: "0 0 0 * * *", + Schedule: "0 0 * * *", Template: "?date={yyyy}-{mm}-{dd}", }, }, @@ -375,7 +381,7 @@ func Test_NewJob(t *testing.T) { }, Expected: &Cronjob{ Topic: "task5", - Schedule: "0 35 13 * * *", + Schedule: "35 13 * * *", Template: "?date={yyyy}-{mm}-{dd}", }, }, @@ -523,37 +529,3 @@ func TestIsReady(t *testing.T) { } trial.New(fn, cases).Test(t) } - -func TestValidatePhase(t *testing.T) { - fn := func(in workflow.Phase) (string, error) { - s := validatePhase(in) - if s != "" { - return "", errors.New(s) - } - return s, nil - } - cases := trial.Cases[workflow.Phase, string]{ - "empty phase": { - Input: workflow.Phase{}, - ExpectedErr: errors.New("invalid phase"), - }, - "valid cron phase": { - Input: workflow.Phase{ - Rule: "cron=* * * * * *", - }, - Expected: "", - }, - "unknown rule": { - Input: workflow.Phase{Rule: "abcedfg"}, - ShouldErr: true, - }, - "dependsOn and rule": { - Input: workflow.Phase{ - Rule: "cron=abc", - DependsOn: "task1", - }, - ShouldErr: true, - }, - } - trial.New(fn, cases).SubTest(t) -} diff --git a/apps/flowlord/test/tasks.json b/apps/flowlord/test/tasks.json new file mode 100644 index 00000000..d90c083d --- /dev/null +++ b/apps/flowlord/test/tasks.json @@ -0,0 +1,210 @@ +[ + { + "id": "task-1", + "type": "csv2json", + "job": "hourly", + "info": "?date=2024-01-15&hour=10", + "result": "complete", + "meta": "cron=2024-01-15T10&job=hourly", + "msg": "Successfully processed 1000 records", + "task_seconds": 325, + "task_time": "00:05:25", + "queue_seconds": 5, + "queue_time": "00:00:05", + "created": "2024-01-15T10:00:00Z", + "started": "2024-01-15T10:00:05Z", + "ended": "2024-01-15T10:05:30Z" + }, + { + "id": "task-2", + "type": "csv2json", + "job": "hourly", + "info": "?date=2024-01-15&hour=11", + "result": "complete", + "meta": "cron=2024-01-15T11&job=hourly", + "msg": "Successfully processed 1200 records", + "task_seconds": 380, + "task_time": "00:06:20", + "queue_seconds": 8, + "queue_time": "00:00:08", + "created": "2024-01-15T11:00:00Z", + "started": "2024-01-15T11:00:08Z", + "ended": "2024-01-15T11:06:28Z" + }, + { + "id": "task-long-id-abcdefg-3", + "type": "csv2json", + "job": "hourly", + "info": "?date=2024-01-15&hour=12", + "result": "error", + "meta": "cron=2024-01-15T12&job=hourly", + "msg": "Failed to read input file: file not found", + "task_seconds": 45, + "task_time": "00:00:45", + "queue_seconds": 3, + "queue_time": "00:00:03", + "created": "2024-01-15T12:00:00Z", + "started": "2024-01-15T12:00:03Z", + "ended": "2024-01-15T12:00:48Z" + }, + { + "id": "task-4", + "type": "filecopy", + "job": "backup", + "info": "?source=/data/users&dest=/backup/users", + "result": "complete", + "meta": "cron=2024-01-15T13&job=backup", + "msg": "Copied 50 files successfully", + "task_seconds": 935, + "task_time": "00:15:35", + "queue_seconds": 10, + "queue_time": "00:00:10", + "created": "2024-01-15T13:00:00Z", + "started": "2024-01-15T13:00:10Z", + "ended": "2024-01-15T13:15:45Z" + }, + { + "id": "task-5", + "type": "filecopy", + "job": "backup", + "info": "?source=/data/orders&dest=/backup/orders", + "result": "complete", + "meta": "cron=2024-01-15T14&job=backup", + "msg": "Copied 75 files successfully", + "task_seconds": 1200, + "task_time": "00:20:00", + "queue_seconds": 12, + "queue_time": "00:00:12", + "created": "2024-01-15T14:00:00Z", + "started": "2024-01-15T14:00:12Z", + "ended": "2024-01-15T14:20:00Z" + }, + { + "id": "task-6", + "type": "filecopy", + "job": "backup", + "info": "?source=/data/products&dest=/backup/products", + "result": "warn", + "meta": "cron=2024-01-15T15&job=backup", + "msg": "Some files were locked and skipped", + "task_seconds": 600, + "task_time": "00:10:00", + "queue_seconds": 5, + "queue_time": "00:00:05", + "created": "2024-01-15T15:00:00Z", + "started": "2024-01-15T15:00:05Z", + "ended": "2024-01-15T15:10:00Z" + }, + { + "id": "task-7", + "type": "transform", + "job": "realtime", + "info": "?input=stream1&output=processed1", + "result": "alert", + "meta": "cron=2024-01-15T16&job=realtime", + "msg": "High memory usage detected", + "task_seconds": 145, + "task_time": "00:02:25", + "queue_seconds": 5, + "queue_time": "00:00:05", + "created": "2024-01-15T16:00:00Z", + "started": "2024-01-15T16:00:05Z", + "ended": "2024-01-15T16:02:30Z" + }, + { + "id": "task-8", + "type": "transform", + "job": "realtime", + "info": "?input=stream2&output=processed2", + "result": "complete", + "meta": "cron=2024-01-15T17&job=realtime", + "msg": "Successfully processed 5000 events", + "task_seconds": 200, + "task_time": "00:03:20", + "queue_seconds": 7, + "queue_time": "00:00:07", + "created": "2024-01-15T17:00:00Z", + "started": "2024-01-15T17:00:07Z", + "ended": "2024-01-15T17:03:27Z" + }, + { + "id": "task-9", + "type": "transform", + "job": "realtime", + "info": "?input=stream3&output=processed3", + "result": "complete", + "meta": "cron=2024-01-15T18&job=realtime", + "msg": "Successfully processed 3000 events", + "task_seconds": 180, + "task_time": "00:03:00", + "queue_seconds": 4, + "queue_time": "00:00:04", + "created": "2024-01-15T18:00:00Z", + "started": "2024-01-15T18:00:04Z", + "ended": "2024-01-15T18:03:04Z" + }, + { + "id": "task-10", + "type": "bigquery", + "job": "analytics", + "info": "?table=events&date=2024-01-15", + "result": "warn", + "meta": "cron=2024-01-15T19&job=analytics", + "msg": "Slow query execution", + "task_seconds": 1515, + "task_time": "00:25:15", + "queue_seconds": 15, + "queue_time": "00:00:15", + "created": "2024-01-15T19:00:00Z", + "started": "2024-01-15T19:00:15Z", + "ended": "2024-01-15T19:25:30Z" + }, + { + "id": "task-11", + "type": "bigquery", + "job": "analytics", + "info": "?table=users&date=2024-01-15", + "result": "complete", + "meta": "cron=2024-01-15T20&job=analytics", + "msg": "Query completed successfully", + "task_seconds": 800, + "task_time": "00:13:20", + "queue_seconds": 10, + "queue_time": "00:00:10", + "created": "2024-01-15T20:00:00Z", + "started": "2024-01-15T20:00:10Z", + "ended": "2024-01-15T20:13:30Z" + }, + { + "id": "task-12", + "type": "bigquery", + "job": "analytics", + "info": "?table=orders&date=2024-01-15", + "result": "error", + "meta": "cron=2024-01-15T21&job=analytics", + "msg": "Query failed: table not found", + "task_seconds": 30, + "task_time": "00:00:30", + "queue_seconds": 5, + "queue_time": "00:00:05", + "created": "2024-01-15T21:00:00Z", + "started": "2024-01-15T21:00:05Z", + "ended": "2024-01-15T21:00:35Z" + }, + { + "id": "task-13", + "type": "transform", + "job": "realtime", + "info": "?input=stream4&output=processed4", + "result": "running", + "meta": "cron=2024-01-15T22&job=realtime", + "msg": "Processing data stream in progress", + "task_seconds": 0, + "task_time": "N/A", + "queue_seconds": 0, + "queue_time": "N/A", + "created": "2024-01-15T22:00:00Z", + "started": "2024-01-15T22:00:02Z", + "ended": null + } +] \ No newline at end of file diff --git a/apps/go.mod b/apps/go.mod index e8b4cb7a..ccaa1434 100644 --- a/apps/go.mod +++ b/apps/go.mod @@ -2,7 +2,7 @@ module github.com/pcelvng/task-tools/apps go 1.23.0 -toolchain go1.23.3 +toolchain go1.24.0 require ( cloud.google.com/go v0.120.0 @@ -31,6 +31,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.10.0 google.golang.org/api v0.228.0 + modernc.org/sqlite v1.37.0 ) require ( @@ -60,6 +61,7 @@ require ( github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/minio-go/v7 v7.0.26 // indirect @@ -67,6 +69,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nsqio/go-nsq v1.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -74,6 +77,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.2.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect @@ -85,15 +89,15 @@ require ( go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect golang.org/x/crypto v0.36.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/mod v0.22.0 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.28.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/tools v0.31.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect @@ -103,4 +107,7 @@ require ( gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.62.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.9.1 // indirect ) diff --git a/apps/go.sum b/apps/go.sum index ea2c9a07..02e0a870 100644 --- a/apps/go.sum +++ b/apps/go.sum @@ -198,6 +198,8 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= @@ -241,8 +243,6 @@ github.com/jbsmith7741/go-tools v0.4.1/go.mod h1:8v8ffjiI3qOs6epawzxmPB7AOKoNNxZ github.com/jbsmith7741/trial v0.3.1 h1:JZ0/w3lhfH4iacf9R2DnZWtTMa/Uf4O13gnuMLTub/M= github.com/jbsmith7741/trial v0.3.1/go.mod h1:M4FQWUgVpPY2+i53L2nSB0AyPc86kSTIigcr9Q7XQlY= github.com/jbsmith7741/uri v0.4.1/go.mod h1:Ctt8YJ5gCFx5BX/FMFg5VkwuI9buBcvsITIiSMH+TeA= -github.com/jbsmith7741/uri v0.6.0 h1:CXpHG6LnzqvaoKZNTZCDuvrZ1hAx1hg5dtcnsKTsjwE= -github.com/jbsmith7741/uri v0.6.0/go.mod h1:Ctt8YJ5gCFx5BX/FMFg5VkwuI9buBcvsITIiSMH+TeA= github.com/jbsmith7741/uri v0.6.1 h1:RloJXpTe1lXMBSMbyGn+YCBR//aO+OTDUgJ2wH5bQEU= github.com/jbsmith7741/uri v0.6.1/go.mod h1:Ctt8YJ5gCFx5BX/FMFg5VkwuI9buBcvsITIiSMH+TeA= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= @@ -273,6 +273,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= @@ -292,6 +294,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nsqio/go-nsq v1.0.8/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= @@ -317,6 +321,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -398,8 +404,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -423,8 +429,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -522,6 +528,7 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -586,8 +593,8 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -722,6 +729,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/cc/v4 v4.25.2 h1:T2oH7sZdGvTaie0BRNFbIYsabzCxUQg8nLqCdQ2i0ic= +modernc.org/cc/v4 v4.25.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.25.1 h1:TFSzPrAGmDsdnhT9X2UrcPMI3N/mJ9/X9ykKXwLhDsU= +modernc.org/ccgo/v4 v4.25.1/go.mod h1:njjuAYiPflywOOrm3B7kCB444ONP5pAVr8PIEoE0uDw= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.62.1 h1:s0+fv5E3FymN8eJVmnk0llBe6rOxCu/DEU+XygRbS8s= +modernc.org/libc v1.62.1/go.mod h1:iXhATfJQLjG3NWy56a6WVU73lWOcdYVxsvwCgoPljuo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.9.1 h1:V/Z1solwAVmMW1yttq3nDdZPJqV1rM05Ccq6KMSZ34g= +modernc.org/memory v1.9.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= +modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/apps/utils/fz/main.go b/apps/utils/fz/main.go index 959d43dc..ab890654 100644 --- a/apps/utils/fz/main.go +++ b/apps/utils/fz/main.go @@ -2,15 +2,17 @@ package main import ( "fmt" - "github.com/pcelvng/task-tools/slack" + "io" "log" "os" - "path/filepath" "strconv" "strings" "time" + "github.com/pcelvng/task-tools/slack" + "github.com/hydronica/go-config" + "github.com/pcelvng/task-tools/file" "github.com/pcelvng/task-tools/file/stat" ) @@ -42,6 +44,8 @@ func main() { err = ls(f1, &conf) case "cat": err = cat(f1, &conf) + case "stat": + err = stats(f1, &conf) case "cp": err = cp(f1, f2, &conf) case "slack": @@ -146,26 +150,26 @@ func cp(from, to string, opt *file.Options) error { if to == "" || from == "" { return fmt.Errorf(usage) } - sts, _ := file.Stat(to, opt) - if sts.IsDir { - _, fName := filepath.Split(from) - to = strings.TrimRight(to, "/") + "/" + fName - } r, err := file.NewReader(from, opt) if err != nil { - return fmt.Errorf("reader init for %s %w", from, err) + return fmt.Errorf("init reader err: %w", err) } w, err := file.NewWriter(to, opt) if err != nil { - return fmt.Errorf("writer init for %s %w", to, err) + return fmt.Errorf("init writer err: %w", err) } - - s := file.NewScanner(r) - for s.Scan() { - if err := w.WriteLine(s.Bytes()); err != nil { - w.Abort() - return fmt.Errorf("write error: %w", err) - } + _, err = io.Copy(w, r) + if err != nil { + return fmt.Errorf("copy err: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("close writer err: %w", err) } - return w.Close() + return r.Close() +} + +func stats(path string, opt *file.Options) error { + sts, err := file.Stat(path, opt) + fmt.Println(sts.JSONString()) + return err } diff --git a/apps/utils/recap/app_test.go b/apps/utils/recap/app_test.go index 3bfacb8f..0a52193a 100644 --- a/apps/utils/recap/app_test.go +++ b/apps/utils/recap/app_test.go @@ -46,17 +46,17 @@ func TestDoneTopic(t *testing.T) { cases := trial.Cases[string, []string]{ "task without job": { Input: `{"type":"test1","info":"?date=2020-01-02","result":"complete"}`, - Expected: []string{"test1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02"}, + Expected: []string{"test1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02T00"}, }, "task with job meta": { Input: `{"type":"test2","info":"?date=2020-01-02","result":"complete","meta":"job=part1"}`, - Expected: []string{"test2:part1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02"}, + Expected: []string{"test2:part1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02T00"}, }, "2 task with job meta": { Input: `{"type":"test3","info":"?date=2020-01-02","result":"complete","meta":"job=part1"} {"type":"test3","info":"?date=2020-01-02","result":"complete","meta":"job=part2"}`, - Expected: []string{"test3:part1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02", - "test3:part2\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02"}, + Expected: []string{"test3:part1\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02T00", + "test3:part2\n\tmin: 0s max 0s avg:0s\n\tComplete 1 2020/01/02T00"}, }, } trial.New(fn, cases).Test(t) diff --git a/go.work.sum b/go.work.sum index 1861a5bf..b0f6940f 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1740,8 +1740,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -1808,8 +1806,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -2074,7 +2070,6 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2534,8 +2529,6 @@ modernc.org/tcl v1.13.2 h1:5PQgL/29XkQ9wsEmmNPjzKs+7iPCaYqUJAhzPvQbjDA= modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= diff --git a/internal/Docs/PG_Tables.md b/internal/docs/PG_Tables.md similarity index 100% rename from internal/Docs/PG_Tables.md rename to internal/docs/PG_Tables.md diff --git a/internal/docs/img/flowlord_alerts.png b/internal/docs/img/flowlord_alerts.png new file mode 100644 index 00000000..254057b6 Binary files /dev/null and b/internal/docs/img/flowlord_alerts.png differ diff --git a/internal/docs/img/flowlord_files.png b/internal/docs/img/flowlord_files.png new file mode 100644 index 00000000..d1e7bf5c Binary files /dev/null and b/internal/docs/img/flowlord_files.png differ diff --git a/internal/docs/img/flowlord_tasks.png b/internal/docs/img/flowlord_tasks.png new file mode 100644 index 00000000..2deaff40 Binary files /dev/null and b/internal/docs/img/flowlord_tasks.png differ diff --git a/internal/docs/img/flowlord_workflow.png b/internal/docs/img/flowlord_workflow.png new file mode 100644 index 00000000..566f61a7 Binary files /dev/null and b/internal/docs/img/flowlord_workflow.png differ diff --git a/internal/test/workflow/jobs.toml b/internal/test/workflow/jobs.toml index 2d0a6fc7..2a2ff051 100644 --- a/internal/test/workflow/jobs.toml +++ b/internal/test/workflow/jobs.toml @@ -1,6 +1,6 @@ [[phase]] task = "worker" -rule = "cron=0 5 * * *?job=parent_job" +rule = "cron=0 */5 * * *?job=parent_job" retry = 3 template = "?date={yyyy}-{mm}-{dd}T{hh}" @@ -19,4 +19,8 @@ template = "?day={yyyy}-{mm}-{dd}" [[phase]] task = "worker:child3" dependsOn="worker:child2" -template = "?day={yyyy}-{mm}-{dd}" \ No newline at end of file +template = "?day={yyyy}-{mm}-{dd}" + +[[phase]] +task = "worker:job" +rule="cron=1 2 */6 4 5 SUN&offset=-24h" \ No newline at end of file diff --git a/testing-framework.md b/testing-framework.md new file mode 100644 index 00000000..d65bce1a --- /dev/null +++ b/testing-framework.md @@ -0,0 +1,334 @@ +# Testing Framework Documentation + +This document outlines the testing patterns and frameworks used in the task-tools repository, focusing on unit test patterns, the hydronica/trial framework, and conversion strategies for testify-based tests. + +## Overview + +The repository uses a consistent testing approach with the following key components: + +1. **hydronica/trial** - Primary testing framework for table-driven tests +2. **Go's built-in testing** - Standard Go testing patterns +3. **testify/assert** - Legacy testing assertions (to be converted) +4. **Example functions** - Documentation-style tests + +## Testing Patterns + +### 1. hydronica/trial Framework + +The `hydronica/trial` framework is the primary testing tool used throughout the repository. It provides a clean, type-safe way to write table-driven tests. + +#### Basic Pattern + +```go +func TestFunctionName(t *testing.T) { + fn := func(input InputType) (OutputType, error) { + // Test logic here + return result, err + } + + cases := trial.Cases[InputType, OutputType]{ + "test case name": { + Input: inputValue, + Expected: expectedValue, + }, + "error case": { + Input: errorInput, + ShouldErr: true, + }, + } + + trial.New(fn, cases).Test(t) +} +``` + +#### Advanced Features + +**Comparers**: Custom comparison logic for complex types +```go +trial.New(fn, cases).Comparer(trial.Contains).Test(t) +trial.New(fn, cases).Comparer(trial.EqualOpt(trial.IgnoreAllUnexported)).Test(t) +``` + +**SubTests**: For complex test scenarios +```go +trial.New(fn, cases).SubTest(t) +``` + +**Timeouts**: For tests that might hang +```go +trial.New(fn, cases).Timeout(time.Second).SubTest(t) +``` + +#### Time Handling + +The repository uses trial's time utilities instead of literal `time.Date()` calls: + +```go +// Preferred (using trial utilities) +trial.TimeDay("2023-01-01") +trial.TimeHour("2023-01-01T12") +trial.Time(time.RFC3339, "2023-01-01T00:00:00Z") + +// Avoid (literal time.Date calls) +time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) +``` + +### 2. Standard Go Testing + +For simple tests that don't require table-driven patterns: + +```go +func TestSimpleFunction(t *testing.T) { + result := functionUnderTest() + if result != expected { + t.Errorf("got %v, expected %v", result, expected) + } +} +``` + +### 3. Example Functions + +Used for documentation and demonstrating API usage: + +```go +func ExampleFunctionName() { + // Example code + fmt.Println("output") + + // Output: + // output +} +``` + +Example functions are found in files like `file/writebyhour_test.go` and serve as both tests and documentation. + +### 4. TestMain Pattern + +For setup and teardown across multiple tests: + +```go +func TestMain(m *testing.M) { + // Setup code + code := m.Run() + // Cleanup code + os.Exit(code) +} +``` + +## Current Test File Analysis + +### Files Using hydronica/trial (41 files) + +The following files use the trial framework: + +- `apps/flowlord/taskmaster_test.go` +- `apps/flowlord/handler_test.go` +- `apps/flowlord/files_test.go` +- `apps/flowlord/batch_test.go` +- `apps/flowlord/cache/cache_test.go` +- `file/file_test.go` +- `file/util/util_test.go` +- `file/nop/nop_test.go` +- `file/minio/client_test.go` +- `file/minio/read_test.go` +- `file/minio/write_test.go` +- `file/local/read_test.go` +- `file/local/write_test.go` +- `file/local/local_test.go` +- `file/buf/buf_test.go` +- `file/stat/stat_test.go` +- `file/scanner_test.go` +- `workflow/workflow_test.go` +- `tmpl/tmpl_test.go` +- `db/prep_test.go` +- `db/batch/batch_test.go` +- `db/batch/stat_test.go` +- `consumer/discover_test.go` +- `bootstrap/bootstrap_test.go` +- `apps/workers/*/worker_test.go` (multiple worker test files) +- `apps/tm-archive/*/app_test.go` (multiple archive test files) +- `apps/utils/*/logger_test.go`, `stats_test.go`, `recap_test.go`, `filewatcher_test.go` + +### Files Using testify/assert (2 files) + +These files need conversion to trial or standard Go testing: + +- `apps/tm-archive/http/http_test.go` +- `apps/utils/filewatcher/watcher_test.go` + +### Example Function Usage + +Files with extensive example functions: +- `file/writebyhour_test.go` (13 example functions) + +## Conversion Strategies + +### From testify/assert to trial + +**Current testify pattern:** +```go +func TestFunction(t *testing.T) { + result := functionUnderTest() + assert.Equal(t, expected, result) + assert.NotNil(t, err) +} +``` + +**Convert to trial pattern:** +```go +func TestFunction(t *testing.T) { + fn := func(input InputType) (OutputType, error) { + return functionUnderTest(input) + } + + cases := trial.Cases[InputType, OutputType]{ + "success case": { + Input: testInput, + Expected: expectedOutput, + }, + "error case": { + Input: errorInput, + ShouldErr: true, + }, + } + + trial.New(fn, cases).Test(t) +} +``` + +### From testify/assert to standard Go testing + +For simple cases that don't benefit from table-driven tests: + +```go +func TestFunction(t *testing.T) { + result := functionUnderTest() + if result != expected { + t.Errorf("got %v, expected %v", result, expected) + } +} +``` + +## Best Practices + +### 1. Test Structure + +- Use descriptive test case names +- Group related test cases logically +- Use table-driven tests for multiple scenarios +- Keep test functions focused and single-purpose + +### 2. Error Testing + +```go +cases := trial.Cases[InputType, OutputType]{ + "error case": { + Input: errorInput, + ShouldErr: true, + }, +} +``` + +### 3. Time Testing + +Always use trial time utilities: +```go +// Good +trial.TimeDay("2023-01-01") +trial.TimeHour("2023-01-01T12") + +// Avoid +time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) +``` + +### 4. Complex Comparisons + +Use appropriate comparers for complex types: +```go +trial.New(fn, cases).Comparer(trial.EqualOpt(trial.IgnoreAllUnexported)).Test(t) +``` + +### 5. Test Organization + +- Place tests in `*_test.go` files +- Use `TestMain` for setup/teardown when needed +- Use example functions for API documentation +- Keep test data in separate files when appropriate + +## Migration Plan + +### Phase 1: Convert testify/assert usage + +1. **apps/tm-archive/http/http_test.go** + - Convert `assert.Equal` calls to trial cases + - Convert `assert.Contains` to appropriate trial comparers + +2. **apps/utils/filewatcher/watcher_test.go** + - Convert `assert.Equal` and `assert.NotNil` calls + - Create table-driven test cases + +### Phase 2: Standardize patterns + +1. Ensure all new tests use trial framework +2. Convert any remaining standard Go tests to trial when beneficial +3. Maintain example functions for documentation + +### Phase 3: Documentation and training + +1. Update this document as patterns evolve +2. Provide examples for common testing scenarios +3. Establish coding standards for test writing + +## Common Test Patterns + +### Testing with external dependencies + +```go +func TestWithDependencies(t *testing.T) { + fn := func(input InputType) (OutputType, error) { + // Setup mocks or test doubles + mockDep := &MockDependency{} + service := NewService(mockDep) + return service.Process(input) + } + + cases := trial.Cases[InputType, OutputType]{ + "success": { + Input: validInput, + Expected: expectedOutput, + }, + } + + trial.New(fn, cases).Test(t) +} +``` + +### Testing async operations + +```go +func TestAsyncOperation(t *testing.T) { + fn := func(input InputType) (OutputType, error) { + result := make(chan OutputType, 1) + err := make(chan error, 1) + + go func() { + output, e := asyncOperation(input) + result <- output + err <- e + }() + + return <-result, <-err + } + + cases := trial.Cases[InputType, OutputType]{ + "async success": { + Input: testInput, + Expected: expectedOutput, + }, + } + + trial.New(fn, cases).Timeout(5 * time.Second).Test(t) +} +``` + +This testing framework provides a consistent, maintainable approach to testing across the entire task-tools repository. diff --git a/tmpl/tmpl.go b/tmpl/tmpl.go index a226e9c2..05fff21d 100644 --- a/tmpl/tmpl.go +++ b/tmpl/tmpl.go @@ -286,61 +286,162 @@ type Getter interface { Get(string) string } +type granularity int + +const ( + granularityHourly granularity = iota + granularityDaily + granularityMonthly +) + +// isConsecutive checks if two times are consecutive based on the granularity +func isConsecutive(t1, t2 time.Time, gran granularity) bool { + // Equal times are always consecutive (handles duplicates) + if t1.Equal(t2) { + return true + } + + switch gran { + case granularityHourly: + return t2.Sub(t1) == time.Hour + case granularityDaily: + // Check if next calendar day (not exactly 24 hours) + y1, m1, d1 := t1.Date() + y2, m2, d2 := t2.Date() + // Add one day to t1 and check if it matches t2's date + nextDay := time.Date(y1, m1, d1+1, 0, 0, 0, 0, t1.Location()) + yn, mn, dn := nextDay.Date() + return y2 == yn && m2 == mn && d2 == dn + case granularityMonthly: + // Check if next month + y1, m1, _ := t1.Date() + y2, m2, _ := t2.Date() + expectedYear := y1 + expectedMonth := m1 + 1 + if expectedMonth > 12 { + expectedMonth = 1 + expectedYear++ + } + return y2 == expectedYear && m2 == expectedMonth + } + return false +} + +// formatTime formats a time based on granularity +func formatTime(t time.Time, gran granularity) string { + switch gran { + case granularityMonthly: + return t.Format("2006/01") + case granularityDaily: + return t.Format("2006/01/02") + case granularityHourly: + return t.Format("2006/01/02T15") + } + return t.Format("2006/01/02T15") +} + // PrintDates takes a slice of times and displays the range of times in a more friendly format. +// It automatically detects the granularity (hourly/daily/monthly) and formats accordingly. +// Examples: +// - Hourly: "2006/01/02T15-2006/01/02T18" +// - Daily: "2006/01/02-2006/01/05" +// - Monthly: "2006/01-2006/04" +// - Mixed: "2006/01-2006/03, 2006/05/01T10" func PrintDates(dates []time.Time) string { - tFormat := "2006/01/02T15" if len(dates) == 0 { return "" } + + // Sort dates sort.Slice(dates, func(i, j int) bool { return dates[i].Before(dates[j]) }) - prev := dates[0] - s := prev.Format(tFormat) - series := false - for _, t := range dates { - diff := t.Truncate(time.Hour).Sub(prev.Truncate(time.Hour)) - if diff != time.Hour && diff != 0 { - if series { - s += "-" + prev.Format(tFormat) - } - s += "," + t.Format(tFormat) - series = false - } else if diff == time.Hour { - series = true - } - prev = t - } - if series { - s += "-" + prev.Format(tFormat) + + // Single timestamp - return full hour format + if len(dates) == 1 { + return dates[0].Format("2006/01/02T15") } - //check for daily records only - if !strings.Contains(s, "-") { - days := strings.Split(s, ",") - prev, _ := time.Parse(tFormat, days[0]) - dailyString := prev.Format("2006/01/02") - series = false - - for i := 1; i < len(days); i++ { - tm, _ := time.Parse(tFormat, days[i]) - if r := tm.Sub(prev) % (24 * time.Hour); r != 0 { - return s - } - if tm.Sub(prev) != 24*time.Hour { - if series { - dailyString += "-" + prev.Format("2006/01/02") - series = false - } - dailyString += "," + tm.Format("2006/01/02") + // Detect granularity in a single pass (skip duplicates) + monthMap := make(map[string]bool) + dayMap := make(map[string]bool) + gran := granularityMonthly // Start optimistic, downgrade as needed + + for i, t := range dates { + // Skip duplicates for granularity detection + if i > 0 && t.Equal(dates[i-1]) { + continue + } + + // Detect granularity while iterating + monthKey := t.Format("2006-01") + dayKey := t.Format("2006-01-02") + + // If we've seen this month before, it's not monthly data + if monthMap[monthKey] && gran == granularityMonthly { + gran = granularityDaily + } + // If we've seen this day before, it's hourly data + if dayMap[dayKey] && gran == granularityDaily { + gran = granularityHourly + } + + monthMap[monthKey] = true + dayMap[dayKey] = true + } + // Build output + var result strings.Builder + rangeStart := dates[0] + prev := dates[0] + inRange := false + + for i := 1; i < len(dates); i++ { + curr := dates[i] + + if isConsecutive(prev, curr, gran) { + // Continue range + inRange = true + prev = curr + continue + } + + // Range broken or gap - write the previous range/item + if inRange { + // Close the range (but check if it's just duplicates) + if rangeStart.Equal(prev) { + // Just duplicates, write as single item + result.WriteString(rangeStart.Format("2006/01/02T15")) } else { - series = true + result.WriteString(formatTime(rangeStart, gran)) + result.WriteString("-") + result.WriteString(formatTime(prev, gran)) } - prev = tm + } else { + // Single item in a multi-timestamp dataset - use hour format + result.WriteString(rangeStart.Format("2006/01/02T15")) } - if series { - return dailyString + "-" + prev.Format("2006/01/02") + result.WriteString(",") + + // Start new range + rangeStart = curr + inRange = false + prev = curr + } + + // Handle the last item/range + if inRange { + // Close final range (but check if it's just duplicates) + if rangeStart.Equal(prev) { + // Just duplicates, write as single item + result.WriteString(rangeStart.Format("2006/01/02T15")) + } else { + result.WriteString(formatTime(rangeStart, gran)) + result.WriteString("-") + result.WriteString(formatTime(prev, gran)) } - return dailyString + return result.String() } - return s + + // Single final item in multi-timestamp dataset - use hour format + result.WriteString(rangeStart.Format("2006/01/02T15")) + return result.String() } diff --git a/tmpl/tmpl_test.go b/tmpl/tmpl_test.go index 8abd7661..3d9fb34b 100644 --- a/tmpl/tmpl_test.go +++ b/tmpl/tmpl_test.go @@ -382,10 +382,53 @@ func TestPrintDates(t *testing.T) { Input: trial.Times(f, "2018/04/09T00", "2018/04/10T00", "2018/04/11T00", "2018/04/12T00"), Expected: "2018/04/09-2018/04/12", }, + "daily records offset": + { + Input: trial.Times(f, "2018/04/09T04", "2018/04/10T04", "2018/04/11T04", "2018/04/12T06"), + Expected: "2018/04/09-2018/04/12", + }, + "daily records offset with duplicates": + { + Input: trial.Times(f, "2018/04/09T04","2018/04/09T04", "2018/04/09T04", "2018/04/10T04", "2018/04/11T04", "2018/04/12T06"), + Expected: "2018/04/09-2018/04/12", + }, "daily records with gaps": { Input: trial.Times(f, "2018/04/09T00", "2018/04/10T00", "2018/04/11T00", "2018/04/12T00", "2018/04/15T00", "2018/04/16T00", "2018/04/17T00"), Expected: "2018/04/09-2018/04/12,2018/04/15-2018/04/17", }, + + "monthly consecutive": { + Input: trial.Times(f, "2018/01/15T10", "2018/02/20T11", "2018/03/10T09"), + Expected: "2018/01-2018/03", + }, + "monthly with gaps": { + Input: trial.Times(f, "2018/01/15T10", "2018/02/20T11", "2018/05/10T09", "2018/06/15T12"), + Expected: "2018/01-2018/02,2018/05-2018/06", + }, + "monthly single": { + Input: trial.Times(f, "2018/01/15T10", "2018/03/20T11", "2018/08/10T09"), + Expected: "2018/01/15T10,2018/03/20T11,2018/08/10T09", + }, + "daily with different hours": { + Input: trial.Times(f, "2020/05/01T08", "2020/05/02T10", "2020/05/03T09", "2020/05/04T11"), + Expected: "2020/05/01-2020/05/04", + }, + "single timestamp": { + Input: trial.Times(f, "2020/05/01T10"), + Expected: "2020/05/01T10", + }, + "empty input": { + Input: []time.Time{}, + Expected: "", + }, + "cross year monthly": { + Input: trial.Times(f, "2020/11/15T10", "2020/12/20T11", "2021/01/10T09"), + Expected: "2020/11-2021/01", + }, + "cross month daily": { + Input: trial.Times(f, "2020/01/30T10", "2020/01/31T10", "2020/02/01T10", "2020/02/02T10"), + Expected: "2020/01/30-2020/02/02", + }, } trial.New(fn, cases).Test(t) diff --git a/workflow/workflow.go b/workflow/workflow.go index 333c630a..826d3e61 100644 --- a/workflow/workflow.go +++ b/workflow/workflow.go @@ -62,6 +62,11 @@ type Cache struct { Workflows map[string]Workflow // the key is the filename for the workflow } +// IsDir returns true if the original workflow path is a folder rather than a file +func (c *Cache) IsDir() bool { + return c.isDir +} + // New returns a Cache used to manage auto updating a workflow func New(path string, opts *file.Options) (*Cache, error) { c := &Cache{