From f3d81fc294fde24c220880aa1a0afb47272d9af7 Mon Sep 17 00:00:00 2001 From: Juergen Kellerer Date: Sun, 12 Nov 2023 00:57:42 +0100 Subject: [PATCH 1/3] base-profiles: supports profiles with limited/no command usage --- commands.go | 44 +++++++++++++++++++++++ commands_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++ config/global.go | 2 ++ config/profile.go | 2 ++ constants/default.go | 1 + main.go | 9 ++--- util/matcher.go | 39 ++++++++++++++++++++ util/matcher_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 util/matcher.go create mode 100644 util/matcher_test.go diff --git a/commands.go b/commands.go index 8cf5adce7..59fb6a8eb 100644 --- a/commands.go +++ b/commands.go @@ -22,6 +22,7 @@ import ( "github.com/creativeprojects/resticprofile/restic" "github.com/creativeprojects/resticprofile/schedule" "github.com/creativeprojects/resticprofile/term" + "github.com/creativeprojects/resticprofile/util" "github.com/creativeprojects/resticprofile/util/templates" "github.com/creativeprojects/resticprofile/win" ) @@ -707,3 +708,46 @@ func elevated(flags commandLineFlags) error { return nil } + +func commandFilter(profile *config.Profile, global *config.Global) func(command string) bool { + var baseProfile, allowed, denied util.MultiPatternMatcher + + if global == nil || global.BaseProfiles == nil { + baseProfile = util.GlobMultiMatcher(constants.DefaultBaseProfile) + } else { + baseProfile = util.GlobMultiMatcher(global.BaseProfiles...) + } + + if profile != nil { + if match, _ := baseProfile(profile.Name); match { + allowed = util.GlobMultiMatcher() + denied = util.GlobMultiMatcher("*") + } else { + if len(profile.AllowedCommands) == 1 && profile.AllowedCommands[0] == "" { + profile.AllowedCommands = nil + } + allowed = util.GlobMultiMatcher(profile.AllowedCommands...) + denied = util.GlobMultiMatcher(profile.DeniedCommands...) + } + } + + if allowed != nil && denied != nil { + return func(command string) bool { + if match, pattern := allowed(command); match { + if pattern != command { + if match, pattern = denied(command); match && pattern == command { + return false // allowed by wildcard and denied by direct match + } + } + return true + } + if match, _ := denied(command); match { + return false + } + // default true if no allowed commands are set + return len(profile.AllowedCommands) == 0 + } + } + + return func(command string) bool { return true } +} diff --git a/commands_test.go b/commands_test.go index 59136611f..0a4db0755 100644 --- a/commands_test.go +++ b/commands_test.go @@ -272,6 +272,89 @@ func TestGenerateCommand(t *testing.T) { }) } +func TestCommandFilter(t *testing.T) { + p, g := new(config.Profile), new(config.Global) + + reset := func() { p.AllowedCommands, p.DeniedCommands, g.BaseProfiles = nil, nil, nil } + + expect := func(t *testing.T, expected bool, command string) { + assert.Equal(t, expected, commandFilter(p, g)(command), "command %s", command) + } + + t.Run("nil-tolerant", func(t *testing.T) { + assert.True(t, commandFilter(nil, nil)("backup")) + }) + + t.Run("default-base", func(t *testing.T) { + reset() + p.Name = "default" + assert.True(t, commandFilter(p, nil)("backup")) + expect(t, true, "backup") + p.Name = "__default" + assert.False(t, commandFilter(p, nil)("backup")) + expect(t, false, "backup") + }) + + t.Run("configured-base", func(t *testing.T) { + reset() + p.Name = "default" + g.BaseProfiles = []string{"*"} + expect(t, false, "backup") + g.BaseProfiles = []string{"default"} + expect(t, false, "backup") + p.Name = "other" + expect(t, true, "backup") + }) + + t.Run("default-all-allowed", func(t *testing.T) { + reset() + for run := 0; run < 3; run++ { + expect(t, true, "backup") + expect(t, true, "restore") + expect(t, true, "another") + + switch run { + case 0: + p.AllowedCommands, p.DeniedCommands = []string{}, []string{} + case 1: + p.AllowedCommands, p.DeniedCommands = []string{""}, []string{""} + } + } + }) + + t.Run("allowed", func(t *testing.T) { + reset() + p.AllowedCommands = []string{"*"} + expect(t, true, "backup") + expect(t, true, "restore") + p.AllowedCommands = []string{"backup"} + expect(t, true, "backup") + expect(t, false, "restore") + }) + + t.Run("denied", func(t *testing.T) { + reset() + p.DeniedCommands = []string{"*"} + expect(t, false, "backup") + expect(t, false, "restore") + p.DeniedCommands = []string{"backup"} + expect(t, false, "backup") + expect(t, true, "restore") + expect(t, true, "another") + }) + + t.Run("allowed-is-not-denied", func(t *testing.T) { + reset() + p.AllowedCommands = []string{"backup", "restore", "repair*"} + p.DeniedCommands = []string{"backup", "repair-snapshot"} + expect(t, true, "backup") + expect(t, true, "restore") + expect(t, true, "repair-index") + expect(t, false, "repair-snapshot") // direct denied match, wildcard allowed results in denied + expect(t, false, "another") + }) +} + func TestShowSchedules(t *testing.T) { buffer := &bytes.Buffer{} schedules := []*config.Schedule{ diff --git a/config/global.go b/config/global.go index 7e8b4db5f..cf995f45a 100644 --- a/config/global.go +++ b/config/global.go @@ -14,6 +14,7 @@ type Global struct { Nice int `mapstructure:"nice" default:"0" range:"[-20:19]" description:"Sets the unix \"nice\" value for resticprofile and child processes (on any OS)"` Priority string `mapstructure:"priority" default:"normal" enum:"idle;background;low;normal;high;highest" description:"Sets process priority class for resticprofile and child processes (on any OS)"` DefaultCommand string `mapstructure:"default-command" default:"snapshots" description:"The restic or resticprofile command to use when no command was specified"` + BaseProfiles []string `mapstructure:"base-profiles" default:"__*" description:"One or more glob expression matching names of profiles that are meant for inheritance only. Profile names matching these expression cannot be executed directly - see https://creativeprojects.github.io/resticprofile/configuration/inheritance/"` Initialize bool `mapstructure:"initialize" default:"false" description:"Initialize a repository if missing"` ResticBinary string `mapstructure:"restic-binary" description:"Full path of the restic executable (detected if not set)"` ResticVersion string // not configurable at the moment. To be set after ResticBinary is known. @@ -40,6 +41,7 @@ func NewGlobal() *Global { IONiceClass: constants.DefaultIONiceClass, Nice: constants.DefaultStandardNiceFlag, DefaultCommand: constants.DefaultCommand, + BaseProfiles: []string{constants.DefaultBaseProfile}, FilterResticFlags: constants.DefaultFilterResticFlags, ResticLockRetryAfter: constants.DefaultResticLockRetryAfter, ResticStaleLockAge: constants.DefaultResticStaleLockAge, diff --git a/config/profile.go b/config/profile.go index e9d4c7c41..3b718c3af 100644 --- a/config/profile.go +++ b/config/profile.go @@ -73,6 +73,8 @@ type Profile struct { resticVersion *semver.Version Name string Description string `mapstructure:"description" description:"Describes the profile"` + AllowedCommands []string `mapstructure:"allowed-commands" description:"A list of commands that may be started on this profile, defaults to empty (= all). Wildcards (glob expressions) are supported"` + DeniedCommands []string `mapstructure:"denied-commands" description:"A list of commands that cannot be started on this profile, defaults to empty (= none). What was explicitly allowed cannot be denied"` BaseDir string `mapstructure:"base-dir" description:"Sets the working directory for this profile. The profile will fail when the working directory cannot be changed. Leave empty to use the current directory instead"` Quiet bool `mapstructure:"quiet" argument:"quiet"` Verbose int `mapstructure:"verbose" argument:"verbose"` diff --git a/constants/default.go b/constants/default.go index a63bf3011..b44639b57 100644 --- a/constants/default.go +++ b/constants/default.go @@ -7,6 +7,7 @@ const ( DefaultConfigurationFile = "profiles" DefaultProfileName = "default" DefaultCommand = "snapshots" + DefaultBaseProfile = "__*" DefaultFilterResticFlags = true DefaultResticLockRetryAfter = 60 * time.Second DefaultResticStaleLockAge = 1 * time.Hour diff --git a/main.go b/main.go index 10c2ab5fd..d653aa83f 100644 --- a/main.go +++ b/main.go @@ -533,11 +533,12 @@ func runProfile(ctx *Context) error { wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, ctx.request.group, version, profile.PrometheusLabels))) } - err = wrapper.runProfile() - if err != nil { - return err + if accept := commandFilter(profile, global); accept(resticCommand) { + err = wrapper.runProfile() + } else { + err = fmt.Errorf("profile %q does not allow running %q", profile.Name, resticCommand) } - return nil + return err } func loadScheduledProfile(ctx *Context) error { diff --git a/util/matcher.go b/util/matcher.go new file mode 100644 index 000000000..3ea483c39 --- /dev/null +++ b/util/matcher.go @@ -0,0 +1,39 @@ +package util + +import ( + "path" + "slices" + "sync" + + "github.com/creativeprojects/clog" +) + +// MultiPatternMatcher is a matcher for values using multiple patterns. The first matched pattern is returned +type MultiPatternMatcher func(value string) (match bool, pattern string) + +// GlobMultiMatcher returns a function that matches path like values using a set of glob expressions +func GlobMultiMatcher(patterns ...string) MultiPatternMatcher { + patterns = slices.Clone(patterns) + slices.Sort(patterns) + slices.Compact(patterns) // unique + slices.SortFunc(patterns, func(a, b string) int { return len(a) - len(b) }) // smallest first + + once := sync.Once{} + + return func(value string) (match bool, pattern string) { + var err error + for _, pattern = range patterns { + match, err = path.Match(pattern, value) + if err != nil { + once.Do(func() { + clog.Warningf("glob matcher (first error is logged): failed matching with %s: %s", pattern, err.Error()) + }) + } + if match { + return + } + } + pattern = "" + return + } +} diff --git a/util/matcher_test.go b/util/matcher_test.go new file mode 100644 index 000000000..21e86d64b --- /dev/null +++ b/util/matcher_test.go @@ -0,0 +1,86 @@ +package util + +import ( + "fmt" + "math/rand" + "slices" + "testing" + + "github.com/creativeprojects/clog" + "github.com/stretchr/testify/assert" +) + +func TestGlobMultiMatcher(t *testing.T) { + t.Run("no-pattern-no-match", func(t *testing.T) { + match, ptn := GlobMultiMatcher()("any") + assert.False(t, match) + assert.Empty(t, ptn) + match, ptn = GlobMultiMatcher(nil...)("any") + assert.False(t, match) + assert.Empty(t, ptn) + }) + + t.Run("matches-glob", func(t *testing.T) { + tests := []struct { + pattern, value string + matches bool + }{ + {pattern: "", value: "", matches: true}, + {pattern: "*", value: "", matches: true}, + {pattern: "*", value: "value", matches: true}, + {pattern: "", value: "value", matches: false}, + {pattern: "[0-9]", value: "5", matches: true}, + {pattern: "[0-9]", value: "10", matches: false}, + {pattern: "direct", value: "direct", matches: true}, + {pattern: "prefix", value: "prefix-suffix", matches: false}, + {pattern: "suffix", value: "prefix-suffix", matches: false}, + {pattern: "prefix*", value: "prefix-suffix", matches: true}, + {pattern: "*suffix", value: "prefix-suffix", matches: true}, + } + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + match, ptn := GlobMultiMatcher(test.pattern)(test.value) + assert.Equal(t, match, test.matches, "%q / %q", test.pattern, test.value) + if match { + assert.Equal(t, test.pattern, ptn) + } else { + assert.Empty(t, ptn) + } + }) + } + }) + + t.Run("logs-first-error", func(t *testing.T) { + defaultLogger := clog.GetDefaultLogger() + defer clog.SetDefaultLogger(defaultLogger) + + log := clog.NewMemoryHandler() + clog.SetDefaultLogger(clog.NewLogger(log)) + + invalidMatcher := GlobMultiMatcher("longer[", "*[") + for run := 0; run < 10; run++ { + invalidMatcher("longer-value") + } + + assert.Equal(t, "glob matcher (first error is logged): failed matching with *[: syntax error in pattern", log.Pop()) + assert.True(t, log.Empty()) + }) + + t.Run("matches-shortest-first", func(t *testing.T) { + patterns := []string{"-----*", "----*", "---*", "--*", "-*"} + shuffle := func() { + rand.Shuffle(len(patterns), func(i, j int) { patterns[i], patterns[j] = patterns[j], patterns[i] }) + } + for run := 0; run < 1000; run++ { + shuffle() + input := slices.Clone(patterns) + matcher := GlobMultiMatcher(input...) + for _, pattern := range patterns { + match, ptn := matcher(pattern) + assert.True(t, match) + assert.Equal(t, "-*", ptn) + } + assert.Equal(t, input, patterns, "input must not change") + } + }) +} From 54bdd2cb62a150d410a74a47b51e9daff518f4a3 Mon Sep 17 00:00:00 2001 From: Juergen Kellerer Date: Sun, 12 Nov 2023 14:50:48 +0100 Subject: [PATCH 2/3] base-profiles: rebased on master --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index d653aa83f..6518f2972 100644 --- a/main.go +++ b/main.go @@ -533,10 +533,10 @@ func runProfile(ctx *Context) error { wrapper.addProgress(prom.NewProgress(profile, prom.NewMetrics(profile.Name, ctx.request.group, version, profile.PrometheusLabels))) } - if accept := commandFilter(profile, global); accept(resticCommand) { + if accept := commandFilter(profile, ctx.global); accept(ctx.request.command) { err = wrapper.runProfile() } else { - err = fmt.Errorf("profile %q does not allow running %q", profile.Name, resticCommand) + err = fmt.Errorf("profile %q does not allow running %q", ctx.request.profile, ctx.request.command) } return err } From fd342c87cc29abe5b46ecd4c667ca558a1ce9c54 Mon Sep 17 00:00:00 2001 From: Juergen Kellerer Date: Sat, 24 Feb 2024 18:46:31 +0100 Subject: [PATCH 3/3] base-profiles: implemented filters for run-schedule (v1 & v2) --- commands.go | 207 ++++++++++++++++++++++++++++++++----------- commands_test.go | 57 +++++++----- config/config.go | 6 +- context.go | 12 +++ main.go | 39 +++++++- util/matcher.go | 18 ++++ util/matcher_test.go | 8 ++ 7 files changed, 265 insertions(+), 82 deletions(-) diff --git a/commands.go b/commands.go index 59fb6a8eb..5ee95b677 100644 --- a/commands.go +++ b/commands.go @@ -23,6 +23,7 @@ import ( "github.com/creativeprojects/resticprofile/schedule" "github.com/creativeprojects/resticprofile/term" "github.com/creativeprojects/resticprofile/util" + "github.com/creativeprojects/resticprofile/util/collect" "github.com/creativeprojects/resticprofile/util/templates" "github.com/creativeprojects/resticprofile/win" ) @@ -428,11 +429,9 @@ func createSchedule(_ io.Writer, ctx commandContext) error { // Step 1: Collect all jobs of all selected profiles for _, profileName := range selectProfiles(c, flags, args) { - profileFlags := flagsForProfile(flags, profileName) - - scheduler, profile, jobs, err := getScheduleJobs(c, profileFlags) + scheduler, profile, jobs, err := getSchedulesForProfile(c, profileName) if err == nil { - err = requireScheduleJobs(jobs, profileFlags) + err = requireSchedules(jobs, profileName) // Skip profile with no schedules when "--all" option is set. if err != nil && slices.Contains(args, "--all") { @@ -473,9 +472,7 @@ func removeSchedule(_ io.Writer, ctx commandContext) error { // Unschedule all jobs of all selected profiles for _, profileName := range selectProfiles(c, flags, args) { - profileFlags := flagsForProfile(flags, profileName) - - scheduler, _, jobs, err := getRemovableScheduleJobs(c, profileFlags) + scheduler, _, jobs, err := getRemovableSchedulesForProfile(c, profileName) if err != nil { return err } @@ -498,12 +495,13 @@ func statusSchedule(w io.Writer, ctx commandContext) error { if !slices.Contains(args, "--all") { // simple case of displaying status for one profile - scheduler, profile, schedules, err := getScheduleJobs(c, flags) + profileName := flags.name + scheduler, profile, schedules, err := getSchedulesForProfile(c, profileName) if err != nil { return err } if len(schedules) == 0 { - clog.Warningf("profile %s has no schedule", flags.name) + clog.Warningf("profile %s has no schedule", profileName) return nil } return statusScheduleProfile(scheduler, profile, schedules, flags) @@ -511,7 +509,7 @@ func statusSchedule(w io.Writer, ctx commandContext) error { for _, profileName := range selectProfiles(c, flags, args) { profileFlags := flagsForProfile(flags, profileName) - scheduler, profile, schedules, err := getScheduleJobs(c, profileFlags) + scheduler, profile, schedules, err := getSchedulesForProfile(c, profileName) if err != nil { return err } @@ -539,32 +537,118 @@ func statusScheduleProfile(scheduler schedule.SchedulerConfig, profile *config.P return nil } -func getScheduleJobs(c *config.Config, flags commandLineFlags) (schedule.SchedulerConfig, *config.Profile, []*config.Schedule, error) { +// getProfilesForSchedule returns all profiles for the given V2 standalone schedule +func getProfilesForSchedule(c *config.Config, s *config.Schedule) (scheduler schedule.SchedulerConfig, profiles []*config.Profile, err error) { + var global *config.Global + if global, err = c.GetGlobalSection(); err != nil { + err = fmt.Errorf("cannot load global section: %w", err) + return + } + + // resolve profile names + if len(s.Group) > 0 && len(s.Profiles) == 0 { + var group *config.Group + if group, err = c.GetProfileGroup(s.Group); err != nil { + err = fmt.Errorf("cannot load group section %q: %w", s.Group, err) + return + } + s.Profiles = slices.Clone(group.Profiles) + } + + // resolve glob expressions in the profiles list, e.g. use "*" to run for all profiles that define a command + hasGlob := func(name string) bool { return strings.ContainsAny(name, "?*[]") } + if slices.ContainsFunc(s.Profiles, hasGlob) { + // glob matcher for expressions in s.Profiles, excluding literal matches in s.Profiles + globMatches := util.GlobMultiMatcher(s.Profiles...).NoLiteralMatchCondition() + // remove glob expressions from s.Profiles + s.Profiles = collect.All(s.Profiles, collect.Not(hasGlob)) + // add newly matched names at the end to preserve declaration order + for _, name := range collect.All(c.GetProfileNames(), globMatches) { + if !slices.Contains(s.Profiles, name) { + s.Profiles = append(s.Profiles, name) + } + } + } + + // collect profiles + profiles = collect.From(s.Profiles, func(name string) (profile *config.Profile) { + if err == nil { + profile, err = c.GetProfile(name) + } + return + }) + if err != nil { + err = fmt.Errorf("cannot load profiles ['%s']: %w", strings.Join(s.Profiles, "', '"), err) + return + } + + // remove profiles that cannot be used + profiles = validProfilesForSchedule(profiles, global, s) + scheduler = schedule.NewSchedulerConfig(global) + return +} + +// validProfilesForSchedule returns all profiles that the given schedule can execute +func validProfilesForSchedule(profiles []*config.Profile, global *config.Global, schedule *config.Schedule) []*config.Profile { + // used in v2 schedules to find the profiles that can be executed + return collect.All(profiles, func(profile *config.Profile) bool { + return validScheduleFilter(profile, global)(schedule) + }) +} + +// validSchedulesFilter creates a filter func that accepts config.Schedule when it is allowed for the given profile +func validScheduleFilter(profile *config.Profile, global *config.Global) func(schedule *config.Schedule) (valid bool) { + acceptCommand := commandFilter(profile, global) + definedCommands := profile.DefinedCommands() + + return func(schedule *config.Schedule) (accepted bool) { + if profile != nil && schedule != nil { + if accepted = slices.Contains(schedule.Profiles, profile.Name); !accepted { + clog.Debugf("not in schedule: profile '%s' has no schedule for %q", profile.Name, schedule.CommandName) + return + } + if accepted = slices.Contains(definedCommands, schedule.CommandName); !accepted { + clog.Debugf("undefined command: cannot schedule %q for profile '%s'", schedule.CommandName, profile.Name) + return + } + if accepted = acceptCommand(schedule.CommandName); !accepted { + clog.Debugf("disallowed command: cannot schedule %q for profile '%s'", schedule.CommandName, profile.Name) + } + } + return + } +} + +// getSchedulesForProfile returns the profile and its inline schedules for a given profile name +func getSchedulesForProfile(c *config.Config, profileName string) (schedule.SchedulerConfig, *config.Profile, []*config.Schedule, error) { global, err := c.GetGlobalSection() if err != nil { return nil, nil, nil, fmt.Errorf("cannot load global section: %w", err) } - profile, err := c.GetProfile(flags.name) + profile, err := c.GetProfile(profileName) if err != nil { if errors.Is(err, config.ErrNotFound) { - return nil, nil, nil, fmt.Errorf("profile '%s' not found", flags.name) + return nil, nil, nil, fmt.Errorf("profile '%s' not found: %w", profileName, err) } - return nil, nil, nil, fmt.Errorf("cannot load profile '%s': %w", flags.name, err) + return nil, nil, nil, fmt.Errorf("cannot load profile '%s': %w", profileName, err) } - return schedule.NewSchedulerConfig(global), profile, profile.Schedules(), nil + // collecting schedules that can be run on this profile + schedules := collect.All(profile.Schedules(), validScheduleFilter(profile, global)) + + return schedule.NewSchedulerConfig(global), profile, schedules, nil } -func requireScheduleJobs(schedules []*config.Schedule, flags commandLineFlags) error { +func requireSchedules(schedules []*config.Schedule, profileName string) error { if len(schedules) == 0 { - return fmt.Errorf("no schedule found for profile '%s'", flags.name) + return fmt.Errorf("no schedule found for profile '%s'", profileName) } return nil } -func getRemovableScheduleJobs(c *config.Config, flags commandLineFlags) (schedule.SchedulerConfig, *config.Profile, []*config.Schedule, error) { - scheduler, profile, schedules, err := getScheduleJobs(c, flags) +func getRemovableSchedulesForProfile(c *config.Config, profileName string) (schedule.SchedulerConfig, *config.Profile, []*config.Schedule, error) { + scheduler, profile, schedules, err := getSchedulesForProfile(c, profileName) if err != nil { return nil, nil, nil, err } @@ -585,45 +669,67 @@ func getRemovableScheduleJobs(c *config.Config, flags commandLineFlags) (schedul return scheduler, profile, schedules, nil } -func preRunSchedule(ctx *Context) error { +func preRunSchedule(ctx *Context) (err error) { if len(ctx.request.arguments) < 1 { return errors.New("run-schedule command expects one argument: schedule name") } + + // extract scheduleName and remove the parameter from the arguments scheduleName := ctx.request.arguments[0] - // temporarily allow v2 configuration to run v1 schedules - // if ctx.config.GetVersion() < config.Version02 - { - // schedule name is in the form "command@profile" - commandName, profileName, ok := strings.Cut(scheduleName, "@") - if !ok { - return errors.New("the expected format of the schedule name is @") - } + ctx.request.arguments = ctx.request.arguments[1:] + ctx.schedule = nil + + if commandName, profileName, ok := strings.Cut(scheduleName, "@"); ok { + // Inline schedules use a name in the form "command@profile" ctx.request.profile = profileName - ctx.request.schedule = scheduleName ctx.command = commandName - // remove the parameter from the arguments - ctx.request.arguments = ctx.request.arguments[1:] - - // don't save the profile in the context now, it's only loaded but not prepared - profile, err := ctx.config.GetProfile(profileName) - if err != nil || profile == nil { - return fmt.Errorf("cannot load profile '%s': %w", profileName, err) - } - // get the list of all scheduled commands to find the current command - schedules := profile.Schedules() - for _, schedule := range schedules { - if schedule.CommandName == ctx.command { - ctx.schedule = schedule - prepareScheduledProfile(ctx) - break + + // find the config.Schedule for the command in the profile + var schedules []*config.Schedule + if _, _, schedules, err = getSchedulesForProfile(ctx.config, profileName); err == nil { + matchesCommand := func(s *config.Schedule) bool { return s.CommandName == commandName } + if s := collect.First(schedules, matchesCommand); s != nil { + ctx.schedule = *s + } + } + } else { + // Standalone V2 schedules use a name that references the schedule section + var ( + schedule *config.Schedule + schedules map[string]*config.Schedule + profiles []*config.Profile + ) + if schedules, err = ctx.config.GetScheduleSections(); err == nil { + schedule = schedules[scheduleName] + } + if schedule != nil { + if _, profiles, err = getProfilesForSchedule(ctx.config, schedule); err == nil { + names := schedule.Profiles + schedule.Profiles = collect.From(profiles, func(p *config.Profile) string { return p.Name }) + if len(schedule.Profiles) > 0 { + ctx.schedule = schedule + } else { + err = fmt.Errorf("none of the profiles ['%s'] in schedule %q can be used", strings.Join(names, "', '"), scheduleName) + } } } } - return nil + + if ctx.schedule != nil { + ctx.request.schedule = scheduleName + prepareContextForSchedule(ctx) + } else if err == nil { + err = fmt.Errorf("schedule %q not found, the expected format of the schedule name is @ or ", scheduleName) + } + return } -func prepareScheduledProfile(ctx *Context) { - clog.Debugf("preparing scheduled profile %q", ctx.request.schedule) +func prepareContextForSchedule(ctx *Context) { + clog.Debugf("preparing schedule %q", ctx.request.schedule) + // requested profile + ctx.request.profile = ctx.schedule.Profiles[0] + // schedule command + ctx.command = ctx.schedule.CommandName // log file if len(ctx.schedule.Log) > 0 { ctx.logTarget = ctx.schedule.Log @@ -648,11 +754,10 @@ func prepareScheduledProfile(ctx *Context) { } func runSchedule(_ io.Writer, cmdCtx commandContext) error { - err := startProfileOrGroup(&cmdCtx.Context) - if err != nil { - return err + if cmdCtx.schedule == nil { + return fmt.Errorf("invalid state: schedule %q not initialized", cmdCtx.request.schedule) } - return nil + return startContext(&cmdCtx.Context) } func testElevationCommand(_ io.Writer, ctx commandContext) error { diff --git a/commands_test.go b/commands_test.go index 0a4db0755..b4743dc28 100644 --- a/commands_test.go +++ b/commands_test.go @@ -50,12 +50,13 @@ schedule = "daily" parsedConfig, err := config.Load(bytes.NewBufferString(testConfig), "toml") assert.Nil(t, err) - // Test that errors from getScheduleJobs are passed through - _, _, _, notFoundErr := getRemovableScheduleJobs(parsedConfig, commandLineFlags{name: "non-existent"}) - assert.EqualError(t, notFoundErr, "profile 'non-existent' not found") + // Test that errors from getSchedulesForProfile are passed through + _, _, _, notFoundErr := getRemovableSchedulesForProfile(parsedConfig, "non-existent") + assert.ErrorContains(t, notFoundErr, "profile 'non-existent' not found") + assert.ErrorIs(t, notFoundErr, config.ErrNotFound) // Test that declared and declarable job configs are returned - _, profile, schedules, err := getRemovableScheduleJobs(parsedConfig, commandLineFlags{name: "default"}) + _, profile, schedules, err := getRemovableSchedulesForProfile(parsedConfig, "default") assert.Nil(t, err) assert.NotNil(t, profile) assert.NotEmpty(t, schedules) @@ -89,26 +90,27 @@ schedule = "daily" assert.Nil(t, err) // Test that non-existent profiles causes an error - _, _, _, notFoundErr := getScheduleJobs(cfg, commandLineFlags{name: "non-existent"}) - assert.EqualError(t, notFoundErr, "profile 'non-existent' not found") + _, _, _, notFoundErr := getSchedulesForProfile(cfg, "non-existent") + assert.ErrorContains(t, notFoundErr, "profile 'non-existent' not found") + assert.ErrorIs(t, notFoundErr, config.ErrNotFound) // Test that non-existent schedule causes no error at first { - flags := commandLineFlags{name: "other"} - _, _, schedules, err := getScheduleJobs(cfg, flags) + name := "other" + _, _, schedules, err := getSchedulesForProfile(cfg, name) assert.Nil(t, err) - err = requireScheduleJobs(schedules, flags) + err = requireSchedules(schedules, name) assert.EqualError(t, err, "no schedule found for profile 'other'") } // Test that only declared job configs are returned { - flags := commandLineFlags{name: "default"} - _, profile, schedules, err := getScheduleJobs(cfg, flags) + name := "default" + _, profile, schedules, err := getSchedulesForProfile(cfg, name) assert.Nil(t, err) - err = requireScheduleJobs(schedules, flags) + err = requireSchedules(schedules, name) assert.Nil(t, err) assert.NotNil(t, profile) @@ -436,19 +438,26 @@ func TestPreRunScheduleNoScheduleName(t *testing.T) { } func TestPreRunScheduleWrongScheduleName(t *testing.T) { + runForConfig := func(t *testing.T, cfg, name string) error { + c, err := config.Load(bytes.NewBufferString(cfg), "toml") + assert.NoError(t, err) + + err = preRunSchedule(&Context{ + request: Request{arguments: []string{name}}, + config: c, + flags: commandLineFlags{name: "default"}, + }) + t.Log(err) + return err + } + // loads an (almost) empty config - cfg, err := config.Load(bytes.NewBufferString("[default]"), "toml") - assert.NoError(t, err) + v1Config := "[default]" + v2Config := "version = 2\n[profiles.default]" - err = preRunSchedule(&Context{ - request: Request{arguments: []string{"wrong"}}, - config: cfg, - flags: commandLineFlags{ - name: "default", - }, - }) - assert.Error(t, err) - t.Log(err) + assert.ErrorContains(t, runForConfig(t, v1Config, "wrong@default"), `schedule "wrong@default" not found`) + assert.Panics(t, func() { _ = runForConfig(t, v1Config, "wrong") }) + assert.ErrorContains(t, runForConfig(t, v2Config, "wrong"), `schedule "wrong" not found`) } func TestPreRunScheduleProfileUnknown(t *testing.T) { @@ -509,5 +518,5 @@ func TestRunScheduleProfileUnknown(t *testing.T) { config: cfg, }, }) - assert.ErrorIs(t, err, ErrProfileNotFound) + assert.ErrorContains(t, err, `invalid state: schedule "" not initialized`) } diff --git a/config/config.go b/config/config.go index 677d373ca..5f97d7b5f 100644 --- a/config/config.go +++ b/config/config.go @@ -661,10 +661,10 @@ func (c *Config) GetSchedules() ([]*Schedule, error) { } // GetScheduleSections returns a list of schedules -func (c *Config) GetScheduleSections() (schedules map[string]Schedule, err error) { +func (c *Config) GetScheduleSections() (schedules map[string]*Schedule, err error) { c.requireMinVersion(Version02) - schedules = map[string]Schedule{} + schedules = map[string]*Schedule{} if section := c.viper.Sub(constants.SectionConfigurationSchedules); section != nil { for sectionKey := range section.AllSettings() { @@ -673,7 +673,7 @@ func (c *Config) GetScheduleSections() (schedules map[string]Schedule, err error if err != nil { break } - schedules[sectionKey] = schedule + schedules[sectionKey] = &schedule } } diff --git a/context.go b/context.go index b41bb1f12..77e06cc13 100644 --- a/context.go +++ b/context.go @@ -127,6 +127,18 @@ func (c *Context) WithProfile(profileName string) *Context { return newContext } +// WithSchedule sets the schedule and profile name of the current schedule run. A new copy of the context is returned. +// Profile information is not copied over, schedule is replaced with the specified value. +func (c *Context) WithSchedule(schedule *config.Schedule, profileName string) *Context { + newContext := c.clone() + newContext.request.profile = profileName + newContext.request.group = "" + newContext.profile = nil + newContext.schedule = schedule + newContext.command = schedule.CommandName + return newContext +} + func (c *Context) clone() *Context { clone := *c return &clone diff --git a/main.go b/main.go index 6518f2972..1eb2be6a8 100644 --- a/main.go +++ b/main.go @@ -253,7 +253,7 @@ func main() { ctx = ctx.WithCommand(ctx.request.command) // it wasn't an internal command so we run a profile - err = startProfileOrGroup(ctx) + err = startContext(ctx) if err != nil { clog.Error(err) if errors.Is(err, ErrProfileNotFound) { @@ -359,8 +359,31 @@ func detectResticBinary(global *config.Global) (string, error) { return resticBinary, nil } -func startProfileOrGroup(ctx *Context) error { - if ctx.config.HasProfile(ctx.request.profile) { +func startContext(ctx *Context) error { + if ctx.schedule != nil { + // if running as a systemd timer + notifyStart() + defer notifyStop() + + // Schedule run + var err error + profiles := ctx.schedule.Profiles + for i, name := range profiles { + clog.Debugf("[%d/%d] starting profile '%s' from schedule %q", i+1, len(profiles), name, ctx.request.schedule) + ctx = ctx.WithSchedule(ctx.schedule, name) + err = runProfile(ctx) + if err != nil { + if len(profiles) > 1 && canContinueGroupOnError(ctx, ctx.schedule.Group) { + clog.Error(err) + err = nil + } else { + break + } + } + } + return err + + } else if ctx.config.HasProfile(ctx.request.profile) { // if running as a systemd timer notifyStart() defer notifyStop() @@ -390,7 +413,7 @@ func startProfileOrGroup(ctx *Context) error { ctx = ctx.WithProfile(profileName).WithGroup(groupName) err = runProfile(ctx) if err != nil { - if ctx.global.GroupContinueOnError && group.ContinueOnError.IsTrueOrUndefined() { + if canContinueGroupOnError(ctx, groupName) { // keep going to the next profile clog.Error(err) continue @@ -407,6 +430,14 @@ func startProfileOrGroup(ctx *Context) error { return nil } +// canContinueGroupOnError returns true if global.GroupContinueOnError and it is not overridden in the specified group +// Note: a non-existing group is evaluated the same as a group that doesn't override the global value +func canContinueGroupOnError(ctx *Context, groupKey string) bool { + group, err := ctx.config.GetProfileGroup(groupKey) + return ctx.global.GroupContinueOnError && + (err != nil || group == nil || group.ContinueOnError.IsTrueOrUndefined()) +} + func openProfile(c *config.Config, profileName string) (profile *config.Profile, cleanup func(), err error) { done := false for attempts := 3; attempts > 0 && !done; attempts-- { diff --git a/util/matcher.go b/util/matcher.go index 3ea483c39..249bdb3f9 100644 --- a/util/matcher.go +++ b/util/matcher.go @@ -37,3 +37,21 @@ func GlobMultiMatcher(patterns ...string) MultiPatternMatcher { return } } + +// Condition returns a condition function that may be used with slices.ContainsFunc or collect.All. +// The function returns true as a pattern matches the value. +func (m MultiPatternMatcher) Condition() func(value string) (match bool) { + return func(value string) (match bool) { + match, _ = m(value) + return + } +} + +// NoLiteralMatchCondition returns a condition function that may be used with slices.ContainsFunc or collect.All. +// The function returns true as a pattern matches the value but does not match the pattern literally. +func (m MultiPatternMatcher) NoLiteralMatchCondition() func(value string) (match bool) { + return func(value string) (match bool) { + match, pattern := m(value) + return match && pattern != value // match only if the pattern didn't match itself + } +} diff --git a/util/matcher_test.go b/util/matcher_test.go index 21e86d64b..e2811b237 100644 --- a/util/matcher_test.go +++ b/util/matcher_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/creativeprojects/clog" + "github.com/creativeprojects/resticprofile/util/collect" "github.com/stretchr/testify/assert" ) @@ -83,4 +84,11 @@ func TestGlobMultiMatcher(t *testing.T) { assert.Equal(t, input, patterns, "input must not change") } }) + + t.Run("can-be-used-as-condition", func(t *testing.T) { + matcher := GlobMultiMatcher("a*", "xx", "z*") + values := []string{"aa", "ab", "ca", "xx", "xy", "z", "z*", "zz"} + assert.Equal(t, []string{"aa", "ab", "xx", "z", "z*", "zz"}, collect.All(values, matcher.Condition())) + assert.Equal(t, []string{"aa", "ab", "z", "zz"}, collect.All(values, matcher.NoLiteralMatchCondition())) + }) }