From 5e1e9493e66b823be8390d96c9e24c8d19fad6c1 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 12:55:01 +0100 Subject: [PATCH 01/24] simple client/server model --- commands.go | 18 +++++ complete.go | 3 +- config/checkdoc/main.go | 3 +- config/config.go | 22 ++++- config/config_test.go | 93 ++++++++++----------- config/error.go | 3 +- config/remote.go | 34 ++++++++ constants/exit_code.go | 10 +++ constants/other.go | 1 + constants/section.go | 1 + examples/linux.yaml | 11 ++- filesearch/filesearch.go | 3 +- filesearch/filesearch_test.go | 3 +- flags.go | 5 ++ fuse/file.go | 25 ++++++ fuse/fs_file.go | 40 +++++++++ fuse/memfs.go | 122 ++++++++++++++++++++++++++++ fuse/memfs_test.go | 77 ++++++++++++++++++ fuse/memfs_windows_test.go | 14 ++++ fuse/mount.go | 40 +++++++++ fuse/mount_windows.go | 7 ++ go.mod | 1 + go.sum | 2 + integration_test.go | 3 +- main.go | 33 +++++++- own_commands.go | 6 ++ remote.go | 137 +++++++++++++++++++++++++++++++ remote/manifest.go | 9 +++ remote/tar.go | 88 ++++++++++++++++++++ send.go | 79 ++++++++++++++++++ serve.go | 97 ++++++++++++++++++++++ ssh/config.go | 44 ++++++++++ ssh/ssh.go | 147 ++++++++++++++++++++++++++++++++++ 33 files changed, 1117 insertions(+), 64 deletions(-) create mode 100644 config/remote.go create mode 100644 constants/exit_code.go create mode 100644 fuse/file.go create mode 100644 fuse/fs_file.go create mode 100644 fuse/memfs.go create mode 100644 fuse/memfs_test.go create mode 100644 fuse/memfs_windows_test.go create mode 100644 fuse/mount.go create mode 100644 fuse/mount_windows.go create mode 100644 remote.go create mode 100644 remote/manifest.go create mode 100644 remote/tar.go create mode 100644 send.go create mode 100644 serve.go create mode 100644 ssh/config.go create mode 100644 ssh/ssh.go diff --git a/commands.go b/commands.go index 629d9c7aa..979ce9180 100644 --- a/commands.go +++ b/commands.go @@ -168,6 +168,24 @@ func getOwnCommands() []ownCommand { needConfiguration: false, hide: true, }, + { + name: "send", + description: "send a configuration profile to a remote client and execute a command", + action: sendProfileCommand, + needConfiguration: true, + noProfile: true, + hide: true, + experimental: true, + }, + { + name: "serve", + description: "serve configuration profiles to remote clients", + action: serveCommand, + needConfiguration: true, + noProfile: true, + hide: true, + experimental: true, + }, } } diff --git a/complete.go b/complete.go index c2ac27f10..318603a57 100644 --- a/complete.go +++ b/complete.go @@ -10,6 +10,7 @@ import ( "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/config" "github.com/creativeprojects/resticprofile/filesearch" + "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -135,7 +136,7 @@ func (c *Completer) listProfileNames() (list []string) { } if file, err := filesearch.NewFinder().FindConfigurationFile(filename); err == nil { - if conf, err := config.LoadFile(file, format); err == nil { + if conf, err := config.LoadFile(afero.NewOsFs(), file, format); err == nil { list = append(list, conf.GetProfileNames()...) for name := range conf.GetProfileGroups() { list = append(list, name) diff --git a/config/checkdoc/main.go b/config/checkdoc/main.go index 2e888187f..79a34de0c 100644 --- a/config/checkdoc/main.go +++ b/config/checkdoc/main.go @@ -11,6 +11,7 @@ import ( "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/config" + "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -174,7 +175,7 @@ func saveConfiguration(content []byte, configType string) (string, error) { // checkConfiguration returns true when the configuration is valid func checkConfiguration(filename, configType string, lineNum int) bool { - cfg, err := config.LoadFile(filename, configType) + cfg, err := config.LoadFile(afero.NewOsFs(), filename, configType) if err != nil { clog.Errorf(" %q on line %d: %s", configType, lineNum, err) return false diff --git a/config/config.go b/config/config.go index aa7a92f08..a99e6bfc4 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "maps" - "os" "path/filepath" "slices" "sort" @@ -21,6 +20,7 @@ import ( "github.com/creativeprojects/resticprofile/util/maybe" "github.com/creativeprojects/resticprofile/util/templates" "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" "github.com/spf13/viper" ) @@ -74,7 +74,7 @@ func formatFromExtension(configFile string) string { // LoadFile loads configuration from file // Leave format blank for auto-detection from the file extension -func LoadFile(configFile, format string) (config *Config, err error) { +func LoadFile(fs afero.Fs, configFile, format string) (config *Config, err error) { if format == "" { format = formatFromExtension(configFile) } @@ -84,7 +84,7 @@ func LoadFile(configFile, format string) (config *Config, err error) { readAndAdd := func(configFile string, replace bool) error { clog.Debugf("loading: %s", configFile) - file, fileErr := os.Open(configFile) + file, fileErr := fs.Open(configFile) if fileErr != nil { return fmt.Errorf("cannot open configuration file for reading: %w", fileErr) } @@ -716,6 +716,22 @@ func (c *Config) getProfilePath(key string) string { return c.flatKey(constants.SectionConfigurationProfiles, key) } +// HasRemote returns true if the remote exists in the configuration +func (c *Config) HasRemote(remoteName string) bool { + return c.IsSet(c.flatKey(constants.SectionConfigurationRemotes, remoteName)) +} + +func (c *Config) GetRemote(remoteName string) (*Remote, error) { + // we don't need to check the file version: the remotes can be in a separate configuration file + + remote := NewRemote(c, remoteName) + err := c.unmarshalKey(c.flatKey(constants.SectionConfigurationRemotes, remoteName), remote) + + rootPath := filepath.Dir(c.GetConfigFile()) + remote.SetRootPath(rootPath) + return remote, err +} + // unmarshalConfig returns the decoder config options depending on the configuration version and format func (c *Config) unmarshalConfig() viper.DecoderConfigOption { if c.GetVersion() == Version01 { diff --git a/config/config_test.go b/config/config_test.go index 4ac352ab0..675337303 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/creativeprojects/resticprofile/util/maybe" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -366,32 +367,22 @@ x=0 } func TestIncludes(t *testing.T) { - files := []string{} - cleanFiles := func() { - for _, file := range files { - os.Remove(file) - } - files = files[:0] - } - defer cleanFiles() - - createFile := func(t *testing.T, suffix, content string) string { + createFile := func(t *testing.T, fs afero.Fs, suffix, content string) string { t.Helper() name := "" - file, err := os.CreateTemp("", "*-"+suffix) + file, err := afero.TempFile(fs, "", "*-"+suffix) if err == nil { defer file.Close() _, err = file.WriteString(content) name = file.Name() - files = append(files, name) } require.NoError(t, err) return name } - mustLoadConfig := func(t *testing.T, configFile string) *Config { + mustLoadConfig := func(t *testing.T, fs afero.Fs, configFile string) *Config { t.Helper() - config, err := LoadFile(configFile, "") + config, err := LoadFile(fs, configFile, "") require.NoError(t, err) return config } @@ -399,15 +390,15 @@ func TestIncludes(t *testing.T) { testID := fmt.Sprintf("%d", time.Now().Unix()) t.Run("multiple-includes", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() content := fmt.Sprintf(`includes=['*%[1]s.inc.toml','*%[1]s.inc.yaml','*%[1]s.inc.json']`, testID) - configFile := createFile(t, "profiles.conf", content) - createFile(t, "d-"+testID+".inc.toml", "[one]\nk='v'") - createFile(t, "o-"+testID+".inc.yaml", `two: { k: v }`) - createFile(t, "j-"+testID+".inc.json", `{"three":{ "k": "v" }}`) + configFile := createFile(t, fs, "profiles.conf", content) + createFile(t, fs, "d-"+testID+".inc.toml", "[one]\nk='v'") + createFile(t, fs, "o-"+testID+".inc.yaml", `two: { k: v }`) + createFile(t, fs, "j-"+testID+".inc.json", `{"three":{ "k": "v" }}`) - config := mustLoadConfig(t, configFile) + config := mustLoadConfig(t, fs, configFile) assert.True(t, config.IsSet("includes")) assert.True(t, config.HasProfile("one")) assert.True(t, config.HasProfile("two")) @@ -415,18 +406,18 @@ func TestIncludes(t *testing.T) { }) t.Run("overrides", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() - configFile := createFile(t, "profiles.conf", ` + configFile := createFile(t, fs, "profiles.conf", ` includes = "*`+testID+`.inc.toml" [default] repository = "default-repo"`) - createFile(t, "override-"+testID+".inc.toml", ` + createFile(t, fs, "override-"+testID+".inc.toml", ` [default] repository = "overridden-repo"`) - config := mustLoadConfig(t, configFile) + config := mustLoadConfig(t, fs, configFile) assert.True(t, config.HasProfile("default")) profile, err := config.GetProfile("default") @@ -435,26 +426,26 @@ repository = "overridden-repo"`) }) t.Run("mixins", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() - configFile := createFile(t, "profiles.conf", ` + configFile := createFile(t, fs, "profiles.conf", ` version = 2 includes = "*`+testID+`.inc.toml" [profiles.default] use = "another-run-before" run-before = "default-before"`) - createFile(t, "mixin-"+testID+".inc.toml", ` + createFile(t, fs, "mixin-"+testID+".inc.toml", ` [mixins.another-run-before] "run-before..." = "another-run-before" [mixins.another-run-before2] "run-before..." = "another-run-before2"`) - createFile(t, "mixin-use-"+testID+".inc.toml", ` + createFile(t, fs, "mixin-use-"+testID+".inc.toml", ` [profiles.default] use = "another-run-before2"`) - config := mustLoadConfig(t, configFile) + config := mustLoadConfig(t, fs, configFile) assert.True(t, config.HasProfile("default")) profile, err := config.GetProfile("default") @@ -463,56 +454,56 @@ use = "another-run-before2"`) }) t.Run("hcl-includes-only-hcl", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() - configFile := createFile(t, "profiles.hcl", `includes = "*`+testID+`.inc.*"`) - createFile(t, "pass-"+testID+".inc.hcl", `one { }`) + configFile := createFile(t, fs, "profiles.hcl", `includes = "*`+testID+`.inc.*"`) + createFile(t, fs, "pass-"+testID+".inc.hcl", `one { }`) - config := mustLoadConfig(t, configFile) + config := mustLoadConfig(t, fs, configFile) assert.True(t, config.HasProfile("one")) - createFile(t, "fail-"+testID+".inc.toml", `[two]`) - _, err := LoadFile(configFile, "") + createFile(t, fs, "fail-"+testID+".inc.toml", `[two]`) + _, err := LoadFile(fs, configFile, "") assert.Error(t, err) assert.Regexp(t, ".+ is in hcl format, includes must use the same format", err.Error()) }) t.Run("non-hcl-include-no-hcl", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() - configFile := createFile(t, "profiles.toml", `includes = "*`+testID+`.inc.*"`) - createFile(t, "pass-"+testID+".inc.toml", "[one]\nk='v'") + configFile := createFile(t, fs, "profiles.toml", `includes = "*`+testID+`.inc.*"`) + createFile(t, fs, "pass-"+testID+".inc.toml", "[one]\nk='v'") - config := mustLoadConfig(t, configFile) + config := mustLoadConfig(t, fs, configFile) assert.True(t, config.HasProfile("one")) - createFile(t, "fail-"+testID+".inc.hcl", `one { }`) - _, err := LoadFile(configFile, "") + createFile(t, fs, "fail-"+testID+".inc.hcl", `one { }`) + _, err := LoadFile(fs, configFile, "") assert.Error(t, err) assert.Regexp(t, "hcl format .+ cannot be used in includes from toml", err.Error()) }) t.Run("cannot-load-different-versions", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() content := fmt.Sprintf(`includes=['*%s.inc.json']`, testID) - configFile := createFile(t, "profiles.conf", content) - createFile(t, "a-"+testID+".inc.json", `{"version": 2, "profiles": {"one":{}}}`) - createFile(t, "b-"+testID+".inc.json", `{"two":{}}`) + configFile := createFile(t, fs, "profiles.conf", content) + createFile(t, fs, "a-"+testID+".inc.json", `{"version": 2, "profiles": {"one":{}}}`) + createFile(t, fs, "b-"+testID+".inc.json", `{"two":{}}`) - _, err := LoadFile(configFile, "") + _, err := LoadFile(fs, configFile, "") assert.ErrorContains(t, err, "cannot include different versions of the configuration file") }) t.Run("cannot-load-different-versions", func(t *testing.T) { - defer cleanFiles() + fs := afero.NewMemMapFs() content := fmt.Sprintf(`{"version": 2, "includes":["*%s.inc.json"]}`, testID) - configFile := createFile(t, "profiles.json", content) - createFile(t, "c-"+testID+".inc.json", `{"version": 1, "two":{}}`) - createFile(t, "d-"+testID+".inc.json", `{"profiles": {"one":{}}}`) + configFile := createFile(t, fs, "profiles.json", content) + createFile(t, fs, "c-"+testID+".inc.json", `{"version": 1, "two":{}}`) + createFile(t, fs, "d-"+testID+".inc.json", `{"profiles": {"one":{}}}`) - _, err := LoadFile(configFile, "") + _, err := LoadFile(fs, configFile, "") assert.ErrorContains(t, err, "cannot include different versions of the configuration file") }) } diff --git a/config/error.go b/config/error.go index eb85ce37b..394c88c01 100644 --- a/config/error.go +++ b/config/error.go @@ -3,5 +3,6 @@ package config import "errors" var ( - ErrNotFound = errors.New("not found") + ErrNotFound = errors.New("not found") + ErrNotSupportedInVersion1 = errors.New("not supported in configuration version 1") ) diff --git a/config/remote.go b/config/remote.go new file mode 100644 index 000000000..dce0e5def --- /dev/null +++ b/config/remote.go @@ -0,0 +1,34 @@ +package config + +type Remote struct { + name string + config *Config + Connection string `mapstructure:"connection" default:"ssh" description:"Connection type to use to connect to the remote client"` + Host string `mapstructure:"host" description:"Address of the remote client. Format: :"` + Username string `mapstructure:"username" description:"User to connect to the remote client"` + PrivateKeyPath string `mapstructure:"private-key" description:"Path to the private key to use for authentication"` + KnownHostsPath string `mapstructure:"known-hosts" description:"Path to the known hosts file"` + BinaryPath string `mapstructure:"binary-path" description:"Path to the resticprofile binary to use on the remote client"` + ConfigurationFile string `mapstructure:"configuration-file" description:"Path to the configuration file to transfer to the remote client"` + ProfileName string `mapstructure:"profile-name" description:"Name of the profile to use on the remote client"` + SendFiles []string `mapstructure:"send-files" description:"Other configuration files to transfer to the remote client"` +} + +func NewRemote(config *Config, name string) *Remote { + remote := &Remote{ + name: name, + config: config, + } + return remote +} + +// SetRootPath changes the path of all the relative paths and files in the configuration +func (r *Remote) SetRootPath(rootPath string) { + r.PrivateKeyPath = fixPath(r.PrivateKeyPath, expandEnv, absolutePrefix(rootPath)) + r.KnownHostsPath = fixPath(r.KnownHostsPath, expandEnv, absolutePrefix(rootPath)) + r.ConfigurationFile = fixPath(r.ConfigurationFile, expandEnv, absolutePrefix(rootPath)) + + for i := range r.SendFiles { + r.SendFiles[i] = fixPath(r.SendFiles[i], expandEnv, absolutePrefix(rootPath)) + } +} diff --git a/constants/exit_code.go b/constants/exit_code.go new file mode 100644 index 000000000..a5cf66a99 --- /dev/null +++ b/constants/exit_code.go @@ -0,0 +1,10 @@ +package constants + +const ( + ExitSuccess = iota + ExitGeneralError + ExitErrorInvalidFlags + ExitRunningOnBattery + ExitCannotSetupRemoteConfiguration + ExitErrorChildHasNoParentPort = 10 +) diff --git a/constants/other.go b/constants/other.go index ec8f74d9d..ff94cc7fa 100644 --- a/constants/other.go +++ b/constants/other.go @@ -3,4 +3,5 @@ package constants const ( TemporaryDirMarker = "temp:" JSONSchema = "$schema" + ManifestFilename = ".manifest.json" ) diff --git a/constants/section.go b/constants/section.go index 9e9bf3a60..1a8224e46 100644 --- a/constants/section.go +++ b/constants/section.go @@ -13,6 +13,7 @@ const ( SectionConfigurationMixins = "mixins" SectionConfigurationMixinUse = "use" SectionConfigurationSchedule = "schedule" + SectionConfigurationRemotes = "remotes" SectionDefinitionCommon = "common" SectionDefinitionForget = "forget" diff --git a/examples/linux.yaml b/examples/linux.yaml index 5790dad53..dd5492282 100644 --- a/examples/linux.yaml +++ b/examples/linux.yaml @@ -106,7 +106,15 @@ self: at: "*:15,20,25" permission: system after-network-online: true + exclude-file: + - root-excludes + - excludes check: + schedule: + at: "*:15" + permission: system + copy: + initialize: true schedule-permission: user schedule: - "*:15" @@ -131,9 +139,8 @@ src: run-after: echo All Done! run-before: - echo Starting! - - ls -al ~/go source: - - ~/go + - ~/go/src/github.com/creativeprojects/resticprofile tag: - test - dev diff --git a/filesearch/filesearch.go b/filesearch/filesearch.go index b4ec9486a..4fec2db33 100644 --- a/filesearch/filesearch.go +++ b/filesearch/filesearch.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" "sort" "strings" @@ -81,7 +82,7 @@ func NewFinder() Finder { // If the file doesn't have an extension, it will search for all possible extensions func (f Finder) FindConfigurationFile(configFile string) (string, error) { found := "" - extension := filepath.Ext(configFile) + extension := path.Ext(configFile) displayFile := "" if extension != "" { displayFile = fmt.Sprintf("'%s'", configFile) diff --git a/filesearch/filesearch_test.go b/filesearch/filesearch_test.go index f736b01c7..32b5347f3 100644 --- a/filesearch/filesearch_test.go +++ b/filesearch/filesearch_test.go @@ -282,6 +282,7 @@ func TestFindResticBinaryWithTilde(t *testing.T) { t.Skip("not supported on Windows") return } + home, err := os.UserHomeDir() require.NoError(t, err) @@ -336,7 +337,6 @@ func TestShellExpand(t *testing.T) { func TestFindConfigurationIncludes(t *testing.T) { t.Parallel() - fs := afero.NewMemMapFs() testID := fmt.Sprintf("%x", time.Now().UnixNano()) tempDir := os.TempDir() files := []string{ @@ -346,6 +346,7 @@ func TestFindConfigurationIncludes(t *testing.T) { filepath.Join(tempDir, "inc3."+testID+".conf"), } + fs := afero.NewMemMapFs() for _, file := range files { require.NoError(t, afero.WriteFile(fs, file, []byte{}, iofs.ModePerm)) } diff --git a/flags.go b/flags.go index 174ce88f9..a3611d772 100644 --- a/flags.go +++ b/flags.go @@ -38,6 +38,7 @@ type commandLineFlags struct { noPriority bool ignoreOnBattery int usagesHelp string + remote string // url of the remote server to download configuration files from } func envValueOverride[T any](defaultValue T, keys ...string) T { @@ -92,6 +93,7 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) { noPriority: envValueOverride(false, "RESTICPROFILE_NO_PRIORITY"), wait: envValueOverride(false, "RESTICPROFILE_WAIT"), ignoreOnBattery: envValueOverride(0, "RESTICPROFILE_IGNORE_ON_BATTERY"), + remote: envValueOverride("", "RESTICPROFILE_REMOTE"), } flagset.BoolVarP(&flags.help, "help", "h", flags.help, "display this help") @@ -113,6 +115,9 @@ func loadFlags(args []string) (*pflag.FlagSet, commandLineFlags, error) { flagset.BoolVarP(&flags.wait, "wait", "w", flags.wait, "wait at the end until the user presses the enter key") flagset.IntVar(&flags.ignoreOnBattery, "ignore-on-battery", flags.ignoreOnBattery, "don't start the profile when the computer is running on battery. You can specify a value to ignore only when the % charge left is less or equal than the value") flagset.Lookup("ignore-on-battery").NoOptDefVal = "100" // 0 is flag not set, 100 is for a flag with no value (meaning just battery discharge) + flagset.StringVarP(&flags.remote, "remote", "r", flags.remote, "remote server to download configuration files from") + // keep the "remote" flag hidden for now + _ = flagset.MarkHidden("remote") flagset.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { switch name { diff --git a/fuse/file.go b/fuse/file.go new file mode 100644 index 000000000..e79cfac6f --- /dev/null +++ b/fuse/file.go @@ -0,0 +1,25 @@ +package fuse + +import "io/fs" + +type File struct { + name string + fileInfo fs.FileInfo + data []byte +} + +func NewFile(name string, fileInfo fs.FileInfo, data []byte) *File { + return &File{ + name: name, + fileInfo: fileInfo, + data: data, + } +} + +func (f *File) Close() { + // emptying file data + for i := range f.data { + f.data[i] = 0 + } + f.data = nil +} diff --git a/fuse/fs_file.go b/fuse/fs_file.go new file mode 100644 index 000000000..e3f7b5ac5 --- /dev/null +++ b/fuse/fs_file.go @@ -0,0 +1,40 @@ +//go:build !windows + +package fuse + +import ( + "context" + "syscall" + + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +type fsFile struct { + fs.Inode + attr fuse.Attr + file File +} + +var _ = (fs.NodeOpener)((*fsFile)(nil)) +var _ = (fs.NodeGetattrer)((*fsFile)(nil)) + +func (fsf *fsFile) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno { + out.Attr = fsf.attr + return 0 +} + +// Open only needs to send the flags back to the kernel +func (fsf *fsFile) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) { + // tell the kernel not to cache the data + return fsf, fuse.FOPEN_DIRECT_IO, fs.OK +} + +// Read simply returns the data from the file +func (fsf *fsFile) Read(ctx context.Context, f fs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { + end := int(off) + len(dest) + if end > len(fsf.file.data) { + end = len(fsf.file.data) + } + return fuse.ReadResultData(fsf.file.data[off:end]), fs.OK +} diff --git a/fuse/memfs.go b/fuse/memfs.go new file mode 100644 index 000000000..eef614674 --- /dev/null +++ b/fuse/memfs.go @@ -0,0 +1,122 @@ +//go:build !windows + +// Simple implementation of a read-only filesystem in memory. +// +// Based on the examples at https://pkg.go.dev/github.com/hanwen/go-fuse/v2/fs#pkg-examples +package fuse + +import ( + "archive/tar" + "context" + iofs "io/fs" + "log" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +type memFS struct { + fs.Inode + + files []File +} + +func newMemFS(files []File) *memFS { + return &memFS{ + files: files, + } +} + +var _ = (fs.InodeEmbedder)((*memFS)(nil)) + +// The root populates the tree in its OnAdd method +var _ = (fs.NodeOnAdder)((*memFS)(nil)) + +// Close erases the data from all the files +func (memfs *memFS) Close() { + for i := range memfs.files { + memfs.files[i].Close() + } + memfs.files = nil +} + +// OnAdd is called once we are attached to an Inode. We can +// then construct a tree. We construct the entire tree, and +// we don't want parts of the tree to disappear when the +// kernel is short on memory, so we use persistent inodes. +func (memfs *memFS) OnAdd(ctx context.Context) { + for _, file := range memfs.files { + dir, base := filepath.Split(filepath.Clean(file.name)) + + p := memfs.EmbeddedInode() + for _, comp := range strings.Split(dir, "/") { + if len(comp) == 0 { + continue + } + ch := p.GetChild(comp) + if ch == nil { + ch = p.NewPersistentInode(ctx, + &fs.Inode{}, + fs.StableAttr{Mode: syscall.S_IFDIR}) + p.AddChild(comp, ch, false) + } + p = ch + } + + attr := fileInfoToAttr(file.fileInfo) + switch { + case file.fileInfo.Mode().Type()&os.ModeSymlink == os.ModeSymlink: + file.data = nil + fsfile := &fsFile{ + attr: attr, + file: file, + } + p.AddChild(base, memfs.NewPersistentInode(ctx, fsfile, fs.StableAttr{Mode: syscall.S_IFLNK}), false) + + case file.fileInfo.Mode().IsDir(): + fsdir := &fsFile{ + attr: attr, + file: file, + } + p.AddChild(base, memfs.NewPersistentInode(ctx, fsdir, fs.StableAttr{Mode: syscall.S_IFDIR}), false) + + case file.fileInfo.Mode().IsRegular(): + fsfile := &fsFile{ + attr: attr, + file: file, + } + p.AddChild(base, memfs.NewPersistentInode(ctx, fsfile, fs.StableAttr{}), false) + + default: + log.Printf("entry %q: unsupported type '%c'", file.name, file.fileInfo.Mode().Type()) + } + } +} + +func fileInfoToAttr(fileInfo iofs.FileInfo) fuse.Attr { + var out fuse.Attr + if header, ok := fileInfo.Sys().(*tar.Header); ok { + out.Mode = uint32(header.Mode) + out.Size = uint64(header.Size) + out.Uid = uint32(header.Uid) + out.Gid = uint32(header.Gid) + out.SetTimes(&header.AccessTime, &header.ModTime, &header.ChangeTime) + } else { + out.Mode = uint32(fileInfo.Mode()) + out.Size = uint64(fileInfo.Size()) + out.Uid = uint32(os.Geteuid()) + out.Gid = uint32(os.Getegid()) + modTime := fileInfo.ModTime() + out.SetTimes(nil, &modTime, nil) + } + out.Nlink = 1 + const bs = 512 + out.Blksize = bs + out.Blocks = (out.Size + bs - 1) / bs + + return out +} diff --git a/fuse/memfs_test.go b/fuse/memfs_test.go new file mode 100644 index 000000000..1845336e9 --- /dev/null +++ b/fuse/memfs_test.go @@ -0,0 +1,77 @@ +//go:build !windows + +package fuse + +import ( + "archive/tar" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var memfsContents = map[string]string{ + "emptydir/": "", + "file.txt": "content", + "dir/subfile.txt": "other content", + "dir with space/other file.txt": "different content", +} + +func TestMemFS(t *testing.T) { + const fileMode = 0o764 + + files := make([]File, 0) + now := time.Now() + for filename, fileContents := range memfsContents { + h := &tar.Header{ + Name: filename, + Size: int64(len(fileContents)), + Mode: fileMode, + Uid: 100, + Gid: 100, + ModTime: now, + } + + isDir := strings.HasSuffix(filename, "/") + if isDir { + h.Typeflag = tar.TypeDir + } + + files = append(files, File{ + name: filename, + fileInfo: h.FileInfo(), + data: []byte(fileContents), + }) + } + + mnt := t.TempDir() + closeMount, err := MountFS(mnt, files) + if err != nil && strings.Contains(err.Error(), "no FUSE mount utility found") { + t.Skip("no FUSE mount utility found") + } + require.NoError(t, err, "cannot mount FS") + defer closeMount() + + for filename, fileContents := range memfsContents { + fullPath := filepath.Join(mnt, filename) + + filestat, err := os.Stat(fullPath) + require.NoErrorf(t, err, "os.Stat %q", filename) + + if strings.HasSuffix(filename, "/") { + assert.True(t, filestat.IsDir(), "is dir %q", filename) + + } else { + assert.False(t, filestat.IsDir(), "is file %q", filename) + + contents, err := os.ReadFile(fullPath) + assert.NoErrorf(t, err, "read %q", filename) + + assert.Equalf(t, fileContents, string(contents), "file %q", filename) + } + } +} diff --git a/fuse/memfs_windows_test.go b/fuse/memfs_windows_test.go new file mode 100644 index 000000000..e99b6381e --- /dev/null +++ b/fuse/memfs_windows_test.go @@ -0,0 +1,14 @@ +//go:build windows + +package fuse + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMountFS(t *testing.T) { + _, err := MountFS("mnt", []File{}) + assert.Error(t, err) +} diff --git a/fuse/mount.go b/fuse/mount.go new file mode 100644 index 000000000..7aae32fef --- /dev/null +++ b/fuse/mount.go @@ -0,0 +1,40 @@ +//go:build !windows + +package fuse + +import ( + "fmt" + + "github.com/creativeprojects/clog" + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +func MountFS(mountpoint string, files []File) (func(), error) { + memFS := newMemFS(files) + + clog.Debugf("mounting filesystem at %s", mountpoint) + + opts := &fs.Options{ + MountOptions: fuse.MountOptions{ + Debug: false, // generates a LOT of logs + FsName: "resticprofile", + DisableXAttrs: true, + EnableLocks: false, + }, + } + server, err := fs.Mount(mountpoint, memFS, opts) + if err != nil { + return nil, fmt.Errorf("failed to mount filesystem: %w", err) + } + closeFS := func() { + clog.Debug("unmounting filesystem") + err := server.Unmount() // don't need to call Wait after Unmount + if err != nil { + clog.Errorf("failed to unmount filesystem: %v", err) + } + + memFS.Close() + } + return closeFS, nil +} diff --git a/fuse/mount_windows.go b/fuse/mount_windows.go new file mode 100644 index 000000000..625407b0e --- /dev/null +++ b/fuse/mount_windows.go @@ -0,0 +1,7 @@ +package fuse + +import "errors" + +func MountFS(_ string, _ []File) (func(), error) { + return nil, errors.New("not supported on Windows") +} diff --git a/go.mod b/go.mod index 15affef83..9345f7bfc 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/govalues/decimal v0.1.36 // indirect + github.com/hanwen/go-fuse/v2 v2.7.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.7.0 // indirect diff --git a/go.sum b/go.sum index b3a87e76d..036be8044 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/govalues/decimal v0.1.36 h1:dojDpsSvrk0ndAx8+saW5h9WDIHdWpIwrH/yhl9olyU= github.com/govalues/decimal v0.1.36/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM= +github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw= +github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= diff --git a/integration_test.go b/integration_test.go index 9eb38254c..2511e8106 100644 --- a/integration_test.go +++ b/integration_test.go @@ -10,6 +10,7 @@ import ( "github.com/creativeprojects/resticprofile/config" "github.com/creativeprojects/resticprofile/platform" "github.com/creativeprojects/resticprofile/term" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -110,7 +111,7 @@ func TestFromConfigFileToCommandLine(t *testing.T) { // try all the config files one by one for _, configFile := range files { t.Run(configFile, func(t *testing.T) { - cfg, err := config.LoadFile(configFile, "") + cfg, err := config.LoadFile(afero.NewOsFs(), configFile, "") require.NoError(t, err) require.NotNil(t, cfg) diff --git a/main.go b/main.go index 8b9046687..cc40953eb 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "github.com/creativeprojects/resticprofile/term" "github.com/creativeprojects/resticprofile/util/shutdown" "github.com/mackerelio/go-osstat/memory" + "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -138,6 +139,26 @@ func main() { banner() + if flags.remote != "" { + closeFS, remoteParameters, err := setupRemoteConfiguration(flags.remote) + if err != nil { + // need to setup console logging to display the error message + closeLogger := setupLogging(nil) + defer closeLogger() + clog.Error(err) + exitCode = constants.ExitCannotSetupRemoteConfiguration + return + } + if flags.config == constants.DefaultConfigurationFile && remoteParameters.ConfigurationFile != "" { + flags.config = remoteParameters.ConfigurationFile + } + if flags.name == constants.DefaultProfileName && remoteParameters.ProfileName != "" { + flags.name = remoteParameters.ProfileName + } + flags.resticArgs = remoteParameters.CommandLineArguments + shutdown.AddHook(closeFS) + } + // resticprofile own commands (configuration file may not be loaded) if len(flags.resticArgs) > 0 { if ownCommands.Exists(flags.resticArgs[0], false) { @@ -270,17 +291,25 @@ func main() { } func banner() { - clog.Debugf("resticprofile %s compiled with %s", version, runtime.Version()) + clog.Debugf( + "resticprofile %s compiled with %s %s/%s", + version, + runtime.Version(), + runtime.GOOS, + runtime.GOARCH, + ) } func loadConfig(flags commandLineFlags, silent bool) (cfg *config.Config, global *config.Global, err error) { + fs := afero.NewOsFs() + var configFile string if configFile, err = filesearch.NewFinder().FindConfigurationFile(flags.config); err == nil { if configFile != flags.config && !silent { clog.Infof("using configuration file: %s", configFile) } - if cfg, err = config.LoadFile(configFile, flags.format); err == nil { + if cfg, err = config.LoadFile(fs, configFile, flags.format); err == nil { global, err = cfg.GetGlobalSection() if err != nil { err = fmt.Errorf("cannot load global configuration: %w", err) diff --git a/own_commands.go b/own_commands.go index ff23beae8..b8caf0e44 100644 --- a/own_commands.go +++ b/own_commands.go @@ -5,6 +5,8 @@ import ( "io" "os" "strings" + + "github.com/creativeprojects/clog" ) // commandContext is the context for running a command. @@ -23,6 +25,7 @@ type ownCommand struct { hide bool // don't display the command in help and completion hideInCompletion bool // don't display the command in completion noProfile bool // true if the command doesn't need a profile name + experimental bool // display a warning when using this command flags map[string]string // own command flags should be simple enough to be handled manually for now } @@ -61,6 +64,9 @@ func (o *OwnCommands) Run(ctx *Context) error { if command == nil { return fmt.Errorf("command not found: %v", ctx.request.command) } + if command.experimental { + clog.Warningf("%s: this command is experimental and its behaviour may change in the future", ctx.request.command) + } return command.action(os.Stdout, commandContext{ ownCommands: o, Context: *ctx, diff --git a/remote.go b/remote.go new file mode 100644 index 000000000..bade3953b --- /dev/null +++ b/remote.go @@ -0,0 +1,137 @@ +package main + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/creativeprojects/clog" + "github.com/creativeprojects/resticprofile/constants" + "github.com/creativeprojects/resticprofile/fuse" + "github.com/creativeprojects/resticprofile/remote" +) + +func loadRemoteFiles(endpoint string) ([]fuse.File, *remote.Manifest, error) { + var parameters *remote.Manifest + + client := http.DefaultClient + request, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, endpoint, http.NoBody) + if err != nil { + return nil, nil, fmt.Errorf("failed to create request: %w", err) + } + request.Header.Set("Accept", "application/x-tar") + + resp, err := client.Do(request) + if err != nil { + return nil, nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + buf := &bytes.Buffer{} + _, _ = buf.ReadFrom(resp.Body) + return nil, nil, fmt.Errorf("http error %d: %q", resp.StatusCode, strings.TrimSpace(buf.String())) + } + + if resp.Header.Get("Content-Type") != "application/x-tar" { + return nil, nil, fmt.Errorf("unexpected content type: %s", resp.Header.Get("Content-Type")) + } + + files := []fuse.File{} + reader := tar.NewReader(resp.Body) + for { + hdr, err := reader.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + return nil, nil, fmt.Errorf("failed to read tar header: %w", err) + } + if !filepath.IsLocal(hdr.Name) { + return nil, nil, fmt.Errorf("invalid file name: %s", hdr.Name) + } + if hdr.Name == constants.ManifestFilename { + clog.Debugf("downloading manifest (%d bytes)", hdr.Size) + parameters, err = getManifestParameters(reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to read manifest: %w", err) + } + } else { + clog.Debugf("downloading file %s (%d bytes)", hdr.Name, hdr.Size) + data := make([]byte, hdr.Size) + read, err := reader.Read(data) + if err != nil && err != io.EOF { + return nil, nil, fmt.Errorf("failed to download file content: %w", err) + } + if read != int(hdr.Size) { + return nil, nil, fmt.Errorf("file size mismatch: expected %d, got %d", hdr.Size, read) + } + files = append(files, *fuse.NewFile(hdr.Name, hdr.FileInfo(), data)) + } + } + + return files, parameters, nil +} + +func getManifestParameters(reader io.Reader) (*remote.Manifest, error) { + manifest := &remote.Manifest{} + decoder := json.NewDecoder(reader) + err := decoder.Decode(manifest) + if err != nil { + return nil, fmt.Errorf("failed to decode manifest: %w", err) + } + return manifest, nil +} + +// setupRemoteConfiguration downloads the configuration files from the remote endpoint and mounts the virtual FS +func setupRemoteConfiguration(remoteEndpoint string) (func(), *remote.Manifest, error) { + files, parameters, err := loadRemoteFiles(remoteEndpoint) + if err != nil { + return nil, nil, err + } + + closeMountpoint := func() {} + mountpoint := parameters.Mountpoint + if mountpoint == "" { + // generates a temporary directory + mountpoint = filepath.Join(os.TempDir(), fmt.Sprintf("%s-%x", "resticprofile", rand.Uint32())) + closeMountpoint = func() { + err = os.Remove(mountpoint) + if err != nil { + clog.Errorf("failed to remove mountpoint: %v", err) + } + } + } + err = os.MkdirAll(mountpoint, 0o755) + if err != nil { + return nil, parameters, fmt.Errorf("failed to create mount directory: %w", err) + } + + closeFs, err := fuse.MountFS(mountpoint, files) + if err != nil { + return closeMountpoint, parameters, err + } + + wd, _ := os.Getwd() + err = os.Chdir(mountpoint) + if err != nil { + return func() { + closeFs() + closeMountpoint() + }, parameters, fmt.Errorf("failed to change directory: %w", err) + } + + return func() { + _ = os.Chdir(wd) + closeFs() + closeMountpoint() + }, parameters, nil +} diff --git a/remote/manifest.go b/remote/manifest.go new file mode 100644 index 000000000..53fdb6a7a --- /dev/null +++ b/remote/manifest.go @@ -0,0 +1,9 @@ +package remote + +type Manifest struct { + Version string // resticprofile version + ConfigurationFile string + ProfileName string + Mountpoint string // Mountpoint of the virtual FS if configured + CommandLineArguments []string +} diff --git a/remote/tar.go b/remote/tar.go new file mode 100644 index 000000000..08938c09a --- /dev/null +++ b/remote/tar.go @@ -0,0 +1,88 @@ +package remote + +import ( + "archive/tar" + "fmt" + "io" + "os" + "time" + + "github.com/creativeprojects/clog" + "github.com/spf13/afero" +) + +type Tar struct { + writer *tar.Writer +} + +func NewTar(w io.Writer) *Tar { + return &Tar{ + writer: tar.NewWriter(w), + } +} + +func (t *Tar) SendFiles(fs afero.Fs, files []string) error { + for _, filename := range files { + fileInfo, err := fs.Stat(filename) + if err != nil { + clog.Errorf("unable to stat file %s: %v", filename, err) + continue + } + fileHeader, err := tar.FileInfoHeader(fileInfo, "") + if err != nil { + clog.Errorf("unable to create tar header for file %s: %v", filename, err) + continue + } + err = t.writer.WriteHeader(fileHeader) + if err != nil { + clog.Errorf("unable to write tar header for file %s: %v", filename, err) + break + } + file, err := fs.Open(filename) + if err != nil { + clog.Errorf("unable to open file %s: %v", filename, err) + continue + } + defer file.Close() + + written, err := io.Copy(t.writer, file) + if err != nil { + clog.Errorf("unable to write file %s: %v", filename, err) + break + } + if written != fileInfo.Size() { + clog.Errorf("file %s: written %d bytes, expected %d", filename, written, fileInfo.Size()) + break + } + clog.Debugf("file %s: written %d bytes", filename, written) + } + return nil +} + +func (t *Tar) SendFile(name string, data []byte) error { + header := &tar.Header{ + Name: name, + Size: int64(len(data)), + ModTime: time.Now(), + Mode: 0o444, + Typeflag: tar.TypeReg, + Uid: os.Geteuid(), + Gid: os.Getegid(), + } + if err := t.writer.WriteHeader(header); err != nil { + return err + } + written, err := t.writer.Write(data) + if err != nil { + return err + } + if written != len(data) { + return fmt.Errorf("manifest written %d bytes, expected %d", written, len(data)) + } + clog.Debugf("manifest written %d bytes", written) + return nil +} + +func (t *Tar) Close() error { + return t.writer.Close() +} diff --git a/send.go b/send.go new file mode 100644 index 000000000..7823751e7 --- /dev/null +++ b/send.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "path" + + "github.com/creativeprojects/resticprofile/constants" + "github.com/creativeprojects/resticprofile/remote" + "github.com/creativeprojects/resticprofile/ssh" + "github.com/spf13/afero" +) + +func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { + if len(cmdCtx.flags.resticArgs) < 2 { + return fmt.Errorf("missing argument: remote name") + } + remoteName := cmdCtx.flags.resticArgs[1] + if !cmdCtx.config.HasRemote(remoteName) { + return fmt.Errorf("remote not found") + } + remoteConfig, err := cmdCtx.config.GetRemote(remoteName) + if err != nil { + return err + } + // send the files to the remote using tar + handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + // prepare manifest file + manifest := remote.Manifest{ + ConfigurationFile: path.Base(remoteConfig.ConfigurationFile), // need to take file path into consideration + ProfileName: remoteConfig.ProfileName, + CommandLineArguments: cmdCtx.flags.resticArgs[2:], + } + manifestData, err := json.Marshal(manifest) + if err != nil { + resp.Header().Set("Content-Type", "text/plain") + resp.WriteHeader(http.StatusInternalServerError) + _, _ = resp.Write([]byte(err.Error())) + return + } + + resp.Header().Set("Content-Type", "application/x-tar") + resp.WriteHeader(http.StatusOK) + + tar := remote.NewTar(resp) + defer tar.Close() + _ = tar.SendFiles(afero.NewOsFs(), append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) + _ = tar.SendFile(constants.ManifestFilename, manifestData) + }) + cnx := ssh.NewSSH(ssh.Config{ + Host: remoteConfig.Host, + Username: remoteConfig.Username, + PrivateKeyPath: remoteConfig.PrivateKeyPath, + KnownHostsPath: remoteConfig.KnownHostsPath, + Handler: handler, + }) + defer cnx.Close() + + err = cnx.Connect() + if err != nil { + return err + } + binaryPath := remoteConfig.BinaryPath + if binaryPath == "" { + binaryPath = "resticprofile" + } + commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", + binaryPath, + cnx.TunnelPort(), + remoteName, + ) + err = cnx.Run(commandLine) + if err != nil { + return fmt.Errorf("failed to run command %q: %w", commandLine, err) + } + return nil +} diff --git a/serve.go b/serve.go new file mode 100644 index 000000000..4e6aa441c --- /dev/null +++ b/serve.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "path" + "time" + + "github.com/creativeprojects/clog" + "github.com/creativeprojects/resticprofile/constants" + "github.com/creativeprojects/resticprofile/remote" + "github.com/spf13/afero" +) + +func serveCommand(w io.Writer, cmdCtx commandContext) error { + if len(cmdCtx.flags.resticArgs) < 2 { + return fmt.Errorf("missing argument: port") + } + handler := http.NewServeMux() + handler.HandleFunc("GET /configuration/{remote}", func(resp http.ResponseWriter, req *http.Request) { + remoteName := req.PathValue("remote") + if !cmdCtx.config.HasRemote(remoteName) { + resp.Header().Set("Content-Type", "text/plain") + resp.WriteHeader(http.StatusNotFound) + _, _ = resp.Write([]byte("remote not found")) + return + } + remoteConfig, err := cmdCtx.config.GetRemote(remoteName) + if err != nil { + resp.Header().Set("Content-Type", "text/plain") + resp.WriteHeader(http.StatusBadRequest) + _, _ = resp.Write([]byte(err.Error())) + return + } + + // prepare manifest file + manifest := remote.Manifest{ + ConfigurationFile: path.Base(remoteConfig.ConfigurationFile), // need to take file path into consideration + ProfileName: remoteConfig.ProfileName, + } + manifestData, err := json.Marshal(manifest) + if err != nil { + resp.Header().Set("Content-Type", "text/plain") + resp.WriteHeader(http.StatusInternalServerError) + _, _ = resp.Write([]byte(err.Error())) + return + } + + clog.Debugf("sending configuration for %q", remoteName) + resp.Header().Set("Content-Type", "application/x-tar") + resp.WriteHeader(http.StatusOK) + + tar := remote.NewTar(resp) + defer tar.Close() + _ = tar.SendFiles(afero.NewOsFs(), append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) + _ = tar.SendFile(constants.ManifestFilename, manifestData) + + }) + + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt) + defer signal.Stop(quit) + + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%s", cmdCtx.flags.resticArgs[1]), + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + } + + // put the shutdown code in a goroutine + go func(server *http.Server, quit chan os.Signal) { + <-quit + + clog.Info("shutting down the server") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err := server.Shutdown(ctx) + if err != nil { + clog.Errorf("error while shutting down the server: %v", err) + } + + }(server, quit) + + // we want to return the server error if any so we need to keep it in the main thread. + clog.Infof("listening on %s", server.Addr) + err := server.ListenAndServe() + if err != nil && err != http.ErrServerClosed { + return err + } + return nil +} diff --git a/ssh/config.go b/ssh/config.go new file mode 100644 index 000000000..db8428527 --- /dev/null +++ b/ssh/config.go @@ -0,0 +1,44 @@ +package ssh + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + "strings" +) + +type Config struct { + Host string + Username string + PrivateKeyPath string + KnownHostsPath string + Handler http.Handler +} + +func (c *Config) Validate() error { + if c.Host == "" { + return fmt.Errorf("host is required") + } + if c.Username == "" { + return fmt.Errorf("username is required") + } + if c.PrivateKeyPath == "" { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("unable to get current user home directory: %w", err) + } + c.PrivateKeyPath = filepath.Join(home, ".ssh/id_rsa") // we can go through all the default name for each key type + } + if c.KnownHostsPath == "" { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("unable to get current user home directory: %w", err) + } + c.KnownHostsPath = filepath.Join(home, ".ssh/known_hosts") + } + if !strings.Contains(c.Host, ":") { + c.Host = c.Host + ":22" + } + return nil +} diff --git a/ssh/ssh.go b/ssh/ssh.go new file mode 100644 index 000000000..2183ff687 --- /dev/null +++ b/ssh/ssh.go @@ -0,0 +1,147 @@ +package ssh + +import ( + "context" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "time" + + "github.com/creativeprojects/clog" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const startPort = 10001 + +type SSH struct { + config Config + port int + client *ssh.Client + tunnel net.Listener + server *http.Server +} + +func NewSSH(config Config) *SSH { + return &SSH{ + config: config, + port: startPort, + } +} + +func (s *SSH) Connect() error { + err := s.config.Validate() + if err != nil { + return err + } + hostKeyCallback, err := knownhosts.New(s.config.KnownHostsPath) + if err != nil { + return fmt.Errorf("cannot load host keys from known_hosts: %w", err) + } + key, err := os.ReadFile(s.config.PrivateKeyPath) + if err != nil { + return fmt.Errorf("unable to read private key: %w", err) + } + + // Create the Signer for this private key. + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return fmt.Errorf("unable to parse private key: %w", err) + } + + config := &ssh.ClientConfig{ + User: s.config.Username, + Auth: []ssh.AuthMethod{ + // Use the PublicKeys method for remote authentication. + ssh.PublicKeys(signer), + }, + HostKeyCallback: hostKeyCallback, + HostKeyAlgorithms: []string{ssh.KeyAlgoED25519, ssh.KeyAlgoECDSA256}, // we might need to make this configurable + } + + // Connect to the remote server and perform the SSH handshake. + s.client, err = ssh.Dial("tcp", s.config.Host, config) + if err != nil { + return fmt.Errorf("unable to connect: %w", err) + } + + // Request the remote side to open a local port + s.tunnel, err = s.client.Listen("tcp", fmt.Sprintf("localhost:%d", s.port)) // increment the port in a loop in case of an error + if err != nil { + log.Fatal("unable to register tcp forward: ", err) + } + + go func() { + s.server = &http.Server{ + Handler: s.config.Handler, + ReadHeaderTimeout: 5 * time.Second, + } + // Serve HTTP with your SSH server acting as a reverse proxy. + err := s.server.Serve(s.tunnel) + if err != nil && err != http.ErrServerClosed && err != io.EOF { + clog.Warningf("unable to serve http: %s", err) + } + }() + time.Sleep(100 * time.Millisecond) // wait for the server to start + return nil +} + +func (s *SSH) TunnelPort() int { + return s.port +} + +func (s *SSH) Run(command string) error { + // Each ClientConn can support multiple interactive sessions, + // represented by a Session. + session, err := s.client.NewSession() + if err != nil { + return fmt.Errorf("failed to create session: %w", err) + } + defer session.Close() + + // request a pseudo terminal to display colors + if termType := os.Getenv("TERM"); termType != "" { + modes := ssh.TerminalModes{ + ssh.ECHO: 0, // disable echoing + } + if err := session.RequestPty(termType, 40, 80, modes); err != nil { + clog.Warningf("request for pseudo terminal failed: %s", err) + } + } + + // Once a Session is created, we can execute a single command on + // the remote side using the Run method. + session.Stdout = os.Stdout + session.Stderr = os.Stderr + if err := session.Run(command); err != nil { + return fmt.Errorf("failed to run: %w", err) + } + return nil +} + +func (s *SSH) Close() { + // close the tunnel first otherwise it fails with error: "ssh: cancel-tcpip-forward failed" + if s.tunnel != nil { + err := s.tunnel.Close() + if err != nil { + clog.Warningf("unable to close tunnel: %s", err) + } + } + if s.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err := s.server.Shutdown(ctx) + if err != nil { + clog.Warningf("unable to close http server: %s", err) + } + } + if s.client != nil { + err := s.client.Close() + if err != nil { + clog.Warningf("unable to close ssh connection: %s", err) + } + } +} From a7e4c75c246e2e618f47c4db15067f97d21f85af Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 13:03:35 +0100 Subject: [PATCH 02/24] fix(tests): replace assert.Error with require.Error for better error handling --- config/config_test.go | 4 ++-- fuse/memfs.go | 14 +++++++------- go.mod | 4 ++-- go.sum | 12 ++++++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 675337303..a4b7ec70c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -464,7 +464,7 @@ use = "another-run-before2"`) createFile(t, fs, "fail-"+testID+".inc.toml", `[two]`) _, err := LoadFile(fs, configFile, "") - assert.Error(t, err) + require.Error(t, err) assert.Regexp(t, ".+ is in hcl format, includes must use the same format", err.Error()) }) @@ -479,7 +479,7 @@ use = "another-run-before2"`) createFile(t, fs, "fail-"+testID+".inc.hcl", `one { }`) _, err := LoadFile(fs, configFile, "") - assert.Error(t, err) + require.Error(t, err) assert.Regexp(t, "hcl format .+ cannot be used in includes from toml", err.Error()) }) diff --git a/fuse/memfs.go b/fuse/memfs.go index eef614674..fb9bc8320 100644 --- a/fuse/memfs.go +++ b/fuse/memfs.go @@ -100,16 +100,16 @@ func (memfs *memFS) OnAdd(ctx context.Context) { func fileInfoToAttr(fileInfo iofs.FileInfo) fuse.Attr { var out fuse.Attr if header, ok := fileInfo.Sys().(*tar.Header); ok { - out.Mode = uint32(header.Mode) - out.Size = uint64(header.Size) - out.Uid = uint32(header.Uid) - out.Gid = uint32(header.Gid) + out.Mode = uint32(header.Mode) //nolint:gosec + out.Size = uint64(header.Size) //nolint:gosec + out.Uid = uint32(header.Uid) //nolint:gosec + out.Gid = uint32(header.Gid) //nolint:gosec out.SetTimes(&header.AccessTime, &header.ModTime, &header.ChangeTime) } else { out.Mode = uint32(fileInfo.Mode()) - out.Size = uint64(fileInfo.Size()) - out.Uid = uint32(os.Geteuid()) - out.Gid = uint32(os.Getegid()) + out.Size = uint64(fileInfo.Size()) //nolint:gosec + out.Uid = uint32(os.Geteuid()) //nolint:gosec + out.Gid = uint32(os.Getegid()) //nolint:gosec modTime := fileInfo.ModTime() out.SetTimes(nil, &modTime, nil) } diff --git a/go.mod b/go.mod index 9345f7bfc..143c09813 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/creativeprojects/go-selfupdate v1.5.0 github.com/distatus/battery v0.11.0 github.com/fatih/color v1.18.0 + github.com/hanwen/go-fuse/v2 v2.8.0 github.com/joho/godotenv v1.5.1 github.com/mackerelio/go-osstat v0.2.6 github.com/mattn/go-colorable v0.1.14 @@ -23,6 +24,7 @@ require ( github.com/spf13/pflag v1.0.7 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 + golang.org/x/crypto v0.40.0 golang.org/x/sys v0.34.0 golang.org/x/term v0.33.0 golang.org/x/text v0.27.0 @@ -44,7 +46,6 @@ require ( github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/govalues/decimal v0.1.36 // indirect - github.com/hanwen/go-fuse/v2 v2.7.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.7.0 // indirect @@ -71,7 +72,6 @@ require ( github.com/xanzy/go-gitlab v0.115.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/time v0.11.0 // indirect diff --git a/go.sum b/go.sum index 036be8044..c79cc3d7c 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/govalues/decimal v0.1.36 h1:dojDpsSvrk0ndAx8+saW5h9WDIHdWpIwrH/yhl9olyU= github.com/govalues/decimal v0.1.36/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM= -github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw= -github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= +github.com/hanwen/go-fuse/v2 v2.8.0 h1:wV8rG7rmCz8XHSOwBZhG5YcVqcYjkzivjmbaMafPlAs= +github.com/hanwen/go-fuse/v2 v2.8.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -67,6 +67,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0= @@ -79,6 +81,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -144,8 +148,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= From ec9550b8704fca1bde8c9e8e11fd8f4100912123 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 14:33:28 +0100 Subject: [PATCH 03/24] remove afero from LoadFile --- complete.go | 3 +- config/checkdoc/main.go | 3 +- config/config.go | 6 +-- config/config_test.go | 83 ++++++++++++++++++---------------------- filesearch/filesearch.go | 3 +- integration_test.go | 3 +- main.go | 5 +-- 7 files changed, 45 insertions(+), 61 deletions(-) diff --git a/complete.go b/complete.go index 318603a57..c2ac27f10 100644 --- a/complete.go +++ b/complete.go @@ -10,7 +10,6 @@ import ( "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/config" "github.com/creativeprojects/resticprofile/filesearch" - "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -136,7 +135,7 @@ func (c *Completer) listProfileNames() (list []string) { } if file, err := filesearch.NewFinder().FindConfigurationFile(filename); err == nil { - if conf, err := config.LoadFile(afero.NewOsFs(), file, format); err == nil { + if conf, err := config.LoadFile(file, format); err == nil { list = append(list, conf.GetProfileNames()...) for name := range conf.GetProfileGroups() { list = append(list, name) diff --git a/config/checkdoc/main.go b/config/checkdoc/main.go index 79a34de0c..2e888187f 100644 --- a/config/checkdoc/main.go +++ b/config/checkdoc/main.go @@ -11,7 +11,6 @@ import ( "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/config" - "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -175,7 +174,7 @@ func saveConfiguration(content []byte, configType string) (string, error) { // checkConfiguration returns true when the configuration is valid func checkConfiguration(filename, configType string, lineNum int) bool { - cfg, err := config.LoadFile(afero.NewOsFs(), filename, configType) + cfg, err := config.LoadFile(filename, configType) if err != nil { clog.Errorf(" %q on line %d: %s", configType, lineNum, err) return false diff --git a/config/config.go b/config/config.go index a99e6bfc4..7c8323959 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "maps" + "os" "path/filepath" "slices" "sort" @@ -20,7 +21,6 @@ import ( "github.com/creativeprojects/resticprofile/util/maybe" "github.com/creativeprojects/resticprofile/util/templates" "github.com/mitchellh/mapstructure" - "github.com/spf13/afero" "github.com/spf13/viper" ) @@ -74,7 +74,7 @@ func formatFromExtension(configFile string) string { // LoadFile loads configuration from file // Leave format blank for auto-detection from the file extension -func LoadFile(fs afero.Fs, configFile, format string) (config *Config, err error) { +func LoadFile(configFile, format string) (config *Config, err error) { if format == "" { format = formatFromExtension(configFile) } @@ -84,7 +84,7 @@ func LoadFile(fs afero.Fs, configFile, format string) (config *Config, err error readAndAdd := func(configFile string, replace bool) error { clog.Debugf("loading: %s", configFile) - file, fileErr := fs.Open(configFile) + file, fileErr := os.Open(configFile) if fileErr != nil { return fmt.Errorf("cannot open configuration file for reading: %w", fileErr) } diff --git a/config/config_test.go b/config/config_test.go index a4b7ec70c..398b161a9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/creativeprojects/resticprofile/util/maybe" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -367,22 +366,25 @@ x=0 } func TestIncludes(t *testing.T) { - createFile := func(t *testing.T, fs afero.Fs, suffix, content string) string { + createFile := func(t *testing.T, suffix, content string) string { t.Helper() name := "" - file, err := afero.TempFile(fs, "", "*-"+suffix) + file, err := os.CreateTemp("", "*-"+suffix) if err == nil { defer file.Close() _, err = file.WriteString(content) name = file.Name() + t.Cleanup(func() { + _ = os.Remove(name) + }) } require.NoError(t, err) return name } - mustLoadConfig := func(t *testing.T, fs afero.Fs, configFile string) *Config { + mustLoadConfig := func(t *testing.T, configFile string) *Config { t.Helper() - config, err := LoadFile(fs, configFile, "") + config, err := LoadFile(configFile, "") require.NoError(t, err) return config } @@ -390,15 +392,14 @@ func TestIncludes(t *testing.T) { testID := fmt.Sprintf("%d", time.Now().Unix()) t.Run("multiple-includes", func(t *testing.T) { - fs := afero.NewMemMapFs() content := fmt.Sprintf(`includes=['*%[1]s.inc.toml','*%[1]s.inc.yaml','*%[1]s.inc.json']`, testID) - configFile := createFile(t, fs, "profiles.conf", content) - createFile(t, fs, "d-"+testID+".inc.toml", "[one]\nk='v'") - createFile(t, fs, "o-"+testID+".inc.yaml", `two: { k: v }`) - createFile(t, fs, "j-"+testID+".inc.json", `{"three":{ "k": "v" }}`) + configFile := createFile(t, "profiles.conf", content) + createFile(t, "d-"+testID+".inc.toml", "[one]\nk='v'") + createFile(t, "o-"+testID+".inc.yaml", `two: { k: v }`) + createFile(t, "j-"+testID+".inc.json", `{"three":{ "k": "v" }}`) - config := mustLoadConfig(t, fs, configFile) + config := mustLoadConfig(t, configFile) assert.True(t, config.IsSet("includes")) assert.True(t, config.HasProfile("one")) assert.True(t, config.HasProfile("two")) @@ -406,18 +407,16 @@ func TestIncludes(t *testing.T) { }) t.Run("overrides", func(t *testing.T) { - fs := afero.NewMemMapFs() - - configFile := createFile(t, fs, "profiles.conf", ` + configFile := createFile(t, "profiles.conf", ` includes = "*`+testID+`.inc.toml" [default] repository = "default-repo"`) - createFile(t, fs, "override-"+testID+".inc.toml", ` + createFile(t, "override-"+testID+".inc.toml", ` [default] repository = "overridden-repo"`) - config := mustLoadConfig(t, fs, configFile) + config := mustLoadConfig(t, configFile) assert.True(t, config.HasProfile("default")) profile, err := config.GetProfile("default") @@ -426,26 +425,24 @@ repository = "overridden-repo"`) }) t.Run("mixins", func(t *testing.T) { - fs := afero.NewMemMapFs() - - configFile := createFile(t, fs, "profiles.conf", ` + configFile := createFile(t, "profiles.conf", ` version = 2 includes = "*`+testID+`.inc.toml" [profiles.default] use = "another-run-before" run-before = "default-before"`) - createFile(t, fs, "mixin-"+testID+".inc.toml", ` + createFile(t, "mixin-"+testID+".inc.toml", ` [mixins.another-run-before] "run-before..." = "another-run-before" [mixins.another-run-before2] "run-before..." = "another-run-before2"`) - createFile(t, fs, "mixin-use-"+testID+".inc.toml", ` + createFile(t, "mixin-use-"+testID+".inc.toml", ` [profiles.default] use = "another-run-before2"`) - config := mustLoadConfig(t, fs, configFile) + config := mustLoadConfig(t, configFile) assert.True(t, config.HasProfile("default")) profile, err := config.GetProfile("default") @@ -454,56 +451,50 @@ use = "another-run-before2"`) }) t.Run("hcl-includes-only-hcl", func(t *testing.T) { - fs := afero.NewMemMapFs() + configFile := createFile(t, "profiles.hcl", `includes = "*`+testID+`.inc.*"`) + createFile(t, "pass-"+testID+".inc.hcl", `one { }`) - configFile := createFile(t, fs, "profiles.hcl", `includes = "*`+testID+`.inc.*"`) - createFile(t, fs, "pass-"+testID+".inc.hcl", `one { }`) - - config := mustLoadConfig(t, fs, configFile) + config := mustLoadConfig(t, configFile) assert.True(t, config.HasProfile("one")) - createFile(t, fs, "fail-"+testID+".inc.toml", `[two]`) - _, err := LoadFile(fs, configFile, "") + createFile(t, "fail-"+testID+".inc.toml", `[two]`) + _, err := LoadFile(configFile, "") require.Error(t, err) assert.Regexp(t, ".+ is in hcl format, includes must use the same format", err.Error()) }) t.Run("non-hcl-include-no-hcl", func(t *testing.T) { - fs := afero.NewMemMapFs() - - configFile := createFile(t, fs, "profiles.toml", `includes = "*`+testID+`.inc.*"`) - createFile(t, fs, "pass-"+testID+".inc.toml", "[one]\nk='v'") + configFile := createFile(t, "profiles.toml", `includes = "*`+testID+`.inc.*"`) + createFile(t, "pass-"+testID+".inc.toml", "[one]\nk='v'") - config := mustLoadConfig(t, fs, configFile) + config := mustLoadConfig(t, configFile) assert.True(t, config.HasProfile("one")) - createFile(t, fs, "fail-"+testID+".inc.hcl", `one { }`) - _, err := LoadFile(fs, configFile, "") + createFile(t, "fail-"+testID+".inc.hcl", `one { }`) + _, err := LoadFile(configFile, "") require.Error(t, err) assert.Regexp(t, "hcl format .+ cannot be used in includes from toml", err.Error()) }) t.Run("cannot-load-different-versions", func(t *testing.T) { - fs := afero.NewMemMapFs() content := fmt.Sprintf(`includes=['*%s.inc.json']`, testID) - configFile := createFile(t, fs, "profiles.conf", content) - createFile(t, fs, "a-"+testID+".inc.json", `{"version": 2, "profiles": {"one":{}}}`) - createFile(t, fs, "b-"+testID+".inc.json", `{"two":{}}`) + configFile := createFile(t, "profiles.conf", content) + createFile(t, "a-"+testID+".inc.json", `{"version": 2, "profiles": {"one":{}}}`) + createFile(t, "b-"+testID+".inc.json", `{"two":{}}`) - _, err := LoadFile(fs, configFile, "") + _, err := LoadFile(configFile, "") assert.ErrorContains(t, err, "cannot include different versions of the configuration file") }) t.Run("cannot-load-different-versions", func(t *testing.T) { - fs := afero.NewMemMapFs() content := fmt.Sprintf(`{"version": 2, "includes":["*%s.inc.json"]}`, testID) - configFile := createFile(t, fs, "profiles.json", content) - createFile(t, fs, "c-"+testID+".inc.json", `{"version": 1, "two":{}}`) - createFile(t, fs, "d-"+testID+".inc.json", `{"profiles": {"one":{}}}`) + configFile := createFile(t, "profiles.json", content) + createFile(t, "c-"+testID+".inc.json", `{"version": 1, "two":{}}`) + createFile(t, "d-"+testID+".inc.json", `{"profiles": {"one":{}}}`) - _, err := LoadFile(fs, configFile, "") + _, err := LoadFile(configFile, "") assert.ErrorContains(t, err, "cannot include different versions of the configuration file") }) } diff --git a/filesearch/filesearch.go b/filesearch/filesearch.go index 4fec2db33..b4ec9486a 100644 --- a/filesearch/filesearch.go +++ b/filesearch/filesearch.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "os/exec" - "path" "path/filepath" "sort" "strings" @@ -82,7 +81,7 @@ func NewFinder() Finder { // If the file doesn't have an extension, it will search for all possible extensions func (f Finder) FindConfigurationFile(configFile string) (string, error) { found := "" - extension := path.Ext(configFile) + extension := filepath.Ext(configFile) displayFile := "" if extension != "" { displayFile = fmt.Sprintf("'%s'", configFile) diff --git a/integration_test.go b/integration_test.go index 2511e8106..9eb38254c 100644 --- a/integration_test.go +++ b/integration_test.go @@ -10,7 +10,6 @@ import ( "github.com/creativeprojects/resticprofile/config" "github.com/creativeprojects/resticprofile/platform" "github.com/creativeprojects/resticprofile/term" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -111,7 +110,7 @@ func TestFromConfigFileToCommandLine(t *testing.T) { // try all the config files one by one for _, configFile := range files { t.Run(configFile, func(t *testing.T) { - cfg, err := config.LoadFile(afero.NewOsFs(), configFile, "") + cfg, err := config.LoadFile(configFile, "") require.NoError(t, err) require.NotNil(t, cfg) diff --git a/main.go b/main.go index cc40953eb..1ea5eedfa 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,6 @@ import ( "github.com/creativeprojects/resticprofile/term" "github.com/creativeprojects/resticprofile/util/shutdown" "github.com/mackerelio/go-osstat/memory" - "github.com/spf13/afero" "github.com/spf13/pflag" ) @@ -301,15 +300,13 @@ func banner() { } func loadConfig(flags commandLineFlags, silent bool) (cfg *config.Config, global *config.Global, err error) { - fs := afero.NewOsFs() - var configFile string if configFile, err = filesearch.NewFinder().FindConfigurationFile(flags.config); err == nil { if configFile != flags.config && !silent { clog.Infof("using configuration file: %s", configFile) } - if cfg, err = config.LoadFile(fs, configFile, flags.format); err == nil { + if cfg, err = config.LoadFile(configFile, flags.format); err == nil { global, err = cfg.GetGlobalSection() if err != nil { err = fmt.Errorf("cannot load global configuration: %w", err) From a2be9011e1fe1b76993a1f8f3dcdf334fb950616 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 16:28:33 +0100 Subject: [PATCH 04/24] refactor: update remote configuration handling to use context and improve error responses --- main.go | 4 +- remote.go | 18 +++-- remote/tar.go | 13 +++- remote/tar_test.go | 170 +++++++++++++++++++++++++++++++++++++++++++++ send.go | 3 +- serve.go | 23 +++--- ssh/ssh.go | 11 ++- 7 files changed, 212 insertions(+), 30 deletions(-) create mode 100644 remote/tar_test.go diff --git a/main.go b/main.go index 1ea5eedfa..b17767040 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "math/rand" @@ -139,7 +140,8 @@ func main() { banner() if flags.remote != "" { - closeFS, remoteParameters, err := setupRemoteConfiguration(flags.remote) + ctx := context.TODO() + closeFS, remoteParameters, err := setupRemoteConfiguration(ctx, flags.remote) if err != nil { // need to setup console logging to display the error message closeLogger := setupLogging(nil) diff --git a/remote.go b/remote.go index bade3953b..1f31fb420 100644 --- a/remote.go +++ b/remote.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "math/rand" "net/http" "os" "path/filepath" @@ -19,11 +18,11 @@ import ( "github.com/creativeprojects/resticprofile/remote" ) -func loadRemoteFiles(endpoint string) ([]fuse.File, *remote.Manifest, error) { +func loadRemoteFiles(ctx context.Context, endpoint string) ([]fuse.File, *remote.Manifest, error) { var parameters *remote.Manifest client := http.DefaultClient - request, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, endpoint, http.NoBody) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) if err != nil { return nil, nil, fmt.Errorf("failed to create request: %w", err) } @@ -92,8 +91,8 @@ func getManifestParameters(reader io.Reader) (*remote.Manifest, error) { } // setupRemoteConfiguration downloads the configuration files from the remote endpoint and mounts the virtual FS -func setupRemoteConfiguration(remoteEndpoint string) (func(), *remote.Manifest, error) { - files, parameters, err := loadRemoteFiles(remoteEndpoint) +func setupRemoteConfiguration(ctx context.Context, remoteEndpoint string) (func(), *remote.Manifest, error) { + files, parameters, err := loadRemoteFiles(ctx, remoteEndpoint) if err != nil { return nil, nil, err } @@ -102,7 +101,10 @@ func setupRemoteConfiguration(remoteEndpoint string) (func(), *remote.Manifest, mountpoint := parameters.Mountpoint if mountpoint == "" { // generates a temporary directory - mountpoint = filepath.Join(os.TempDir(), fmt.Sprintf("%s-%x", "resticprofile", rand.Uint32())) + mountpoint, err = os.MkdirTemp("", "resticprofile-") + if err != nil { + return nil, parameters, fmt.Errorf("failed to create mount directory: %w", err) + } closeMountpoint = func() { err = os.Remove(mountpoint) if err != nil { @@ -110,10 +112,6 @@ func setupRemoteConfiguration(remoteEndpoint string) (func(), *remote.Manifest, } } } - err = os.MkdirAll(mountpoint, 0o755) - if err != nil { - return nil, parameters, fmt.Errorf("failed to create mount directory: %w", err) - } closeFs, err := fuse.MountFS(mountpoint, files) if err != nil { diff --git a/remote/tar.go b/remote/tar.go index 08938c09a..5cfd3357a 100644 --- a/remote/tar.go +++ b/remote/tar.go @@ -13,17 +13,24 @@ import ( type Tar struct { writer *tar.Writer + fs afero.Fs } func NewTar(w io.Writer) *Tar { return &Tar{ writer: tar.NewWriter(w), + fs: afero.NewOsFs(), } } -func (t *Tar) SendFiles(fs afero.Fs, files []string) error { +func (t *Tar) WithFs(fs afero.Fs) *Tar { + t.fs = fs + return t +} + +func (t *Tar) SendFiles(files []string) error { for _, filename := range files { - fileInfo, err := fs.Stat(filename) + fileInfo, err := t.fs.Stat(filename) if err != nil { clog.Errorf("unable to stat file %s: %v", filename, err) continue @@ -38,7 +45,7 @@ func (t *Tar) SendFiles(fs afero.Fs, files []string) error { clog.Errorf("unable to write tar header for file %s: %v", filename, err) break } - file, err := fs.Open(filename) + file, err := t.fs.Open(filename) if err != nil { clog.Errorf("unable to open file %s: %v", filename, err) continue diff --git a/remote/tar_test.go b/remote/tar_test.go new file mode 100644 index 000000000..6961960fe --- /dev/null +++ b/remote/tar_test.go @@ -0,0 +1,170 @@ +package remote + +import ( + "archive/tar" + "bytes" + "io" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSendFile(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fileName string + data []byte + }{ + { + name: "send empty file", + fileName: "empty.txt", + data: []byte{}, + }, + { + name: "send file with content", + fileName: "test.txt", + data: []byte("test content"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup + buf := new(bytes.Buffer) + tar := NewTar(buf) + + // Execute + err := tar.SendFile(tt.fileName, tt.data) + require.NoError(t, err) + tar.Close() + + // Verify the tar contains the correct file + fs := afero.NewMemMapFs() + err = extractTarToFs(buf.Bytes(), fs) + assert.NoError(t, err) + + // Check file exists and has correct content + fileExists, err := afero.Exists(fs, tt.fileName) + assert.NoError(t, err) + assert.True(t, fileExists) + + content, err := afero.ReadFile(fs, tt.fileName) + assert.NoError(t, err) + assert.Equal(t, tt.data, content) + }) + } +} + +func TestSendFiles(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + files map[string]string + filePaths []string + }{ + { + name: "send multiple files", + files: map[string]string{"file1.txt": "content1", "file2.txt": "content2"}, + filePaths: []string{"file1.txt", "file2.txt"}, + }, + { + name: "send empty file among others", + files: map[string]string{"empty.txt": "", "notempty.txt": "content"}, + filePaths: []string{"empty.txt", "notempty.txt"}, + }, + { + name: "send non-existent file", + files: map[string]string{"exists.txt": "content"}, + filePaths: []string{"exists.txt", "nonexistent.txt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup + buf := new(bytes.Buffer) + tar := NewTar(buf) + + // Create a memory filesystem with the test files + memFs := afero.NewMemMapFs() + for name, content := range tt.files { + err := afero.WriteFile(memFs, name, []byte(content), 0644) + assert.NoError(t, err) + } + + tar.WithFs(memFs) + + // Execute + err := tar.SendFiles(tt.filePaths) + require.NoError(t, err) + tar.Close() + + // Verify the tar contains the correct files + outputFs := afero.NewMemMapFs() + err = extractTarToFs(buf.Bytes(), outputFs) + assert.NoError(t, err) + + // Check each expected file exists and has correct content + for name, expectedContent := range tt.files { + // Only check files that were in the filePaths list + included := false + for _, path := range tt.filePaths { + if path == name { + included = true + break + } + } + + if !included { + continue + } + + fileExists, err := afero.Exists(outputFs, name) + assert.NoError(t, err) + + if _, ok := tt.files[name]; ok { + assert.True(t, fileExists) + + content, err := afero.ReadFile(outputFs, name) + assert.NoError(t, err) + assert.Equal(t, []byte(expectedContent), content) + } + } + }) + } +} + +// Helper function to extract tar contents to an afero filesystem +func extractTarToFs(tarData []byte, fs afero.Fs) error { + reader := bytes.NewReader(tarData) + tr := tar.NewReader(reader) + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + switch header.Typeflag { + case tar.TypeReg: + file, err := fs.Create(header.Name) + if err != nil { + return err + } + if _, err := io.CopyN(file, tr, 1000); err != nil && err != io.EOF { + file.Close() + return err + } + file.Close() + } + } + return nil +} diff --git a/send.go b/send.go index 7823751e7..793dd2b87 100644 --- a/send.go +++ b/send.go @@ -10,7 +10,6 @@ import ( "github.com/creativeprojects/resticprofile/constants" "github.com/creativeprojects/resticprofile/remote" "github.com/creativeprojects/resticprofile/ssh" - "github.com/spf13/afero" ) func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { @@ -46,7 +45,7 @@ func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { tar := remote.NewTar(resp) defer tar.Close() - _ = tar.SendFiles(afero.NewOsFs(), append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) + _ = tar.SendFiles(append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) _ = tar.SendFile(constants.ManifestFilename, manifestData) }) cnx := ssh.NewSSH(ssh.Config{ diff --git a/serve.go b/serve.go index 4e6aa441c..95a53ff94 100644 --- a/serve.go +++ b/serve.go @@ -14,7 +14,6 @@ import ( "github.com/creativeprojects/clog" "github.com/creativeprojects/resticprofile/constants" "github.com/creativeprojects/resticprofile/remote" - "github.com/spf13/afero" ) func serveCommand(w io.Writer, cmdCtx commandContext) error { @@ -25,16 +24,12 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { handler.HandleFunc("GET /configuration/{remote}", func(resp http.ResponseWriter, req *http.Request) { remoteName := req.PathValue("remote") if !cmdCtx.config.HasRemote(remoteName) { - resp.Header().Set("Content-Type", "text/plain") - resp.WriteHeader(http.StatusNotFound) - _, _ = resp.Write([]byte("remote not found")) + sendError(resp, http.StatusNotFound, fmt.Errorf("remote %q not found", remoteName)) return } remoteConfig, err := cmdCtx.config.GetRemote(remoteName) if err != nil { - resp.Header().Set("Content-Type", "text/plain") - resp.WriteHeader(http.StatusBadRequest) - _, _ = resp.Write([]byte(err.Error())) + sendError(resp, http.StatusBadRequest, fmt.Errorf("error while getting remote configuration: %w", err)) return } @@ -45,9 +40,7 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { } manifestData, err := json.Marshal(manifest) if err != nil { - resp.Header().Set("Content-Type", "text/plain") - resp.WriteHeader(http.StatusInternalServerError) - _, _ = resp.Write([]byte(err.Error())) + sendError(resp, http.StatusInternalServerError, fmt.Errorf("error while generating manifest: %w", err)) return } @@ -57,7 +50,7 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { tar := remote.NewTar(resp) defer tar.Close() - _ = tar.SendFiles(afero.NewOsFs(), append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) + _ = tar.SendFiles(append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) _ = tar.SendFile(constants.ManifestFilename, manifestData) }) @@ -95,3 +88,11 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { } return nil } + +func sendError(resp http.ResponseWriter, status int, err error) { + resp.Header().Set("Content-Type", "text/plain") + resp.WriteHeader(status) + _, _ = resp.Write([]byte(err.Error())) + _, _ = resp.Write([]byte("\n")) + clog.Error(err) +} diff --git a/ssh/ssh.go b/ssh/ssh.go index 2183ff687..ee4603dfd 100644 --- a/ssh/ssh.go +++ b/ssh/ssh.go @@ -37,9 +37,14 @@ func (s *SSH) Connect() error { if err != nil { return err } - hostKeyCallback, err := knownhosts.New(s.config.KnownHostsPath) - if err != nil { - return fmt.Errorf("cannot load host keys from known_hosts: %w", err) + var hostKeyCallback ssh.HostKeyCallback = func(hostname string, remote net.Addr, key ssh.PublicKey) error { + return nil + } + if s.config.KnownHostsPath != "" && s.config.KnownHostsPath != "none" && s.config.KnownHostsPath != "/dev/null" { + hostKeyCallback, err = knownhosts.New(s.config.KnownHostsPath) + if err != nil { + return fmt.Errorf("cannot load host keys from known_hosts: %w", err) + } } key, err := os.ReadFile(s.config.PrivateKeyPath) if err != nil { From 5f5d522f202511afdda2072db2947816463cd413 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 17:42:40 +0100 Subject: [PATCH 05/24] refactor: restructure sendProfileCommand and extract sendRemoteFiles for better code organization --- send.go | 78 ----------------------------------------------- serve.go | 92 ++++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 72 insertions(+), 98 deletions(-) delete mode 100644 send.go diff --git a/send.go b/send.go deleted file mode 100644 index 793dd2b87..000000000 --- a/send.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "path" - - "github.com/creativeprojects/resticprofile/constants" - "github.com/creativeprojects/resticprofile/remote" - "github.com/creativeprojects/resticprofile/ssh" -) - -func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { - if len(cmdCtx.flags.resticArgs) < 2 { - return fmt.Errorf("missing argument: remote name") - } - remoteName := cmdCtx.flags.resticArgs[1] - if !cmdCtx.config.HasRemote(remoteName) { - return fmt.Errorf("remote not found") - } - remoteConfig, err := cmdCtx.config.GetRemote(remoteName) - if err != nil { - return err - } - // send the files to the remote using tar - handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - // prepare manifest file - manifest := remote.Manifest{ - ConfigurationFile: path.Base(remoteConfig.ConfigurationFile), // need to take file path into consideration - ProfileName: remoteConfig.ProfileName, - CommandLineArguments: cmdCtx.flags.resticArgs[2:], - } - manifestData, err := json.Marshal(manifest) - if err != nil { - resp.Header().Set("Content-Type", "text/plain") - resp.WriteHeader(http.StatusInternalServerError) - _, _ = resp.Write([]byte(err.Error())) - return - } - - resp.Header().Set("Content-Type", "application/x-tar") - resp.WriteHeader(http.StatusOK) - - tar := remote.NewTar(resp) - defer tar.Close() - _ = tar.SendFiles(append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) - _ = tar.SendFile(constants.ManifestFilename, manifestData) - }) - cnx := ssh.NewSSH(ssh.Config{ - Host: remoteConfig.Host, - Username: remoteConfig.Username, - PrivateKeyPath: remoteConfig.PrivateKeyPath, - KnownHostsPath: remoteConfig.KnownHostsPath, - Handler: handler, - }) - defer cnx.Close() - - err = cnx.Connect() - if err != nil { - return err - } - binaryPath := remoteConfig.BinaryPath - if binaryPath == "" { - binaryPath = "resticprofile" - } - commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", - binaryPath, - cnx.TunnelPort(), - remoteName, - ) - err = cnx.Run(commandLine) - if err != nil { - return fmt.Errorf("failed to run command %q: %w", commandLine, err) - } - return nil -} diff --git a/serve.go b/serve.go index 95a53ff94..e5639cc1f 100644 --- a/serve.go +++ b/serve.go @@ -12,8 +12,10 @@ import ( "time" "github.com/creativeprojects/clog" + "github.com/creativeprojects/resticprofile/config" "github.com/creativeprojects/resticprofile/constants" "github.com/creativeprojects/resticprofile/remote" + "github.com/creativeprojects/resticprofile/ssh" ) func serveCommand(w io.Writer, cmdCtx commandContext) error { @@ -33,26 +35,7 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { return } - // prepare manifest file - manifest := remote.Manifest{ - ConfigurationFile: path.Base(remoteConfig.ConfigurationFile), // need to take file path into consideration - ProfileName: remoteConfig.ProfileName, - } - manifestData, err := json.Marshal(manifest) - if err != nil { - sendError(resp, http.StatusInternalServerError, fmt.Errorf("error while generating manifest: %w", err)) - return - } - - clog.Debugf("sending configuration for %q", remoteName) - resp.Header().Set("Content-Type", "application/x-tar") - resp.WriteHeader(http.StatusOK) - - tar := remote.NewTar(resp) - defer tar.Close() - _ = tar.SendFiles(append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) - _ = tar.SendFile(constants.ManifestFilename, manifestData) - + sendRemoteFiles(remoteConfig, remoteName, nil, resp) }) quit := make(chan os.Signal, 1) @@ -89,6 +72,75 @@ func serveCommand(w io.Writer, cmdCtx commandContext) error { return nil } +func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { + if len(cmdCtx.flags.resticArgs) < 2 { + return fmt.Errorf("missing argument: remote name") + } + remoteName := cmdCtx.flags.resticArgs[1] + if !cmdCtx.config.HasRemote(remoteName) { + return fmt.Errorf("remote not found") + } + remoteConfig, err := cmdCtx.config.GetRemote(remoteName) + if err != nil { + return err + } + // send the files to the remote using tar + handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + sendRemoteFiles(remoteConfig, remoteName, cmdCtx.flags.resticArgs[2:], resp) + }) + cnx := ssh.NewSSH(ssh.Config{ + Host: remoteConfig.Host, + Username: remoteConfig.Username, + PrivateKeyPath: remoteConfig.PrivateKeyPath, + KnownHostsPath: remoteConfig.KnownHostsPath, + Handler: handler, + }) + defer cnx.Close() + + err = cnx.Connect() + if err != nil { + return err + } + binaryPath := remoteConfig.BinaryPath + if binaryPath == "" { + binaryPath = "resticprofile" + } + commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", + binaryPath, + cnx.TunnelPort(), + remoteName, + ) + err = cnx.Run(commandLine) + if err != nil { + return fmt.Errorf("failed to run command %q: %w", commandLine, err) + } + return nil +} + +func sendRemoteFiles(remoteConfig *config.Remote, remoteName string, extraArgs []string, resp http.ResponseWriter) { + // prepare manifest file + manifest := remote.Manifest{ + Version: version, + ConfigurationFile: path.Base(remoteConfig.ConfigurationFile), // need to take file path into consideration + ProfileName: remoteConfig.ProfileName, + CommandLineArguments: extraArgs, + } + manifestData, err := json.Marshal(manifest) + if err != nil { + sendError(resp, http.StatusInternalServerError, fmt.Errorf("error while generating manifest: %w", err)) + return + } + + clog.Debugf("sending configuration for %q", remoteName) + resp.Header().Set("Content-Type", "application/x-tar") + resp.WriteHeader(http.StatusOK) + + tar := remote.NewTar(resp) + defer tar.Close() + _ = tar.SendFiles(append(remoteConfig.SendFiles, remoteConfig.ConfigurationFile)) + _ = tar.SendFile(constants.ManifestFilename, manifestData) +} + func sendError(resp http.ResponseWriter, status int, err error) { resp.Header().Set("Content-Type", "text/plain") resp.WriteHeader(status) From 2b05ae240fd22f660d7709be0967e4c572a0715d Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 5 Aug 2025 17:50:05 +0100 Subject: [PATCH 06/24] add simple test on sendRemoteFiles --- serve_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 serve_test.go diff --git a/serve_test.go b/serve_test.go new file mode 100644 index 000000000..d8f68c890 --- /dev/null +++ b/serve_test.go @@ -0,0 +1,19 @@ +package main + +import ( + "net/http/httptest" + "testing" + + "github.com/creativeprojects/resticprofile/config" + "github.com/stretchr/testify/assert" +) + +func TestSendRemoteFiles(t *testing.T) { + recorder := httptest.NewRecorder() + sendRemoteFiles(&config.Remote{ + ConfigurationFile: "test_config.json", + ProfileName: "test_profile", + }, "test_remote", []string{"arg1", "arg2"}, recorder) + assert.Equal(t, recorder.Code, 200) + assert.Equal(t, recorder.Header().Get("Content-Type"), "application/x-tar") +} From d976130967acbb0082adc24a62cee770b611e44b Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 6 Aug 2025 17:49:19 +0100 Subject: [PATCH 07/24] start using openssh client instead of home-made one --- config/remote.go | 2 ++ serve.go | 86 ++++++++++++++++++++++++++++++++++++++---------- ssh/config.go | 1 + ssh/ssh.go | 14 +++++++- 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/config/remote.go b/config/remote.go index dce0e5def..a8d70fba4 100644 --- a/config/remote.go +++ b/config/remote.go @@ -12,6 +12,7 @@ type Remote struct { ConfigurationFile string `mapstructure:"configuration-file" description:"Path to the configuration file to transfer to the remote client"` ProfileName string `mapstructure:"profile-name" description:"Name of the profile to use on the remote client"` SendFiles []string `mapstructure:"send-files" description:"Other configuration files to transfer to the remote client"` + SSHConfig string `mapstructure:"ssh-config" description:"Path to the OpenSSH config file to use for the connection"` } func NewRemote(config *Config, name string) *Remote { @@ -27,6 +28,7 @@ func (r *Remote) SetRootPath(rootPath string) { r.PrivateKeyPath = fixPath(r.PrivateKeyPath, expandEnv, absolutePrefix(rootPath)) r.KnownHostsPath = fixPath(r.KnownHostsPath, expandEnv, absolutePrefix(rootPath)) r.ConfigurationFile = fixPath(r.ConfigurationFile, expandEnv, absolutePrefix(rootPath)) + r.SSHConfig = fixPath(r.SSHConfig, expandEnv, absolutePrefix(rootPath)) for i := range r.SendFiles { r.SendFiles[i] = fixPath(r.SendFiles[i], expandEnv, absolutePrefix(rootPath)) diff --git a/serve.go b/serve.go index e5639cc1f..418e92905 100644 --- a/serve.go +++ b/serve.go @@ -5,10 +5,13 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "os" + "os/exec" "os/signal" "path" + "sync" "time" "github.com/creativeprojects/clog" @@ -88,31 +91,37 @@ func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { sendRemoteFiles(remoteConfig, remoteName, cmdCtx.flags.resticArgs[2:], resp) }) - cnx := ssh.NewSSH(ssh.Config{ + sshConfig := ssh.Config{ Host: remoteConfig.Host, Username: remoteConfig.Username, PrivateKeyPath: remoteConfig.PrivateKeyPath, KnownHostsPath: remoteConfig.KnownHostsPath, + SSHConfigPath: remoteConfig.SSHConfig, Handler: handler, - }) - defer cnx.Close() - - err = cnx.Connect() - if err != nil { - return err - } - binaryPath := remoteConfig.BinaryPath - if binaryPath == "" { - binaryPath = "resticprofile" } - commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", - binaryPath, - cnx.TunnelPort(), - remoteName, - ) - err = cnx.Run(commandLine) + // cnx := ssh.NewSSH(sshConfig) + // defer cnx.Close() + + // err = cnx.Connect() + // if err != nil { + // return err + // } + // binaryPath := remoteConfig.BinaryPath + // if binaryPath == "" { + // binaryPath = "resticprofile" + // } + // commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", + // binaryPath, + // cnx.TunnelPort(), + // remoteName, + // ) + // err = cnx.Run(commandLine) + // if err != nil { + // return fmt.Errorf("failed to run command %q: %w", commandLine, err) + // } + err = callOpenSSH(context.Background(), sshConfig, handler) if err != nil { - return fmt.Errorf("failed to run command %q: %w", commandLine, err) + return err } return nil } @@ -148,3 +157,44 @@ func sendError(resp http.ResponseWriter, status int, err error) { _, _ = resp.Write([]byte("\n")) clog.Error(err) } + +func callOpenSSH(ctx context.Context, config ssh.Config, handler http.HandlerFunc) error { + var wg sync.WaitGroup + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + return err + } + server := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + } + wg.Add(1) + go func() { + defer wg.Done() + defer ln.Close() + err := server.Serve(ln) + if err != nil && err != http.ErrServerClosed { + clog.Error("error while serving HTTP:", err) + } + }() + ctx = context.WithoutCancel(ctx) + err = server.Shutdown(ctx) + if err != nil { + return fmt.Errorf("error while shutting down server: %w", err) + } + cmd := exec.CommandContext(ctx, + "ssh", "-v", + "-F", config.SSHConfigPath, + "-R", fmt.Sprintf("0:localhost:%d", ln.Addr().(*net.TCPAddr).Port), + fmt.Sprintf("%s@%s", config.Username, config.Host), + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + clog.Debugf("running command: %s", cmd.String()) + err = cmd.Run() + if err != nil { + return fmt.Errorf("error while running ssh command: %w", err) + } + wg.Wait() + return nil +} diff --git a/ssh/config.go b/ssh/config.go index db8428527..99ced1a7e 100644 --- a/ssh/config.go +++ b/ssh/config.go @@ -13,6 +13,7 @@ type Config struct { Username string PrivateKeyPath string KnownHostsPath string + SSHConfigPath string // Path to the OpenSSH config file, if any Handler http.Handler } diff --git a/ssh/ssh.go b/ssh/ssh.go index ee4603dfd..a85edb4c7 100644 --- a/ssh/ssh.go +++ b/ssh/ssh.go @@ -38,6 +38,7 @@ func (s *SSH) Connect() error { return err } var hostKeyCallback ssh.HostKeyCallback = func(hostname string, remote net.Addr, key ssh.PublicKey) error { + clog.Debugf("Initiating SSH connection to %s", remote.String()) return nil } if s.config.KnownHostsPath != "" && s.config.KnownHostsPath != "none" && s.config.KnownHostsPath != "/dev/null" { @@ -57,6 +58,12 @@ func (s *SSH) Connect() error { return fmt.Errorf("unable to parse private key: %w", err) } + // The algorithms returned by ssh.SupportedAlgorithms() are different from + // the default ones and do not include algorithms that are considered + // insecure, such as those using SHA-1, returned by + // ssh.InsecureAlgorithms(). + algorithms := ssh.SupportedAlgorithms() + config := &ssh.ClientConfig{ User: s.config.Username, Auth: []ssh.AuthMethod{ @@ -64,7 +71,12 @@ func (s *SSH) Connect() error { ssh.PublicKeys(signer), }, HostKeyCallback: hostKeyCallback, - HostKeyAlgorithms: []string{ssh.KeyAlgoED25519, ssh.KeyAlgoECDSA256}, // we might need to make this configurable + HostKeyAlgorithms: algorithms.HostKeys, + Config: ssh.Config{ + KeyExchanges: algorithms.KeyExchanges, + Ciphers: algorithms.Ciphers, + MACs: algorithms.MACs, + }, } // Connect to the remote server and perform the SSH handshake. From 69485a5c8ebf8566c097b75dfe69414ba422f590 Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 6 Aug 2025 22:06:04 +0100 Subject: [PATCH 08/24] use openssh instead of go ssh library --- serve.go | 82 +++--------- ssh/client.go | 8 ++ ssh/{ssh.go => internal_client.go} | 19 +-- ssh/openssh_client.go | 200 +++++++++++++++++++++++++++++ ssh/parse_host.go | 21 +++ ssh/parse_host_test.go | 69 ++++++++++ 6 files changed, 326 insertions(+), 73 deletions(-) create mode 100644 ssh/client.go rename ssh/{ssh.go => internal_client.go} (89%) create mode 100644 ssh/openssh_client.go create mode 100644 ssh/parse_host.go create mode 100644 ssh/parse_host_test.go diff --git a/serve.go b/serve.go index 418e92905..4307057e5 100644 --- a/serve.go +++ b/serve.go @@ -5,13 +5,10 @@ import ( "encoding/json" "fmt" "io" - "net" "net/http" "os" - "os/exec" "os/signal" "path" - "sync" "time" "github.com/creativeprojects/clog" @@ -99,30 +96,26 @@ func sendProfileCommand(w io.Writer, cmdCtx commandContext) error { SSHConfigPath: remoteConfig.SSHConfig, Handler: handler, } - // cnx := ssh.NewSSH(sshConfig) - // defer cnx.Close() - - // err = cnx.Connect() - // if err != nil { - // return err - // } - // binaryPath := remoteConfig.BinaryPath - // if binaryPath == "" { - // binaryPath = "resticprofile" - // } - // commandLine := fmt.Sprintf("%s -v -r http://localhost:%d/configuration/%s ", - // binaryPath, - // cnx.TunnelPort(), - // remoteName, - // ) - // err = cnx.Run(commandLine) - // if err != nil { - // return fmt.Errorf("failed to run command %q: %w", commandLine, err) - // } - err = callOpenSSH(context.Background(), sshConfig, handler) + // cnx := ssh.NewInternalClient(sshConfig) + cnx := ssh.NewOpenSSHClient(sshConfig) + defer cnx.Close() + + err = cnx.Connect() if err != nil { return err } + binaryPath := remoteConfig.BinaryPath + if binaryPath == "" { + binaryPath = "resticprofile" + } + arguments := []string{ + "-v", + "-r", fmt.Sprintf("http://localhost:%d/configuration/%s", cnx.TunnelPeerPort(), remoteName), + } + err = cnx.Run(binaryPath, arguments...) + if err != nil { + return fmt.Errorf("failed to run resticprofile on peer: %w", err) + } return nil } @@ -157,44 +150,3 @@ func sendError(resp http.ResponseWriter, status int, err error) { _, _ = resp.Write([]byte("\n")) clog.Error(err) } - -func callOpenSSH(ctx context.Context, config ssh.Config, handler http.HandlerFunc) error { - var wg sync.WaitGroup - ln, err := net.Listen("tcp", "localhost:0") - if err != nil { - return err - } - server := &http.Server{ - Handler: handler, - ReadHeaderTimeout: 5 * time.Second, - } - wg.Add(1) - go func() { - defer wg.Done() - defer ln.Close() - err := server.Serve(ln) - if err != nil && err != http.ErrServerClosed { - clog.Error("error while serving HTTP:", err) - } - }() - ctx = context.WithoutCancel(ctx) - err = server.Shutdown(ctx) - if err != nil { - return fmt.Errorf("error while shutting down server: %w", err) - } - cmd := exec.CommandContext(ctx, - "ssh", "-v", - "-F", config.SSHConfigPath, - "-R", fmt.Sprintf("0:localhost:%d", ln.Addr().(*net.TCPAddr).Port), - fmt.Sprintf("%s@%s", config.Username, config.Host), - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - clog.Debugf("running command: %s", cmd.String()) - err = cmd.Run() - if err != nil { - return fmt.Errorf("error while running ssh command: %w", err) - } - wg.Wait() - return nil -} diff --git a/ssh/client.go b/ssh/client.go new file mode 100644 index 000000000..3f1b9bb84 --- /dev/null +++ b/ssh/client.go @@ -0,0 +1,8 @@ +package ssh + +type Client interface { + Connect() error + Close() + Run(command string, arguments ...string) error + TunnelPeerPort() int +} diff --git a/ssh/ssh.go b/ssh/internal_client.go similarity index 89% rename from ssh/ssh.go rename to ssh/internal_client.go index a85edb4c7..b9dc314f1 100644 --- a/ssh/ssh.go +++ b/ssh/internal_client.go @@ -17,7 +17,7 @@ import ( const startPort = 10001 -type SSH struct { +type InternalClient struct { config Config port int client *ssh.Client @@ -25,20 +25,20 @@ type SSH struct { server *http.Server } -func NewSSH(config Config) *SSH { - return &SSH{ +func NewInternalClient(config Config) *InternalClient { + return &InternalClient{ config: config, port: startPort, } } -func (s *SSH) Connect() error { +func (s *InternalClient) Connect() error { err := s.config.Validate() if err != nil { return err } var hostKeyCallback ssh.HostKeyCallback = func(hostname string, remote net.Addr, key ssh.PublicKey) error { - clog.Debugf("Initiating SSH connection to %s", remote.String()) + clog.Debugf("initiating SSH connection to %s using internal client", remote.String()) return nil } if s.config.KnownHostsPath != "" && s.config.KnownHostsPath != "none" && s.config.KnownHostsPath != "/dev/null" { @@ -106,11 +106,11 @@ func (s *SSH) Connect() error { return nil } -func (s *SSH) TunnelPort() int { +func (s *InternalClient) TunnelPeerPort() int { return s.port } -func (s *SSH) Run(command string) error { +func (s *InternalClient) Run(command string, arguments ...string) error { // Each ClientConn can support multiple interactive sessions, // represented by a Session. session, err := s.client.NewSession() @@ -139,7 +139,7 @@ func (s *SSH) Run(command string) error { return nil } -func (s *SSH) Close() { +func (s *InternalClient) Close() { // close the tunnel first otherwise it fails with error: "ssh: cancel-tcpip-forward failed" if s.tunnel != nil { err := s.tunnel.Close() @@ -162,3 +162,6 @@ func (s *SSH) Close() { } } } + +// verify interface +var _ Client = (*InternalClient)(nil) diff --git a/ssh/openssh_client.go b/ssh/openssh_client.go new file mode 100644 index 000000000..5f6cf9296 --- /dev/null +++ b/ssh/openssh_client.go @@ -0,0 +1,200 @@ +package ssh + +import ( + "bytes" + "context" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/creativeprojects/clog" +) + +type OpenSSHClient struct { + config Config + sshHost string + sshUserHost string + sshPort int + listener net.Listener + server *http.Server + wg sync.WaitGroup + socket string + peerTunnelPort int +} + +func NewOpenSSHClient(config Config) *OpenSSHClient { + return &OpenSSHClient{ + config: config, + } +} + +// Connect establishes the SSH connection and starts the file server. +// It returns an error if the connection or server setup fails. +func (c *OpenSSHClient) Connect() error { + c.sshHost, c.sshPort = parseHost(c.config.Host) + c.sshUserHost = c.sshHost + if c.config.Username != "" { + c.sshUserHost = fmt.Sprintf("%s@%s", c.config.Username, c.sshHost) + } + err := c.startFileServer() + if err != nil { + return err + } + err = c.startSSH(context.Background()) + if err != nil { + return fmt.Errorf("error while starting SSH connection: %w", err) + } + err = c.startTunnel(context.Background()) + if err != nil { + return fmt.Errorf("error while starting SSH tunnel: %w", err) + } + return nil +} + +func (c *OpenSSHClient) startFileServer() error { + var err error + c.listener, err = net.Listen("tcp", "localhost:0") + if err != nil { + return err + } + c.server = &http.Server{ + Handler: c.config.Handler, + ReadHeaderTimeout: 5 * time.Second, + } + c.wg.Add(1) + go func() { + defer c.wg.Done() + defer c.listener.Close() + + clog.Debugf("file server listening locally on %s", c.listener.Addr().String()) + err := c.server.Serve(c.listener) + if err != nil && err != http.ErrServerClosed { + clog.Error("error while serving HTTP:", err) + } + }() + return nil +} + +func (c *OpenSSHClient) startSSH(ctx context.Context) error { + c.socket = filepath.Join(os.TempDir(), fmt.Sprintf("ssh-%d.sock", os.Getpid())) + args := make([]string, 0, 10) + args = append(args, + "-f", // Requests ssh to go to background just before command execution + "-M", // Places the ssh client into “master” mode for connection sharing + "-N", // Do not execute a remote command + "-S", c.socket, // Specifies the location of the control socket + ) + if c.config.SSHConfigPath != "" { + args = append(args, "-F", c.config.SSHConfigPath) + } + args = append(args, c.sshUserHost) + cmd := exec.CommandContext(ctx, "ssh", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + clog.Debugf("running command: %s", cmd.String()) + err := cmd.Run() + if err != nil { + return fmt.Errorf("error while running ssh command: %w", err) + } + return nil +} + +func (c *OpenSSHClient) stopSSH(ctx context.Context) error { + if c.socket == "" { + return nil + } + args := []string{ + "-S", c.socket, // Specifies the location of the control socket + "-O", "exit", // Requests the master to exit + c.sshUserHost, // Not used in this case, but required by ssh + } + cmd := exec.CommandContext(ctx, "ssh", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + clog.Debugf("running command: %s", cmd.String()) + err := cmd.Run() + if err != nil { + return fmt.Errorf("error while running ssh command: %w", err) + } + return nil +} + +func (c *OpenSSHClient) startTunnel(ctx context.Context) error { + if c.socket == "" { + return nil + } + args := []string{ + "-S", c.socket, // Specifies the location of the control socket + "-O", "forward", // Requests the master to exit + fmt.Sprintf("-R 0:localhost:%d", c.listener.Addr().(*net.TCPAddr).Port), // Forward random remote port to local port + c.sshUserHost, // Not used in this case, but required by ssh + } + cmd := exec.CommandContext(ctx, "ssh", args...) + clog.Debugf("running command: %s", cmd.String()) + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("error while running ssh command: %w", err) + } + if len(output) == 0 { + return fmt.Errorf("no output from SSH tunnel command") + } + output = bytes.TrimSpace(output) + port, err := strconv.Atoi(string(output)) + if err != nil { + return fmt.Errorf("error parsing SSH tunnel output: %w", err) + } + c.peerTunnelPort = port + clog.Debugf("port %d opened in tunnel", c.peerTunnelPort) + return nil +} + +func (c *OpenSSHClient) Close() { + ctx := context.Background() + if c.server != nil { + err := c.server.Shutdown(ctx) + if err != nil { + clog.Warningf("unable to shutdown server: %s", err) + } + c.server = nil + } + err := c.stopSSH(ctx) + if err != nil { + clog.Warningf("unable to stop SSH connection: %s", err) + } + c.wg.Wait() +} + +func (c *OpenSSHClient) Run(command string, arguments ...string) error { + if c.socket == "" { + return nil + } + args := append([]string{ + "-t", // Force pseudo-terminal allocation + "-t", // Even when stdin is not attached + "-S", c.socket, // Specifies the location of the control socket + c.sshUserHost, // Not used in this case, but required by ssh + command, + }, arguments...) + cmd := exec.CommandContext(context.Background(), "ssh", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + clog.Debugf("running command: %s", cmd.String()) + err := cmd.Run() + if err != nil { + return fmt.Errorf("error while running ssh command: %w", err) + } + return nil +} + +func (c *OpenSSHClient) TunnelPeerPort() int { + return c.peerTunnelPort +} + +// verify interface +var _ Client = (*OpenSSHClient)(nil) diff --git a/ssh/parse_host.go b/ssh/parse_host.go new file mode 100644 index 000000000..17c264162 --- /dev/null +++ b/ssh/parse_host.go @@ -0,0 +1,21 @@ +package ssh + +import ( + "strconv" + "strings" +) + +func parseHost(host string) (string, int) { + if strings.Contains(host, ":") { + parts := strings.Split(host, ":") + if len(parts) > 2 { + // If there are more than two parts, we assume the first part is the host and the rest is the port + host = strings.Join(parts[:len(parts)-1], ":") + port, _ := strconv.Atoi(parts[len(parts)-1]) + return host, port + } + port, _ := strconv.Atoi(parts[1]) + return parts[0], port + } + return host, 0 +} diff --git a/ssh/parse_host_test.go b/ssh/parse_host_test.go new file mode 100644 index 000000000..f4974db81 --- /dev/null +++ b/ssh/parse_host_test.go @@ -0,0 +1,69 @@ +package ssh + +import ( + "testing" +) + +func TestParseHost(t *testing.T) { + tests := []struct { + name string + host string + wantHost string + wantPort int + }{ + { + name: "host only", + host: "example.com", + wantHost: "example.com", + wantPort: 0, + }, + { + name: "host with port", + host: "example.com:22", + wantHost: "example.com", + wantPort: 22, + }, + { + name: "IPv4 with port", + host: "192.168.1.1:2222", + wantHost: "192.168.1.1", + wantPort: 2222, + }, + { + name: "IPv6 with port", + host: "[2001:db8::1]:22", + wantHost: "[2001:db8::1]", + wantPort: 22, + }, + { + name: "IPv6 without brackets with port", + host: "2001:db8::1:22", + wantHost: "2001:db8::1", + wantPort: 22, + }, + { + name: "host with multiple colons", + host: "user:pass@example.com:22", + wantHost: "user:pass@example.com", + wantPort: 22, + }, + { + name: "invalid port", + host: "example.com:abc", + wantHost: "example.com", + wantPort: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHost, gotPort := parseHost(tt.host) + if gotHost != tt.wantHost { + t.Errorf("parseHost() host = %v, want %v", gotHost, tt.wantHost) + } + if gotPort != tt.wantPort { + t.Errorf("parseHost() port = %v, want %v", gotPort, tt.wantPort) + } + }) + } +} From 8b8f9fc277afc9882025f02460688e6566ec5410 Mon Sep 17 00:00:00 2001 From: Fred Date: Thu, 7 Aug 2025 21:54:38 +0100 Subject: [PATCH 09/24] feat: add SSH client interface implementation and testing framework --- .gitignore | 2 ++ Makefile | 40 ++++++++++++++++++++++++++++++++++++ ssh/client.go | 1 + ssh/internal_client.go | 4 ++++ ssh/openssh_client.go | 10 +++++++++ ssh/ssh_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+) create mode 100644 ssh/ssh_test.go diff --git a/.gitignore b/.gitignore index bc2543301..aa53cac96 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ go.work.sum /*.txt output.xml + +/ssh/tests/ diff --git a/Makefile b/Makefile index 15f1a00d4..de20dec0b 100644 --- a/Makefile +++ b/Makefile @@ -336,3 +336,43 @@ deploy-current: build-linux build-pi rsync -avz --progress $(BINARY_PI) $$server: ; \ ssh $$server "sudo -S install $(BINARY_PI) /usr/local/bin/resticprofile" ; \ done + +.PHONY: start-ssh-server +start-ssh-server: + @echo "[*] $@" + $(eval KEYS_TMP_DIR := $(shell mktemp --directory --tmpdir resticprofile.XXXXXXXXXX)) + @echo "Using temporary directory for SSH keys: $(KEYS_TMP_DIR)" + @echo $(KEYS_TMP_DIR) > ./ssh/tests/running.tmpdir + @ssh-keygen -t rsa -b 2048 -f $(KEYS_TMP_DIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)" + @cp $(KEYS_TMP_DIR)/id_rsa.pub ./ssh/tests/authorized_keys + @docker run -d \ + --name=openssh-server \ + --hostname=openssh-server \ + -e PUID=1000 \ + -e PGID=1000 \ + -e TZ=Etc/UTC \ + -e SUDO_ACCESS=false \ + -e PASSWORD_ACCESS=false \ + -e USER_NAME=resticprofile \ + -e LOG_STDOUT=false \ + -p 2222:2222 \ + -v ./ssh/tests/authorized_keys:/config/.ssh/authorized_keys \ + lscr.io/linuxserver/openssh-server:latest + @ssh-keyscan -p 2222 -H localhost > ./ssh/tests/known_hosts + +.PHONY: stop-ssh-server +stop-ssh-server: + @echo "[*] $@" + @docker stop openssh-server || echo "No running SSH server to stop" + @docker rm openssh-server || echo "No container to remove" + $(eval KEYS_TMP_DIR := $(shell cat ./ssh/tests/running.tmpdir)) + @test -d "$(KEYS_TMP_DIR)" && rm -rf "$(KEYS_TMP_DIR)" || echo "Failed to remove temporary SSH keys directory" + @test -f ./ssh/tests/running.tmpdir && rm ./ssh/tests/running.tmpdir || echo "Failed to remove temporary file" + @test -f ./ssh/tests/authorized_keys && rm ./ssh/tests/authorized_keys || echo "Failed to remove authorized_keys file" + @test -f ./ssh/tests/known_hosts && rm ./ssh/tests/known_hosts || echo "Failed to remove known_hosts file" + +known_hosts: + @echo "[*] $@" + $(eval KEYS_TMP_DIR := $(shell cat ./ssh/tests/running.tmpdir)) + @ssh-keyscan -p 2222 -H localhost > $(KEYS_TMP_DIR)/known_hosts + @echo "Known hosts file created at $(KEYS_TMP_DIR)/known_hosts" \ No newline at end of file diff --git a/ssh/client.go b/ssh/client.go index 3f1b9bb84..112078d34 100644 --- a/ssh/client.go +++ b/ssh/client.go @@ -1,6 +1,7 @@ package ssh type Client interface { + Name() string Connect() error Close() Run(command string, arguments ...string) error diff --git a/ssh/internal_client.go b/ssh/internal_client.go index b9dc314f1..7502b957c 100644 --- a/ssh/internal_client.go +++ b/ssh/internal_client.go @@ -32,6 +32,10 @@ func NewInternalClient(config Config) *InternalClient { } } +func (s *InternalClient) Name() string { + return "InternalSSH" +} + func (s *InternalClient) Connect() error { err := s.config.Validate() if err != nil { diff --git a/ssh/openssh_client.go b/ssh/openssh_client.go index 5f6cf9296..192d2eec3 100644 --- a/ssh/openssh_client.go +++ b/ssh/openssh_client.go @@ -34,6 +34,10 @@ func NewOpenSSHClient(config Config) *OpenSSHClient { } } +func (c *OpenSSHClient) Name() string { + return "OpenSSH" +} + // Connect establishes the SSH connection and starts the file server. // It returns an error if the connection or server setup fails. func (c *OpenSSHClient) Connect() error { @@ -93,6 +97,12 @@ func (c *OpenSSHClient) startSSH(ctx context.Context) error { if c.config.SSHConfigPath != "" { args = append(args, "-F", c.config.SSHConfigPath) } + if c.sshPort > 0 { + args = append(args, "-p", strconv.Itoa(c.sshPort)) // Specifies the port to connect to on the remote host + } + if c.config.KnownHostsPath != "" { + args = append(args, "-o", fmt.Sprintf("UserKnownHostsFile=%s", c.config.KnownHostsPath)) + } args = append(args, c.sshUserHost) cmd := exec.CommandContext(ctx, "ssh", args...) cmd.Stdout = os.Stdout diff --git a/ssh/ssh_test.go b/ssh/ssh_test.go new file mode 100644 index 000000000..a93755705 --- /dev/null +++ b/ssh/ssh_test.go @@ -0,0 +1,46 @@ +package ssh + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSSHClient(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + t.Logf("Current working directory: %s", wd) + + fixtures := []struct { + name string + config Config + connectErr bool + }{ + { + name: "no public key", + config: Config{ + Host: "localhost:2222", + Username: "resticprofile", + KnownHostsPath: filepath.Join(wd, "./tests/known_hosts"), + }, + connectErr: true, + }, + } + + for _, fixture := range fixtures { + for _, client := range []Client{NewOpenSSHClient(fixture.config), NewInternalClient(fixture.config)} { + t.Run(client.Name()+" "+fixture.name, func(t *testing.T) { + err := client.Connect() + if fixture.connectErr { + require.Error(t, err, "expected error for config: %v", fixture.config) + t.Log(err) + return + } + require.NoError(t, err, "unexpected error for config: %v", fixture.config) + }) + } + } +} From 62fb3ccc457edac3f31af81755dc700a4c209e09 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 17:40:34 +0100 Subject: [PATCH 10/24] prepare CI job to test ssh sessions --- .github/workflows/build.yml | 35 +++++++++++++++++++++++++++++++- .gitignore | 2 -- .vscode/settings.json | 3 ++- Makefile | 2 ++ ssh/openssh_client.go | 5 ++++- ssh/ssh_test.go | 30 +++++++++++++++++++++++++-- ssh/tests/ssh_gateway_ports.conf | 2 ++ 7 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 ssh/tests/ssh_gateway_ports.conf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9685fda85..1e8dc94d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,6 @@ jobs: GO: ${{ matrix.go_version }} steps: - - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -71,6 +70,40 @@ jobs: name: code-coverage-report-${{ matrix.os }} path: coverage.out + test-ssh: + name: Test SSH connections + runs-on: ubuntu-latest + + services: + openssh-server: + image: lscr.io/linuxserver/openssh-server:latest + ports: + - 2222:2222 + volumes: + - ${{ github.workspace }}/ssh/tests/ssh_gateway_ports.conf:/config/sshd/sshd_config.d/ssh_gateway_ports.conf + env: + PUID: 1000 + PGID: 1000 + TZ: Etc/UTC + SUDO_ACCESS: false + PASSWORD_ACCESS: false + USER_NAME: resticprofile + LOG_STDOUT: true + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24 + check-latest: true + cache: true + + - name: Download host keys + run: ssh-keyscan -p 2222 -H localhost > ${{ github.workspace }}/ssh/tests/known_hosts + sonarCloudTrigger: needs: build name: SonarCloud Trigger diff --git a/.gitignore b/.gitignore index aa53cac96..bc2543301 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,3 @@ go.work.sum /*.txt output.xml - -/ssh/tests/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 3af104a0f..cd83a24e9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,5 +18,6 @@ "go.lintTool": "golangci-lint", "githubPullRequests.ignoredPullRequestBranches": [ "master" - ] + ], + "go.buildTags": "ssh" } \ No newline at end of file diff --git a/Makefile b/Makefile index de20dec0b..919d84e4f 100644 --- a/Makefile +++ b/Makefile @@ -357,7 +357,9 @@ start-ssh-server: -e LOG_STDOUT=false \ -p 2222:2222 \ -v ./ssh/tests/authorized_keys:/config/.ssh/authorized_keys \ + -v ./ssh/tests/ssh_gateway_ports.conf:/config/sshd/sshd_config.d/ssh_gateway_ports.conf \ lscr.io/linuxserver/openssh-server:latest + @sleep 1 @ssh-keyscan -p 2222 -H localhost > ./ssh/tests/known_hosts .PHONY: stop-ssh-server diff --git a/ssh/openssh_client.go b/ssh/openssh_client.go index 192d2eec3..a9448a983 100644 --- a/ssh/openssh_client.go +++ b/ssh/openssh_client.go @@ -98,11 +98,14 @@ func (c *OpenSSHClient) startSSH(ctx context.Context) error { args = append(args, "-F", c.config.SSHConfigPath) } if c.sshPort > 0 { - args = append(args, "-p", strconv.Itoa(c.sshPort)) // Specifies the port to connect to on the remote host + args = append(args, "-p", strconv.Itoa(c.sshPort)) } if c.config.KnownHostsPath != "" { args = append(args, "-o", fmt.Sprintf("UserKnownHostsFile=%s", c.config.KnownHostsPath)) } + if c.config.PrivateKeyPath != "" { + args = append(args, "-i", c.config.PrivateKeyPath) + } args = append(args, c.sshUserHost) cmd := exec.CommandContext(ctx, "ssh", args...) cmd.Stdout = os.Stdout diff --git a/ssh/ssh_test.go b/ssh/ssh_test.go index a93755705..f7349cd69 100644 --- a/ssh/ssh_test.go +++ b/ssh/ssh_test.go @@ -1,3 +1,5 @@ +//go:build ssh + package ssh import ( @@ -14,6 +16,8 @@ func TestSSHClient(t *testing.T) { t.Logf("Current working directory: %s", wd) + tmpDir := os.Getenv("KEYS_TMP_DIR") + fixtures := []struct { name string config Config @@ -28,18 +32,40 @@ func TestSSHClient(t *testing.T) { }, connectErr: true, }, + { + name: "wrong username", + config: Config{ + Host: "localhost:2222", + Username: "otheruser", + KnownHostsPath: filepath.Join(wd, "tests/known_hosts"), + PrivateKeyPath: filepath.Join(tmpDir, "id_rsa"), + }, + connectErr: true, + }, + { + name: "successful connection", + config: Config{ + Host: "localhost:2222", + Username: "resticprofile", + KnownHostsPath: filepath.Join(wd, "tests/known_hosts"), + PrivateKeyPath: filepath.Join(tmpDir, "id_rsa"), + }, + connectErr: false, + }, } for _, fixture := range fixtures { for _, client := range []Client{NewOpenSSHClient(fixture.config), NewInternalClient(fixture.config)} { t.Run(client.Name()+" "+fixture.name, func(t *testing.T) { + defer client.Close() + err := client.Connect() if fixture.connectErr { - require.Error(t, err, "expected error for config: %v", fixture.config) + require.Error(t, err) t.Log(err) return } - require.NoError(t, err, "unexpected error for config: %v", fixture.config) + require.NoError(t, err) }) } } diff --git a/ssh/tests/ssh_gateway_ports.conf b/ssh/tests/ssh_gateway_ports.conf new file mode 100644 index 000000000..79af9a816 --- /dev/null +++ b/ssh/tests/ssh_gateway_ports.conf @@ -0,0 +1,2 @@ +Match User resticprofile + AllowTcpForwarding yes From d6b4a122b8cc7a04f3d360bf091f821c352ca19b Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 18:23:52 +0100 Subject: [PATCH 11/24] fix: update SSH server configuration and improve known_hosts handling --- .github/workflows/build.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e8dc94d5..cc1b63d63 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,10 +77,11 @@ jobs: services: openssh-server: image: lscr.io/linuxserver/openssh-server:latest + options: --name openssh-server ports: - 2222:2222 volumes: - - ${{ github.workspace }}/ssh/tests/ssh_gateway_ports.conf:/config/sshd/sshd_config.d/ssh_gateway_ports.conf + - config:/config env: PUID: 1000 PGID: 1000 @@ -102,7 +103,15 @@ jobs: cache: true - name: Download host keys - run: ssh-keyscan -p 2222 -H localhost > ${{ github.workspace }}/ssh/tests/known_hosts + run: | + id + ssh-keyscan -p 2222 -H localhost > ${{ runner.temp }}/known_hosts + cat ${{ runner.temp }}/known_hosts + + - name: Restart to feed the volume + uses: docker://docker + with: + args: docker restart openssh-server sonarCloudTrigger: needs: build From c6d101165cf4c54208be093f44653b7959f46f3f Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 22:40:35 +0100 Subject: [PATCH 12/24] very comvoluted procrss to please both GNU Make and Github Actions --- .github/workflows/build.yml | 33 ++++++++++++++++++------- Makefile | 41 +++++++++++++++----------------- ssh/ssh_test.go | 16 ++++++------- ssh/tests/ssh_gateway_ports.conf | 2 -- 4 files changed, 51 insertions(+), 41 deletions(-) delete mode 100644 ssh/tests/ssh_gateway_ports.conf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc1b63d63..4b7b0213b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,8 +83,8 @@ jobs: volumes: - config:/config env: - PUID: 1000 - PGID: 1000 + PUID: 1001 # runner user ID + PGID: 100 # users group ID TZ: Etc/UTC SUDO_ACCESS: false PASSWORD_ACCESS: false @@ -99,20 +99,37 @@ jobs: uses: actions/setup-go@v5 with: go-version: 1.24 - check-latest: true - cache: true - - name: Download host keys + - name: Setup OpenSSH server + env: + SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests run: | id - ssh-keyscan -p 2222 -H localhost > ${{ runner.temp }}/known_hosts - cat ${{ runner.temp }}/known_hosts + mkdir -p ${SSH_TESTS_TMPDIR} + ssh-keygen -t rsa -b 2048 -f ${SSH_TESTS_TMPDIR}/id_rsa -N "" -C "resticprofile@localhost" + ssh-keyscan -p 2222 -H localhost > ${SSH_TESTS_TMPDIR}/known_hosts + cat ${SSH_TESTS_TMPDIR}/known_hosts + + - name: Tweak OpenSSH server configuration + uses: docker://docker + with: + args: echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" - - name: Restart to feed the volume + - name: Insert public key into authorized_keys + uses: docker://docker + with: + args: cat ${SSH_TESTS_TMPDIR}/id_rsa.pub | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" + + - name: Restart with the new configuration uses: docker://docker with: args: docker restart openssh-server + - name: Test + env: + SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests + run: go test -v -tags ssh ./ssh + sonarCloudTrigger: needs: build name: SonarCloud Trigger diff --git a/Makefile b/Makefile index 919d84e4f..e4b69340a 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ ifeq ($(UNAME),Darwin) TMP_MOUNT=${TMP_MOUNT_DARWIN} endif +TMPDIR ?= /tmp +SSH_TESTS_TMPDIR=$(shell echo "$(TMPDIR)/resticprofile-ssh-tests" | tr -s /) + TOC_START=<\!--ts--> TOC_END=<\!--te--> TOC_PATH=toc.md @@ -337,14 +340,12 @@ deploy-current: build-linux build-pi ssh $$server "sudo -S install $(BINARY_PI) /usr/local/bin/resticprofile" ; \ done +# this is a convoluted step to mimic the GitHub Actions environment .PHONY: start-ssh-server -start-ssh-server: +start-ssh-server: stop-ssh-server @echo "[*] $@" - $(eval KEYS_TMP_DIR := $(shell mktemp --directory --tmpdir resticprofile.XXXXXXXXXX)) - @echo "Using temporary directory for SSH keys: $(KEYS_TMP_DIR)" - @echo $(KEYS_TMP_DIR) > ./ssh/tests/running.tmpdir - @ssh-keygen -t rsa -b 2048 -f $(KEYS_TMP_DIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)" - @cp $(KEYS_TMP_DIR)/id_rsa.pub ./ssh/tests/authorized_keys + @mkdir -p $(SSH_TESTS_TMPDIR) || echo "Failed to create temporary directory" + @ssh-keygen -t rsa -b 2048 -f $(SSH_TESTS_TMPDIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)" @docker run -d \ --name=openssh-server \ --hostname=openssh-server \ @@ -356,25 +357,21 @@ start-ssh-server: -e USER_NAME=resticprofile \ -e LOG_STDOUT=false \ -p 2222:2222 \ - -v ./ssh/tests/authorized_keys:/config/.ssh/authorized_keys \ - -v ./ssh/tests/ssh_gateway_ports.conf:/config/sshd/sshd_config.d/ssh_gateway_ports.conf \ lscr.io/linuxserver/openssh-server:latest @sleep 1 - @ssh-keyscan -p 2222 -H localhost > ./ssh/tests/known_hosts + @ssh-keyscan -p 2222 -H localhost > $(SSH_TESTS_TMPDIR)/known_hosts + @echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" + @cat $(SSH_TESTS_TMPDIR)/id_rsa.pub | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" + @docker restart openssh-server .PHONY: stop-ssh-server stop-ssh-server: @echo "[*] $@" - @docker stop openssh-server || echo "No running SSH server to stop" - @docker rm openssh-server || echo "No container to remove" - $(eval KEYS_TMP_DIR := $(shell cat ./ssh/tests/running.tmpdir)) - @test -d "$(KEYS_TMP_DIR)" && rm -rf "$(KEYS_TMP_DIR)" || echo "Failed to remove temporary SSH keys directory" - @test -f ./ssh/tests/running.tmpdir && rm ./ssh/tests/running.tmpdir || echo "Failed to remove temporary file" - @test -f ./ssh/tests/authorized_keys && rm ./ssh/tests/authorized_keys || echo "Failed to remove authorized_keys file" - @test -f ./ssh/tests/known_hosts && rm ./ssh/tests/known_hosts || echo "Failed to remove known_hosts file" - -known_hosts: - @echo "[*] $@" - $(eval KEYS_TMP_DIR := $(shell cat ./ssh/tests/running.tmpdir)) - @ssh-keyscan -p 2222 -H localhost > $(KEYS_TMP_DIR)/known_hosts - @echo "Known hosts file created at $(KEYS_TMP_DIR)/known_hosts" \ No newline at end of file + @echo "stopping and removing openssh-server container..." + @docker ps --filter "name=openssh-server" --format "{{.State}}" | grep -q "running" && \ + docker stop openssh-server || \ + echo "container is not running, nothing to stop" + @docker ps --all --filter "name=openssh-server" --format "{{.Names}}" | grep -q "openssh-server" && \ + docker rm openssh-server || \ + echo "container is absent, nothing to remove" + @test -d "$(SSH_TESTS_TMPDIR)" && rm -rf "$(SSH_TESTS_TMPDIR)" || echo "temporary directory not found, nothing to remove" diff --git a/ssh/ssh_test.go b/ssh/ssh_test.go index f7349cd69..0dc927ecf 100644 --- a/ssh/ssh_test.go +++ b/ssh/ssh_test.go @@ -11,12 +11,10 @@ import ( ) func TestSSHClient(t *testing.T) { - wd, err := os.Getwd() - require.NoError(t, err) - - t.Logf("Current working directory: %s", wd) - - tmpDir := os.Getenv("KEYS_TMP_DIR") + tmpDir := os.Getenv("SSH_TESTS_TMPDIR") + if tmpDir == "" { + tmpDir = filepath.Join(os.TempDir(), "resticprofile-ssh-tests") + } fixtures := []struct { name string @@ -28,7 +26,7 @@ func TestSSHClient(t *testing.T) { config: Config{ Host: "localhost:2222", Username: "resticprofile", - KnownHostsPath: filepath.Join(wd, "./tests/known_hosts"), + KnownHostsPath: filepath.Join(tmpDir, "known_hosts"), }, connectErr: true, }, @@ -37,7 +35,7 @@ func TestSSHClient(t *testing.T) { config: Config{ Host: "localhost:2222", Username: "otheruser", - KnownHostsPath: filepath.Join(wd, "tests/known_hosts"), + KnownHostsPath: filepath.Join(tmpDir, "known_hosts"), PrivateKeyPath: filepath.Join(tmpDir, "id_rsa"), }, connectErr: true, @@ -47,7 +45,7 @@ func TestSSHClient(t *testing.T) { config: Config{ Host: "localhost:2222", Username: "resticprofile", - KnownHostsPath: filepath.Join(wd, "tests/known_hosts"), + KnownHostsPath: filepath.Join(tmpDir, "known_hosts"), PrivateKeyPath: filepath.Join(tmpDir, "id_rsa"), }, connectErr: false, diff --git a/ssh/tests/ssh_gateway_ports.conf b/ssh/tests/ssh_gateway_ports.conf deleted file mode 100644 index 79af9a816..000000000 --- a/ssh/tests/ssh_gateway_ports.conf +++ /dev/null @@ -1,2 +0,0 @@ -Match User resticprofile - AllowTcpForwarding yes From 0f6f4f39625171ab84868f7d8bd3319058ac37b6 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 22:46:43 +0100 Subject: [PATCH 13/24] really stupid scope for runner --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b7b0213b..5f5713f24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,11 +114,15 @@ jobs: uses: docker://docker with: args: echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" + env: + SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - name: Insert public key into authorized_keys uses: docker://docker with: args: cat ${SSH_TESTS_TMPDIR}/id_rsa.pub | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" + env: + SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - name: Restart with the new configuration uses: docker://docker @@ -126,9 +130,9 @@ jobs: args: docker restart openssh-server - name: Test + run: go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - run: go test -v -tags ssh ./ssh sonarCloudTrigger: needs: build From f1bf043cb84b3114928449c92e94588042ddb685 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 22:51:09 +0100 Subject: [PATCH 14/24] fucking environment variable --- .github/workflows/build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f5713f24..ed43c6325 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,15 +114,11 @@ jobs: uses: docker://docker with: args: echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" - env: - SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - name: Insert public key into authorized_keys uses: docker://docker with: - args: cat ${SSH_TESTS_TMPDIR}/id_rsa.pub | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" - env: - SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests + args: cat "${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub" | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" - name: Restart with the new configuration uses: docker://docker From f93c855c2883534d68757e49745485db9c11ec96 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:00:32 +0100 Subject: [PATCH 15/24] try with reading the public key into a variable --- .github/workflows/build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed43c6325..52e5533fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -115,10 +115,16 @@ jobs: with: args: echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" + - name: Read public key + id: publickey + uses: jaywcjlove/github-action-read-file@main + with: + path: ${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub + - name: Insert public key into authorized_keys uses: docker://docker with: - args: cat "${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub" | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" + args: echo "${{ steps.publickey.outputs.content }}" | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" - name: Restart with the new configuration uses: docker://docker From e476f809ad3acb7dacfdecc09aee1d7c361ac2a2 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:03:54 +0100 Subject: [PATCH 16/24] oops. need to use localfile --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52e5533fe..f30b09ced 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,7 +119,7 @@ jobs: id: publickey uses: jaywcjlove/github-action-read-file@main with: - path: ${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub + localfile: ${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub - name: Insert public key into authorized_keys uses: docker://docker From b19a8aad9e66c0231979e27958b8c4c87f5c9af1 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:13:33 +0100 Subject: [PATCH 17/24] try to connect manually first --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f30b09ced..c346e81e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,7 +132,9 @@ jobs: args: docker restart openssh-server - name: Test - run: go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh + run: | + ssh -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@openssh-server env + go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests From 5fdc7f448e9f32652208bff835cc0e8e35917194 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:15:30 +0100 Subject: [PATCH 18/24] try localhost instead --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c346e81e7..a4202074d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,7 +133,7 @@ jobs: - name: Test run: | - ssh -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@openssh-server env + ssh -v -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@localhost env go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests From 054dd845aa1a944beaae1788e887cab8c8ae4c8d Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:22:00 +0100 Subject: [PATCH 19/24] check ssh client config, just in case --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4202074d..ee1e13a7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,6 +133,7 @@ jobs: - name: Test run: | + cat /etc/ssh/ssh_config ssh -v -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@localhost env go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: From d579f846fb1f18f5f7d8f00daa3c5b360813c2fb Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:39:24 +0100 Subject: [PATCH 20/24] check config files --- .github/workflows/build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee1e13a7d..a52d4be16 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,8 +80,6 @@ jobs: options: --name openssh-server ports: - 2222:2222 - volumes: - - config:/config env: PUID: 1001 # runner user ID PGID: 100 # users group ID @@ -130,10 +128,15 @@ jobs: uses: docker://docker with: args: docker restart openssh-server + + - name: Copy configuration files + uses: docker://docker + with: + args: docker cp openssh-server:/config config - name: Test run: | - cat /etc/ssh/ssh_config + cat config/**/* ssh -v -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@localhost env go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: From 572aa8ed58a5b57f1ee0dbfafa9e0e4fb29af754 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:41:44 +0100 Subject: [PATCH 21/24] copy to temp --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a52d4be16..47350c92a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,11 +132,11 @@ jobs: - name: Copy configuration files uses: docker://docker with: - args: docker cp openssh-server:/config config + args: docker cp openssh-server:/config ${{ runner.temp }}/config - name: Test run: | - cat config/**/* + cat ${{ runner.temp }}/config/**/* ssh -v -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@localhost env go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh env: From d374a0077adbce3842b8c2486c094a37f4439fce Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 8 Aug 2025 23:55:31 +0100 Subject: [PATCH 22/24] this was a fucking waste of time --- .github/workflows/build.yml | 72 ------------------------------------- 1 file changed, 72 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47350c92a..9dac7817a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,78 +70,6 @@ jobs: name: code-coverage-report-${{ matrix.os }} path: coverage.out - test-ssh: - name: Test SSH connections - runs-on: ubuntu-latest - - services: - openssh-server: - image: lscr.io/linuxserver/openssh-server:latest - options: --name openssh-server - ports: - - 2222:2222 - env: - PUID: 1001 # runner user ID - PGID: 100 # users group ID - TZ: Etc/UTC - SUDO_ACCESS: false - PASSWORD_ACCESS: false - USER_NAME: resticprofile - LOG_STDOUT: true - - steps: - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.24 - - - name: Setup OpenSSH server - env: - SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - run: | - id - mkdir -p ${SSH_TESTS_TMPDIR} - ssh-keygen -t rsa -b 2048 -f ${SSH_TESTS_TMPDIR}/id_rsa -N "" -C "resticprofile@localhost" - ssh-keyscan -p 2222 -H localhost > ${SSH_TESTS_TMPDIR}/known_hosts - cat ${SSH_TESTS_TMPDIR}/known_hosts - - - name: Tweak OpenSSH server configuration - uses: docker://docker - with: - args: echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" - - - name: Read public key - id: publickey - uses: jaywcjlove/github-action-read-file@main - with: - localfile: ${{ runner.temp }}/resticprofile-ssh-tests/id_rsa.pub - - - name: Insert public key into authorized_keys - uses: docker://docker - with: - args: echo "${{ steps.publickey.outputs.content }}" | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" - - - name: Restart with the new configuration - uses: docker://docker - with: - args: docker restart openssh-server - - - name: Copy configuration files - uses: docker://docker - with: - args: docker cp openssh-server:/config ${{ runner.temp }}/config - - - name: Test - run: | - cat ${{ runner.temp }}/config/**/* - ssh -v -i ${SSH_TESTS_TMPDIR}/id_rsa -p 2222 -o UserKnownHostsFile=${SSH_TESTS_TMPDIR}/known_hosts resticprofile@localhost env - go test -v -tags ssh -coverprofile='coverage-ssh.out' ./ssh - env: - SSH_TESTS_TMPDIR: ${{ runner.temp }}/resticprofile-ssh-tests - sonarCloudTrigger: needs: build name: SonarCloud Trigger From d6d752a9f1719dd6512d30bba463fb6b560b3786 Mon Sep 17 00:00:00 2001 From: Fred Date: Sun, 10 Aug 2025 22:00:38 +0100 Subject: [PATCH 23/24] use docker compose to run ssh tests --- .github/workflows/build.yml | 20 ++++++++++++++++- Makefile | 36 +++++++++--------------------- ssh/test/allow_tcp_forwarding.conf | 2 ++ ssh/test/docker-compose.yml | 20 +++++++++++++++++ 4 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 ssh/test/allow_tcp_forwarding.conf create mode 100644 ssh/test/docker-compose.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9dac7817a..13cf97e34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,6 @@ on: - 'docs/**' jobs: - build: name: Build and test runs-on: ${{ matrix.os }} @@ -70,6 +69,25 @@ jobs: name: code-coverage-report-${{ matrix.os }} path: coverage.out + test-ssh: + name: Test SSH client + runs-on: ubuntu-latest + + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24 + + - name: Run tests + run: | + make start-ssh-server + make ssh-test + make stop-ssh-server + sonarCloudTrigger: needs: build name: SonarCloud Trigger diff --git a/Makefile b/Makefile index e4b69340a..17368908d 100644 --- a/Makefile +++ b/Makefile @@ -340,38 +340,24 @@ deploy-current: build-linux build-pi ssh $$server "sudo -S install $(BINARY_PI) /usr/local/bin/resticprofile" ; \ done -# this is a convoluted step to mimic the GitHub Actions environment .PHONY: start-ssh-server -start-ssh-server: stop-ssh-server +start-ssh-server: @echo "[*] $@" - @mkdir -p $(SSH_TESTS_TMPDIR) || echo "Failed to create temporary directory" + @mkdir -p $(SSH_TESTS_TMPDIR) && rm -f $(SSH_TESTS_TMPDIR)/id_rsa* || echo "Failed to create temporary directory" @ssh-keygen -t rsa -b 2048 -f $(SSH_TESTS_TMPDIR)/id_rsa -N "" -C "resticprofile@$(shell hostname)" - @docker run -d \ - --name=openssh-server \ - --hostname=openssh-server \ - -e PUID=1000 \ - -e PGID=1000 \ - -e TZ=Etc/UTC \ - -e SUDO_ACCESS=false \ - -e PASSWORD_ACCESS=false \ - -e USER_NAME=resticprofile \ - -e LOG_STDOUT=false \ - -p 2222:2222 \ - lscr.io/linuxserver/openssh-server:latest + @cd ./ssh/test && \ + USER_ID=$(shell id -u) GROUP_ID=$(shell id -g) SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) \ + docker compose up -d --force-recreate @sleep 1 @ssh-keyscan -p 2222 -H localhost > $(SSH_TESTS_TMPDIR)/known_hosts - @echo "Match User resticprofile\n AllowTcpForwarding yes\n" | docker exec -i openssh-server sh -c "tee -a /config/sshd/sshd_config" - @cat $(SSH_TESTS_TMPDIR)/id_rsa.pub | docker exec -i openssh-server sh -c "tee -a /config/.ssh/authorized_keys" - @docker restart openssh-server .PHONY: stop-ssh-server stop-ssh-server: @echo "[*] $@" - @echo "stopping and removing openssh-server container..." - @docker ps --filter "name=openssh-server" --format "{{.State}}" | grep -q "running" && \ - docker stop openssh-server || \ - echo "container is not running, nothing to stop" - @docker ps --all --filter "name=openssh-server" --format "{{.Names}}" | grep -q "openssh-server" && \ - docker rm openssh-server || \ - echo "container is absent, nothing to remove" + cd ./ssh/test && SSH_TESTS_TMPDIR=$(SSH_TESTS_TMPDIR) docker compose down --remove-orphans @test -d "$(SSH_TESTS_TMPDIR)" && rm -rf "$(SSH_TESTS_TMPDIR)" || echo "temporary directory not found, nothing to remove" + +.PHONY: ssh-test +ssh-test: + @echo "[*] $@" + @go test -run TestSSHClient -v -tags ssh ./ssh diff --git a/ssh/test/allow_tcp_forwarding.conf b/ssh/test/allow_tcp_forwarding.conf new file mode 100644 index 000000000..79af9a816 --- /dev/null +++ b/ssh/test/allow_tcp_forwarding.conf @@ -0,0 +1,2 @@ +Match User resticprofile + AllowTcpForwarding yes diff --git a/ssh/test/docker-compose.yml b/ssh/test/docker-compose.yml new file mode 100644 index 000000000..029412e90 --- /dev/null +++ b/ssh/test/docker-compose.yml @@ -0,0 +1,20 @@ +--- +services: + openssh-server: + image: lscr.io/linuxserver/openssh-server:latest + container_name: openssh-server + hostname: openssh-server + environment: + - PUID=${USER_ID:-1000} + - PGID=${GROUP_ID:-1000} + - TZ=${TZ:-Etc/UTC} + - PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE} + - SUDO_ACCESS=false + - PASSWORD_ACCESS=false + - USER_NAME=${SSH_USER:-resticprofile} + - LOG_STDOUT=true + volumes: + - ${PWD}/allow_tcp_forwarding.conf:/config/sshd/sshd_config.d/allow_tcp_forwarding.conf + - ${SSH_TESTS_TMPDIR}/id_rsa.pub:${PUBLIC_KEY_FILE} + ports: + - ${SSH_PORT:-2222}:2222 From e1fdff11f7bfe79aea3853ed5b44fab312d1687a Mon Sep 17 00:00:00 2001 From: Fred Date: Sun, 10 Aug 2025 22:06:33 +0100 Subject: [PATCH 24/24] don't use env file --- ssh/test/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ssh/test/docker-compose.yml b/ssh/test/docker-compose.yml index 029412e90..ca650602d 100644 --- a/ssh/test/docker-compose.yml +++ b/ssh/test/docker-compose.yml @@ -7,14 +7,14 @@ services: environment: - PUID=${USER_ID:-1000} - PGID=${GROUP_ID:-1000} - - TZ=${TZ:-Etc/UTC} - - PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE} + - TZ=Europe/London + - PUBLIC_KEY_FILE=/id_rsa.pub - SUDO_ACCESS=false - PASSWORD_ACCESS=false - - USER_NAME=${SSH_USER:-resticprofile} + - USER_NAME=resticprofile - LOG_STDOUT=true volumes: - ${PWD}/allow_tcp_forwarding.conf:/config/sshd/sshd_config.d/allow_tcp_forwarding.conf - - ${SSH_TESTS_TMPDIR}/id_rsa.pub:${PUBLIC_KEY_FILE} + - ${SSH_TESTS_TMPDIR}/id_rsa.pub:/id_rsa.pub ports: - - ${SSH_PORT:-2222}:2222 + - 2222:2222