From 914506475262ae946f5430b96cc57f4f401dca96 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:44:28 +0000 Subject: [PATCH 1/9] importer: remove unused log file --- rclone/importer/importer.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/rclone/importer/importer.go b/rclone/importer/importer.go index c6d53ee..6f472a7 100644 --- a/rclone/importer/importer.go +++ b/rclone/importer/importer.go @@ -53,7 +53,6 @@ type RcloneImporter struct { Typee string Base string confFile *os.File - logFile *os.File } // NewRcloneImporter creates a new RcloneImporter instance. It expects the location @@ -79,13 +78,10 @@ func NewRcloneImporter(ctx context.Context, opts *connectors.Options, providerNa librclone.Initialize() - f, _ := os.Create("/home/ptr/dev/plakar/plakar/log2.txt") - return &RcloneImporter{ Typee: typee, Base: base, confFile: file, - logFile: f, }, nil } @@ -301,8 +297,6 @@ func (p *RcloneImporter) NewReader(pathname string) (io.ReadCloser, error) { return nil, err } - fmt.Fprintf(p.logFile, "downloading and processing %s %s to %s\n", pathname, relativePath, name) - payload := map[string]string{ "srcFs": fmt.Sprintf("%s:%s", p.Typee, p.Base), "srcRemote": strings.TrimPrefix(relativePath, "/"), @@ -331,7 +325,6 @@ func (p *RcloneImporter) NewReader(pathname string) (io.ReadCloser, error) { } func (p *RcloneImporter) Close(ctx context.Context) error { - p.logFile.Close() utils.DeleteTempConf(p.confFile.Name()) librclone.Finalize() return nil From 85f851d6bdc04a11c2e98286ac4e606a6d0d3969 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:35:23 +0000 Subject: [PATCH 2/9] importer: remove unused function --- rclone/importer/importer.go | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/rclone/importer/importer.go b/rclone/importer/importer.go index 6f472a7..272d456 100644 --- a/rclone/importer/importer.go +++ b/rclone/importer/importer.go @@ -126,38 +126,6 @@ func (p *RcloneImporter) GetPathInBackup(path string) string { return stdpath.Clean(path) } -// generatePathComponents is a helper function that returns a slice of strings -// containing all the hierarchical components of an absolute path, starting -// from the full path down to the root. -// -// The path given as an argument must be an absolute clean path within the -// backup. -// -// Example: -// -// Input: "/path/to/dir" -// Output: []string{"/path/to/dir", "/path/to", "/path", "/"} -// -// Input: "/relative/path" -// Output: []string{"/relative/path", "/relative", "/"} -// -// Input: "/" -// Output: []string{"/"} -func generatePathComponents(path string) []string { - components := []string{} - tmp := path - - for { - components = append(components, tmp) - parent := stdpath.Dir(tmp) - if parent == tmp { // Reached the root - break - } - tmp = parent - } - return components -} - func (p *RcloneImporter) scanRecursive(results chan<- *connectors.Record, path string, wg *sync.WaitGroup) { response, _ := p.ListFolder(results, path) p.scanFolder(results, path, response, wg) From 1f828c2d541ff9378c4d8ad535cbd93213762ce3 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:38:41 +0000 Subject: [PATCH 3/9] importer: use map[string]string in ListFolder --- rclone/importer/importer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rclone/importer/importer.go b/rclone/importer/importer.go index 272d456..f6586d2 100644 --- a/rclone/importer/importer.go +++ b/rclone/importer/importer.go @@ -132,7 +132,7 @@ func (p *RcloneImporter) scanRecursive(results chan<- *connectors.Record, path s } func (p *RcloneImporter) ListFolder(results chan<- *connectors.Record, path string) (Response, error) { - payload := map[string]interface{}{ + payload := map[string]string{ "fs": fmt.Sprintf("%s:%s", p.Typee, p.Base), "remote": path, } From a00977ac23d25d8e8ff176244863b4ed135b6d49 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:38:09 +0000 Subject: [PATCH 4/9] importer: remove unused path argument scanFolder does not use the path. It's ListFolder that request a specific path, scanFolder only consumes the reply. --- rclone/importer/importer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rclone/importer/importer.go b/rclone/importer/importer.go index f6586d2..628bce8 100644 --- a/rclone/importer/importer.go +++ b/rclone/importer/importer.go @@ -105,7 +105,7 @@ func (p *RcloneImporter) Import(ctx context.Context, records chan<- *connectors. } wg := &sync.WaitGroup{} - p.scanFolder(records, "", response, wg) + p.scanFolder(records, response, wg) wg.Wait() return nil @@ -128,7 +128,7 @@ func (p *RcloneImporter) GetPathInBackup(path string) string { func (p *RcloneImporter) scanRecursive(results chan<- *connectors.Record, path string, wg *sync.WaitGroup) { response, _ := p.ListFolder(results, path) - p.scanFolder(results, path, response, wg) + p.scanFolder(results, response, wg) } func (p *RcloneImporter) ListFolder(results chan<- *connectors.Record, path string) (Response, error) { @@ -159,7 +159,7 @@ func (p *RcloneImporter) ListFolder(results chan<- *connectors.Record, path stri return response, nil } -func (p *RcloneImporter) scanFolder(results chan<- *connectors.Record, path string, response Response, wg *sync.WaitGroup) { +func (p *RcloneImporter) scanFolder(results chan<- *connectors.Record, response Response, wg *sync.WaitGroup) { for _, file := range response.List { wg.Add(1) go func() { From d068edd1a27b0f55bdfb00e5b558a9057eda9d22 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:51:58 +0000 Subject: [PATCH 5/9] importer: kill createTempPath with fire it's a TOCTOU not needed. we already have in the stdlib CreateTemp() which deals with concurrency. --- rclone/importer/importer.go | 39 ++++--------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/rclone/importer/importer.go b/rclone/importer/importer.go index 628bce8..ecfbad7 100644 --- a/rclone/importer/importer.go +++ b/rclone/importer/importer.go @@ -2,7 +2,6 @@ package importer import ( "context" - "crypto/rand" "encoding/json" "fmt" "io" @@ -219,33 +218,6 @@ func (p *RcloneImporter) scanFolder(results chan<- *connectors.Record, response } } -func nextRandom() string { - b := make([]byte, 8) - _, err := rand.Read(b) - if err != nil { - panic(err) - } - return fmt.Sprintf("%x", b) -} - -func createTempPath(originalPath string) (path string, err error) { - tmpPath := os.TempDir() + "/" + originalPath - prefix, suffix := "", "" - if i := strings.LastIndex(tmpPath, "*"); i >= 0 { - prefix, suffix = tmpPath[:i], tmpPath[i+1:] - } else { - prefix = tmpPath - } - - for i := 0; i < 10000; i++ { - name := prefix + nextRandom() + suffix - if _, err := os.Stat(name); os.IsNotExist(err) { - return name, nil - } - } - return "", fmt.Errorf("failed to find a folder to create the temporary file") -} - // AutoremoveTmpFile is a wrapper around an os.File that removes the file when it's closed. type AutoremoveTmpFile struct { *os.File @@ -260,10 +232,12 @@ func (p *RcloneImporter) NewReader(pathname string) (io.ReadCloser, error) { // pathname is an absolute path within the backup. Let's convert it to a // relative path to the base path. relativePath := strings.TrimPrefix(pathname, p.GetPathInBackup("")) - name, err := createTempPath("plakar_temp_*") + + fp, err := os.CreateTemp("", "plakar_rclone_tmp_*") if err != nil { return nil, err } + name := fp.Name() payload := map[string]string{ "srcFs": fmt.Sprintf("%s:%s", p.Typee, p.Base), @@ -284,12 +258,7 @@ func (p *RcloneImporter) NewReader(pathname string) (io.ReadCloser, error) { return nil, fmt.Errorf("failed to copy file: %s", body) } - tmpFile, err := os.Open(name) - if err != nil { - return nil, err - } - - return &AutoremoveTmpFile{tmpFile}, nil + return &AutoremoveTmpFile{fp}, nil } func (p *RcloneImporter) Close(ctx context.Context) error { From 4e4e04fa67347cc569f0b49a39c3ff4ef9f5812c Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 12:55:51 +0000 Subject: [PATCH 6/9] storage: use right type in listFolder for map[string]string --- rclone/storage/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rclone/storage/storage.go b/rclone/storage/storage.go index 2a2b24c..8464c0d 100644 --- a/rclone/storage/storage.go +++ b/rclone/storage/storage.go @@ -170,7 +170,7 @@ func (r *RcloneStorage) deleteFile(pathname string) error { } func (r *RcloneStorage) listFolder(pathname string) ([]string, error) { - payload := map[string]interface{}{ + payload := map[string]string{ "fs": fmt.Sprintf("%s:%s", r.Typee, r.Base), "remote": pathname, } From 80f49b5028bf298853fd11589e5d0efbe5e12796 Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 13:01:50 +0000 Subject: [PATCH 7/9] exporter: don't check cancellation Otherwise we might leave records unprocessed on the channel. It'll be the responsability of the caller to close records on cancellation. --- rclone/exporter/exporter.go | 59 +++++++++++++++---------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/rclone/exporter/exporter.go b/rclone/exporter/exporter.go index 2c47dc4..c72bd7c 100644 --- a/rclone/exporter/exporter.go +++ b/rclone/exporter/exporter.go @@ -86,49 +86,36 @@ func (p *RcloneExporter) Export(ctx context.Context, records <-chan *connectors. g, ctx := errgroup.WithContext(ctx) g.SetLimit(p.maxConcurrency) -loop: - for { - select { - case <-ctx.Done(): - ret = ctx.Err() - break loop - - case record, ok := <-records: - if !ok { - break loop - } + for record := range records { + if record.Err != nil { + results <- record.Ok() + continue + } - if record.Err != nil { - results <- record.Ok() - continue - } + if record.IsXattr || record.FileInfo.Lmode&os.ModeSymlink != 0 { + results <- record.Ok() + continue + } - if record.IsXattr || record.FileInfo.Lmode&os.ModeSymlink != 0 { + pathname := stdpath.Join(p.Root(), record.Pathname) + if record.FileInfo.Lmode.IsDir() { + if err := p.mkdir(ctx, pathname); err != nil { + results <- record.Error(err) + } else { results <- record.Ok() - continue } - pathname := stdpath.Join(p.Root(), record.Pathname) - if record.FileInfo.Lmode.IsDir() { - if err := p.mkdir(ctx, pathname); err != nil { - results <- record.Error(err) - } else { - results <- record.Ok() - } + continue + } - continue + g.Go(func() error { + if err := p.storeFile(ctx, pathname, record); err != nil { + results <- record.Error(err) + } else { + results <- record.Ok() } - - g.Go(func() error { - if err := p.storeFile(ctx, pathname, record); err != nil { - results <- record.Error(err) - } else { - results <- record.Ok() - } - return nil - }) - - } + return nil + }) } if err := g.Wait(); err != nil && ret == nil { From 376721d298b18d71b7081e8c0ef072dca8e8dd2e Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 13:02:49 +0000 Subject: [PATCH 8/9] exporter: no need to use errgroup.WithContext() The functions run with Go() won't fail, and we don't need ctx.Done() to close on Wait(). --- rclone/exporter/exporter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rclone/exporter/exporter.go b/rclone/exporter/exporter.go index c72bd7c..d1e801e 100644 --- a/rclone/exporter/exporter.go +++ b/rclone/exporter/exporter.go @@ -83,7 +83,8 @@ func (p *RcloneExporter) Ping(ctx context.Context) error { func (p *RcloneExporter) Export(ctx context.Context, records <-chan *connectors.Record, results chan<- *connectors.Result) (ret error) { defer close(results) - g, ctx := errgroup.WithContext(ctx) + + var g errgroup.Group g.SetLimit(p.maxConcurrency) for record := range records { From 00731c6d3a774cdc496e8badf701f1fe585af3ab Mon Sep 17 00:00:00 2001 From: Omar Polo Date: Wed, 10 Jun 2026 13:04:56 +0000 Subject: [PATCH 9/9] exporter: simplify error handling --- rclone/exporter/exporter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rclone/exporter/exporter.go b/rclone/exporter/exporter.go index d1e801e..ae48454 100644 --- a/rclone/exporter/exporter.go +++ b/rclone/exporter/exporter.go @@ -81,7 +81,7 @@ func (p *RcloneExporter) Ping(ctx context.Context) error { return nil } -func (p *RcloneExporter) Export(ctx context.Context, records <-chan *connectors.Record, results chan<- *connectors.Result) (ret error) { +func (p *RcloneExporter) Export(ctx context.Context, records <-chan *connectors.Record, results chan<- *connectors.Result) error { defer close(results) var g errgroup.Group @@ -119,11 +119,11 @@ func (p *RcloneExporter) Export(ctx context.Context, records <-chan *connectors. }) } - if err := g.Wait(); err != nil && ret == nil { - ret = err + if err := g.Wait(); err != nil { + return err } - return ret + return nil } func (p *RcloneExporter) mkdir(ctx context.Context, pathname string) error {