diff --git a/README.md b/README.md index 2540732..5875b85 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ sftp-sync --dest=ftp://username:password@hostname:port/path/to/remote/folder \ ### Sync Command Options - `--dest`: The destination FTP server URL. It should follow the format `ftp://username:password@hostname:port/path/to/remote/folder`. -- `--exclude`: (Optional) Specifies paths or patterns to exclude from the synchronization process. You can specify multiple `--exclude` options to exclude multiple paths or patterns. +- `--exclude`: (Optional) Specifies paths or glob patterns to exclude from synchronization. Supports `*`, `**`, and `?`. You can specify multiple `--exclude` options. ### Sync Command Arguments @@ -195,7 +195,7 @@ The application uses structured error handling with specific exit codes: ## Roadmap -- [ ] Support for patterns in the `--exclude` option. +- [x] Support for patterns in the `--exclude` option. - [ ] Support of Secure FTP (SFTP) protocol. - [ ] Improved error handling and error messages. - [ ] Integration with Git for automatic syncing on commit or branch changes. diff --git a/go.mod b/go.mod index 1ae086f..a6b3aa9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/capcom6/sftp-sync go 1.24.3 require ( + github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/fsnotify/fsnotify v1.6.0 github.com/go-core-fx/cli-logger v0.0.0-20260319073231-90ee4649c242 github.com/jlaffaye/ftp v0.2.0 diff --git a/go.sum b/go.sum index 756b090..66f5b2e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= diff --git a/internal/cli/commands/sync/sync.go b/internal/cli/commands/sync/sync.go index d47c243..3c8dd45 100644 --- a/internal/cli/commands/sync/sync.go +++ b/internal/cli/commands/sync/sync.go @@ -6,6 +6,7 @@ import ( "github.com/capcom6/sftp-sync/internal/cli/codes" "github.com/capcom6/sftp-sync/internal/client" + "github.com/capcom6/sftp-sync/internal/exclude" "github.com/capcom6/sftp-sync/internal/syncer" "github.com/capcom6/sftp-sync/internal/watcher" logger "github.com/go-core-fx/cli-logger" @@ -33,7 +34,7 @@ func Command() *cli.Command { }, &cli.StringSliceFlag{ Name: "exclude", - Usage: "paths or patterns to exclude from the synchronization process", + Usage: "paths or glob patterns to exclude (supports *, **, ?)", }, }, ArgsUsage: "[source]", @@ -73,8 +74,14 @@ func Action(ctx context.Context, cmd *cli.Command) error { return cli.Exit(err.Error(), codes.ClientError) } - watcher := watcher.New(cfg.Source, cfg.Excludes, log) - syncer := syncer.New(cfg.Source, remote, log) + excludeMatcher, err := exclude.New(cfg.Excludes, cfg.Source) + if err != nil { + log.Error(ctx, "Failed to build exclude matcher", err) + return cli.Exit(err.Error(), codes.ParamsError) + } + + watcher := watcher.New(cfg.Source, excludeMatcher, log) + syncer := syncer.New(cfg.Source, remote, excludeMatcher, log) var wg sync.WaitGroup diff --git a/internal/exclude/errors.go b/internal/exclude/errors.go new file mode 100644 index 0000000..5247e82 --- /dev/null +++ b/internal/exclude/errors.go @@ -0,0 +1,7 @@ +package exclude + +import "errors" + +var ( + ErrInvalidPattern = errors.New("invalid exclude pattern") +) diff --git a/internal/exclude/matcher.go b/internal/exclude/matcher.go new file mode 100644 index 0000000..1c8617a --- /dev/null +++ b/internal/exclude/matcher.go @@ -0,0 +1,95 @@ +package exclude + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +type rule struct { + value string + isPattern bool +} + +type Matcher struct { + sourceRoot string + rules []rule +} + +func New(rules []string, sourceRoot string) (*Matcher, error) { + absSourceRoot, err := filepath.Abs(sourceRoot) + if err != nil { + return nil, fmt.Errorf("failed to get absolute path: %w", err) + } + + compiled := make([]rule, 0, len(rules)) + for _, raw := range rules { + normalized := filepath.ToSlash(raw) + if !doublestar.ValidatePattern(normalized) { + return nil, fmt.Errorf("%w: exclude rule %q is invalid", ErrInvalidPattern, raw) + } + + compiled = append(compiled, rule{ + value: normalized, + isPattern: hasMeta(normalized), + }) + } + + return &Matcher{ + sourceRoot: absSourceRoot, + rules: compiled, + }, nil +} + +func (m *Matcher) Match(filePath string) bool { + matched, _ := m.MatchRule(filePath) + return matched +} + +func (m *Matcher) MatchRule(filePath string) (bool, string) { + candidate := filePath + if filepath.IsAbs(candidate) { + if rel, err := filepath.Rel(m.sourceRoot, candidate); err == nil { + candidate = rel + } + } + normalized := path.Clean(filepath.ToSlash(candidate)) + + for _, r := range m.rules { + if !r.isPattern { + if normalized == r.value || strings.HasPrefix(normalized, r.value+"/") { + return true, r.value + } + + continue + } + + // Try direct match first + matched, matchErr := doublestar.Match(r.value, normalized) + if matchErr == nil && matched { + return true, r.value + } + + // If direct match fails, try matching against path prefixes + // This handles cases like: + // - pattern "build/*" matching "build/out/main.bin" + // - pattern "**/node_modules" matching "web/node_modules/react/index.js" + parts := strings.Split(normalized, "/") + for i := 1; i <= len(parts); i++ { + prefix := path.Join(parts[:i]...) + matched, matchErr = doublestar.Match(r.value, prefix) + if matchErr == nil && matched { + return true, r.value + } + } + } + + return false, "" +} + +func hasMeta(pattern string) bool { + return strings.ContainsAny(pattern, "*?{") +} diff --git a/internal/exclude/matcher_test.go b/internal/exclude/matcher_test.go new file mode 100644 index 0000000..4a6d385 --- /dev/null +++ b/internal/exclude/matcher_test.go @@ -0,0 +1,115 @@ +package exclude_test + +import ( + "path/filepath" + "testing" + + "github.com/capcom6/sftp-sync/internal/exclude" +) + +func TestNewRejectsInvalidPattern(t *testing.T) { + t.Parallel() + + _, err := exclude.New([]string{"**/["}, ".") + if err == nil { + t.Fatal("expected invalid pattern error") + } +} + +func TestMatcherLiteralPathMatching(t *testing.T) { + t.Parallel() + + matcher, err := exclude.New([]string{".git", "vendor/cache"}, "/repo") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + tests := []struct { + name string + path string + want bool + }{ + {name: "exact dir", path: "./.git", want: true}, + {name: "descendant", path: "./.git/config", want: true}, + {name: "other path", path: "./pkg/main.go", want: false}, + {name: "nested descendant", path: "vendor/cache/tmp.txt", want: true}, + {name: "meta-like characters stay literal", path: "vendor/cache[file].txt", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := matcher.Match(tt.path); got != tt.want { + t.Fatalf("Match(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} + +func TestMatcherLiteralBackwardCompatibilityWithMetaCharacters(t *testing.T) { + t.Parallel() + + matcher, err := exclude.New([]string{"cache[file].txt"}, "/repo") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + if got := matcher.Match("cache[file].txt"); !got { + t.Fatalf("literal path containing [] should still match exactly") + } + + if got := matcher.Match("cachef.txt"); got { + t.Fatalf("literal-first behavior expected; glob expansion should not alter matching") + } +} + +func TestMatcherPatternMatching(t *testing.T) { + t.Parallel() + + matcher, err := exclude.New([]string{"**/*.tmp", "build/*", "**/node_modules"}, "/repo") + if err != nil { + t.Fatalf("New() error = %v", err) + } + + tests := []struct { + name string + path string + want bool + }{ + {name: "recursive extension", path: "a/b/c.tmp", want: true}, + {name: "single segment", path: "build/main.bin", want: true}, + {name: "single segment nested via matched ancestor", path: "build/out/main.bin", want: true}, + {name: "pattern matches ancestor directory", path: "web/node_modules/react/index.js", want: true}, + {name: "non match", path: "src/main.go", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := matcher.Match(tt.path); got != tt.want { + t.Fatalf("Match(%q) = %v, want %v", tt.path, got, tt.want) + } + }) + } +} + +func TestMatcherAbsolutePathAndRootBoundary(t *testing.T) { + t.Parallel() + + root := filepath.FromSlash("/repo") + inside := filepath.Join(root, "dist", "bundle.js") + outside := filepath.FromSlash("/other/dist/bundle.js") + + matcher, err := exclude.New([]string{"dist/**"}, root) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + if got := matcher.Match(inside); !got { + t.Fatalf("Match(%q) = %v, want true", inside, got) + } + + if got := matcher.Match(outside); got { + t.Fatalf("Match(%q) = %v, want false", outside, got) + } +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 0a91eae..2a49d7e 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -5,35 +5,32 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "github.com/capcom6/sftp-sync/internal/client" + "github.com/capcom6/sftp-sync/internal/exclude" logger "github.com/go-core-fx/cli-logger" ) type Syncer struct { rootPath string client client.Client + matcher *exclude.Matcher logger logger.Logger } -func New(rootPath string, client client.Client, logger logger.Logger) *Syncer { +func New(rootPath string, client client.Client, matcher *exclude.Matcher, logger logger.Logger) *Syncer { return &Syncer{ rootPath: rootPath, client: client, + matcher: matcher, logger: logger.WithContext("syncer", ""), } } func (s *Syncer) Sync(ctx context.Context, absPath string) error { - exists, isDir, err := fsInfo(absPath) - if err != nil { - return fmt.Errorf("fsInfo: %w", err) - } - absRoot, err := filepath.Abs(s.rootPath) if err != nil { return fmt.Errorf("filepath.Abs: %w", err) @@ -44,6 +41,19 @@ func (s *Syncer) Sync(ctx context.Context, absPath string) error { return fmt.Errorf("filepath.Rel: %w", err) } + if matched, rule := s.isExcluded(absPath); matched { + s.logger.Debug(ctx, "Excluded path skipped", logger.Fields{ + "path": relPath, + "rule": rule, + }) + return nil + } + + exists, isDir, err := fsInfo(absPath) + if err != nil { + return fmt.Errorf("fsInfo: %w", err) + } + if !exists { if rmErr := s.client.Remove(ctx, pathNormalize(relPath)); rmErr != nil { return fmt.Errorf("c.Remove: %w", rmErr) @@ -95,14 +105,25 @@ func (s *Syncer) syncDir(ctx context.Context, absPath, relPath string) error { default: } + childAbsPath := filepath.Join(absPath, file.Name()) + childRelPath := filepath.Join(relPath, file.Name()) + if matched, rule := s.isExcluded(childAbsPath); matched { + s.logger.Debug(ctx, "Excluded path skipped", logger.Fields{ + "path": childRelPath, + "rule": rule, + }) + continue + } + if file.IsDir() { - if dErr := s.syncDir(ctx, path.Join(absPath, file.Name()), path.Join(relPath, file.Name())); dErr != nil { + if dErr := s.syncDir(ctx, childAbsPath, childRelPath); dErr != nil { return dErr } - } else { - if fErr := s.syncFile(ctx, path.Join(absPath, file.Name()), path.Join(relPath, file.Name())); fErr != nil { - return fErr - } + continue + } + + if fErr := s.syncFile(ctx, childAbsPath, childRelPath); fErr != nil { + return fErr } } @@ -124,3 +145,11 @@ func fsInfo(path string) (bool, bool, error) { func pathNormalize(path string) string { return filepath.ToSlash(path) } + +func (s *Syncer) isExcluded(absPath string) (bool, string) { + if s.matcher == nil { + return false, "" + } + + return s.matcher.MatchRule(absPath) +} diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 6d0927a..c3feacb 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -8,31 +8,30 @@ import ( "strings" "sync" + "github.com/capcom6/sftp-sync/internal/exclude" "github.com/fsnotify/fsnotify" logger "github.com/go-core-fx/cli-logger" ) type Watcher struct { rootPath string - excludes []string + matcher *exclude.Matcher logger logger.Logger absRootPath string - absExcludes []string fswatcher *fsnotify.Watcher events chan Event } -func New(rootPath string, excludes []string, logger logger.Logger) *Watcher { +func New(rootPath string, matcher *exclude.Matcher, logger logger.Logger) *Watcher { return &Watcher{ rootPath: rootPath, - excludes: excludes, + matcher: matcher, logger: logger.WithContext("watcher", ""), absRootPath: "", - absExcludes: nil, fswatcher: nil, events: nil, } @@ -47,18 +46,13 @@ func (w *Watcher) Watch(ctx context.Context, wg *sync.WaitGroup) (EventsChannel, return nil, fmt.Errorf("prepareRoot: %w", err) } - w.absExcludes = make([]string, 0, len(w.excludes)) - for _, exclude := range w.excludes { - w.absExcludes = append(w.absExcludes, filepath.Join(w.absRootPath, exclude)) - } - var err error w.fswatcher, err = fsnotify.NewWatcher() if err != nil { return nil, fmt.Errorf("fsnotify.NewWatcher: %w", err) } - if addErr := w.addRecursive(w.absRootPath); addErr != nil { + if addErr := w.addRecursive(ctx, w.absRootPath); addErr != nil { _ = w.fswatcher.Close() w.fswatcher = nil return nil, fmt.Errorf("addRecursive: %w", addErr) @@ -80,7 +74,6 @@ func (w *Watcher) runWatcher(ctx context.Context) { _ = w.fswatcher.Close() close(w.events) w.fswatcher = nil - w.absExcludes = nil w.events = nil }() @@ -91,7 +84,7 @@ func (w *Watcher) runWatcher(ctx context.Context) { return } - if w.isExcluded(event.Name) { + if w.isExcluded(ctx, event.Name) { continue } @@ -121,7 +114,7 @@ func (w *Watcher) processEvent(ctx context.Context, source fsnotify.Event) error return nil } - proceed, err := w.updateObservers(source) + proceed, err := w.updateObservers(ctx, source) if err != nil { return fmt.Errorf("updateObservers: %w", err) } @@ -169,7 +162,7 @@ func (w *Watcher) processEvent(ctx context.Context, source fsnotify.Event) error // When fsnotify.Write event is received, it skips recursive sync. // The function returns true if the event needs to be processed // further and false otherwise. -func (w *Watcher) updateObservers(source fsnotify.Event) (bool, error) { +func (w *Watcher) updateObservers(ctx context.Context, source fsnotify.Event) (bool, error) { if source.Has(fsnotify.Remove) || source.Has(fsnotify.Rename) { wl := w.fswatcher.WatchList() for _, entry := range wl { @@ -192,7 +185,7 @@ func (w *Watcher) updateObservers(source fsnotify.Event) (bool, error) { } if source.Op.Has(fsnotify.Create) { - if addErr := w.addRecursive(fullpath); addErr != nil { + if addErr := w.addRecursive(ctx, fullpath); addErr != nil { return false, fmt.Errorf("addRecursive: %w", addErr) } } else if source.Op.Has(fsnotify.Write) { @@ -231,8 +224,8 @@ func (w *Watcher) prepareRoot() error { return nil } -func (w *Watcher) addRecursive(path string) error { - if w.isExcluded(path) { +func (w *Watcher) addRecursive(ctx context.Context, path string) error { + if w.isExcluded(ctx, path) { return nil } @@ -247,12 +240,18 @@ func (w *Watcher) addRecursive(path string) error { } for _, entry := range entries { + select { + case <-ctx.Done(): + return nil + default: + } + if !entry.IsDir() { continue } entryPath := filepath.Join(path, entry.Name()) - if addErr := w.addRecursive(entryPath); addErr != nil { + if addErr := w.addRecursive(ctx, entryPath); addErr != nil { return addErr } } @@ -260,11 +259,18 @@ func (w *Watcher) addRecursive(path string) error { return nil } -func (w *Watcher) isExcluded(fullpath string) bool { - for _, exclude := range w.absExcludes { - if fullpath == exclude || strings.HasPrefix(fullpath, exclude+string(filepath.Separator)) { - return true - } +func (w *Watcher) isExcluded(ctx context.Context, fullpath string) bool { + if w.matcher == nil { + return false } + + if matched, rule := w.matcher.MatchRule(fullpath); matched { + w.logger.Debug(ctx, "Excluded path skipped", logger.Fields{ + "path": fullpath, + "rule": rule, + }) + return true + } + return false } diff --git a/main.go b/main.go index d311e59..f1749e2 100644 --- a/main.go +++ b/main.go @@ -80,7 +80,7 @@ func main() { }, &cli.StringSliceFlag{ Name: "exclude", - Usage: "paths or patterns to exclude from the synchronization process", + Usage: "paths or glob patterns to exclude (supports *, **, ?)", }, }, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {