diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 01fd4045b7f..1ba99929b05 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "path" + "path/filepath" "sort" "strconv" "strings" @@ -47,6 +48,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -70,6 +72,7 @@ type config struct { CustomMimeTypesJSON string `mapstructure:"custom_mimetypes_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` MountID string `mapstructure:"mount_id"` UploadExpiration int64 `mapstructure:"upload_expiration" docs:"0;Duration for how long uploads will be valid."` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` Events eventconfig `mapstructure:"events" docs:"0;Event stream configuration"` } @@ -81,6 +84,8 @@ type eventconfig struct { EnableTLS bool `mapstructure:"nats_enable_tls" docs:"events tls switch"` AuthUsername string `mapstructure:"nats_username" docs:"event stream username"` AuthPassword string `mapstructure:"nats_password" docs:"event stream password"` + ConsumerGroup string `mapstructure:"consumer_group" docs:"dcfs;Consumer group for the upload coordinator"` + NumConsumers int `mapstructure:"numconsumers" docs:"1;Number of concurrent postprocessing event consumers"` } func (c *config) init() { @@ -101,11 +106,20 @@ func (c *config) init() { if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} } + + if c.Events.ConsumerGroup == "" { + c.Events.ConsumerGroup = "dcfs" + } + if c.Events.NumConsumers <= 0 { + c.Events.NumConsumers = 1 + } } + type Service struct { conf *config Storage storage.FS + Coordinator pkgupload.Coordinator dataServerURL *url.URL availableXS []*provider.ResourceChecksumPriority } @@ -202,9 +216,33 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } + store := pkgupload.NewFileStoreFromConfig(c.UploadDirectory, c.Drivers[c.Driver], log) + if store == nil { + return nil, fmt.Errorf("storageprovider: cannot determine upload directory, set upload_directory in config or driver root") + } + if err := store.Setup(); err != nil { + return nil, fmt.Errorf("storageprovider: upload directory setup failed: %w", err) + } + evstream, err := estreamFromConfig(c.Events) + if err != nil { + return nil, err + } + async := evstream != nil + coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, + c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log, + filepath.Join(store.Root(), "uploads")) + if err != nil { + return nil, err + } + if async { + if err := coord.Start(evstream); err != nil { + return nil, err + } + } service := &Service{ conf: c, Storage: fs, + Coordinator: coord, dataServerURL: u, availableXS: xsTypes, } @@ -427,7 +465,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate metadata["expires"] = strconv.Itoa(int(expirationTimestamp.Seconds)) } - uploadIDs, err := s.Storage.InitiateUpload(ctx, req.Ref, uploadLength, metadata) + uploadIDs, err := s.Coordinator.InitiateUpload(ctx, req.Ref, uploadLength, metadata) if err != nil { var st *rpc.Status switch err.(type) { @@ -1299,7 +1337,15 @@ func estreamFromConfig(c eventconfig) (events.Stream, error) { return nil, nil } - return stream.NatsFromConfig("storageprovider", false, stream.NatsConfig(c)) + return stream.NatsFromConfig("storageprovider", false, stream.NatsConfig{ + Endpoint: c.Endpoint, + Cluster: c.Cluster, + TLSInsecure: c.TLSInsecure, + TLSRootCACertificate: c.TLSRootCACertificate, + EnableTLS: c.EnableTLS, + AuthUsername: c.AuthUsername, + AuthPassword: c.AuthPassword, + }) } func canLockPublicShare(ctx context.Context) bool { diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index bffe73bebe1..5a43601303b 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -21,6 +21,7 @@ package dataprovider import ( "fmt" "net/http" + "path/filepath" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" @@ -33,6 +34,7 @@ import ( "github.com/owncloud/reva/v2/pkg/rhttp/router" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/fs/registry" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" ) func init() { @@ -44,6 +46,7 @@ type config struct { Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go;The configuration for the storage driver"` DataTXs map[string]map[string]interface{} `mapstructure:"data_txs" docs:"url:pkg/rhttp/datatx/manager/simple/simple.go;The configuration for the data tx protocols"` + MountID string `mapstructure:"mount_id"` NatsAddress string `mapstructure:"nats_address"` NatsClusterID string `mapstructure:"nats_clusterID"` NatsTLSInsecure bool `mapstructure:"nats_tls_insecure"` @@ -51,6 +54,9 @@ type config struct { NatsEnableTLS bool `mapstructure:"nats_enable_tls"` NatsUsername string `mapstructure:"nats_username"` NatsPassword string `mapstructure:"nats_password"` + ConsumerGroup string `mapstructure:"consumer_group"` + NumConsumers int `mapstructure:"numconsumers"` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` } func (c *config) init() { @@ -60,6 +66,12 @@ func (c *config) init() { if c.Driver == "" { c.Driver = "localhome" } + if c.ConsumerGroup == "" { + c.ConsumerGroup = "dcfs" + } + if c.NumConsumers <= 0 { + c.NumConsumers = 1 + } } type svc struct { @@ -104,7 +116,27 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - dataTXs, err := getDataTXs(conf, fs, evstream, log) + store := pkgupload.NewFileStoreFromConfig(conf.UploadDirectory, conf.Drivers[conf.Driver], log) + if store == nil { + return nil, fmt.Errorf("dataprovider: cannot determine upload directory, set upload_directory in config or driver root") + } + if err := store.Setup(); err != nil { + return nil, fmt.Errorf("dataprovider: upload directory setup failed: %w", err) + } + async := evstream != nil + coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, + conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log, + filepath.Join(store.Root(), "uploads")) + if err != nil { + return nil, err + } + if async { + if err := coord.Start(evstream); err != nil { + return nil, err + } + } + + dataTXs, err := getDataTXs(conf, coord, fs, evstream, log) if err != nil { return nil, err } @@ -126,7 +158,7 @@ func getFS(c *config, stream events.Stream, log *zerolog.Logger) (storage.FS, er return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { +func getDataTXs(c *config, coord pkgupload.Coordinator, driver storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { if c.DataTXs == nil { c.DataTXs = make(map[string]map[string]interface{}) } @@ -146,7 +178,7 @@ func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerol for t := range c.DataTXs { if f, ok := datatxregistry.NewFuncs[t]; ok { if tx, err := f(c.DataTXs[t], publisher, log); err == nil { - if handler, err := tx.Handler(fs); err == nil { + if handler, err := tx.Handler(coord, driver); err == nil { txs[t] = handler } } diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index f7e3c0d2de2..e51f5b01d96 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -666,6 +666,13 @@ func (s *svc) prepareCopy(ctx context.Context, w http.ResponseWriter, r *http.Re return nil } + if !srcStatRes.GetInfo().GetPermissionSet().GetInitiateFileDownload() { + w.WriteHeader(http.StatusForbidden) + b, err := errors.Marshal(http.StatusForbidden, "no permission to copy from this resource", "", "") + errors.HandleWebdavError(log, w, b, err) + return nil + } + dstStatReq := &provider.StatRequest{Ref: dstRef} dstStatRes, err := client.Stat(ctx, dstStatReq) switch { diff --git a/internal/http/services/owncloud/ocdav/propfind/propfind.go b/internal/http/services/owncloud/ocdav/propfind/propfind.go index 4ec66360802..aaf1e317335 100644 --- a/internal/http/services/owncloud/ocdav/propfind/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind/propfind.go @@ -587,6 +587,10 @@ func (p *Handler) getResourceInfos(ctx context.Context, w http.ResponseWriter, r var status *rpc.Status info, status, err = p.statSpace(ctx, spaceRef, metadataKeys, fieldMaskPaths) if err != nil || status.GetCode() != rpc.Code_CODE_OK { + if status.GetCode() == rpc.Code_CODE_TOO_EARLY { + w.WriteHeader(http.StatusTooEarly) + return nil, false, false + } continue } } diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go index fe2bb04ab8e..85f8fcdd757 100644 --- a/pkg/ocm/storage/received/ocm.go +++ b/pkg/ocm/storage/received/ocm.go @@ -539,6 +539,9 @@ func (d *driver) GetLock(ctx context.Context, ref *provider.Reference) (*provide return nil, err } + if token == "" { + return nil, errtypes.NotFound("no lock found") + } return &provider.Lock{LockId: token, Type: provider.LockType_LOCK_TYPE_EXCL}, nil } diff --git a/pkg/ocm/storage/received/upload.go b/pkg/ocm/storage/received/upload.go index 1f9f6630e04..a122d77c505 100644 --- a/pkg/ocm/storage/received/upload.go +++ b/pkg/ocm/storage/received/upload.go @@ -76,8 +76,8 @@ func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, up }, nil } -func (d *driver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { - return errtypes.NotSupported("op not supported") +func (d *driver) MarkProcessing(_ context.Context, _ *provider.Reference, _ bool, _ string) error { + return nil } func (d *driver) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { diff --git a/pkg/rhttp/datatx/datatx.go b/pkg/rhttp/datatx/datatx.go index b770f73b890..80af29ba8df 100644 --- a/pkg/rhttp/datatx/datatx.go +++ b/pkg/rhttp/datatx/datatx.go @@ -28,12 +28,13 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/storage" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) // DataTX provides an abstraction around various data transfer protocols. type DataTX interface { - Handler(fs storage.FS) (http.Handler, error) + Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) } // EmitFileUploadedEvent is a helper function which publishes a FileUploaded event diff --git a/pkg/rhttp/datatx/manager/simple/simple.go b/pkg/rhttp/datatx/manager/simple/simple.go index 39e60e15662..dfad2743a21 100644 --- a/pkg/rhttp/datatx/manager/simple/simple.go +++ b/pkg/rhttp/datatx/manager/simple/simple.go @@ -40,6 +40,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/cache" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -78,7 +79,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("path", r.URL.Path).Logger() r = r.WithContext(appctx.WithLogger(r.Context(), &sublog)) @@ -92,7 +93,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, "") + download.GetOrHeadFile(w, r, driver, "") case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -114,7 +115,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { ctx = ctxpkg.ContextSetLockID(ctx, lockID) } - info, err := fs.Upload(ctx, storage.UploadRequest{ + info, err := coord.Upload(ctx, storage.UploadRequest{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/pkg/rhttp/datatx/manager/spaces/spaces.go b/pkg/rhttp/datatx/manager/spaces/spaces.go index b0b2f3b6ad8..6250f3d9a5b 100644 --- a/pkg/rhttp/datatx/manager/spaces/spaces.go +++ b/pkg/rhttp/datatx/manager/spaces/spaces.go @@ -42,6 +42,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/cache" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -80,7 +81,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var spaceID string spaceID, r.URL.Path = router.ShiftPath(r.URL.Path) @@ -97,7 +98,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, spaceID) + download.GetOrHeadFile(w, r, driver, spaceID) case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -117,7 +118,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { Path: fn, } var info *provider.ResourceInfo - info, err = fs.Upload(ctx, storage.UploadRequest{ + info, err = coord.Upload(ctx, storage.UploadRequest{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/pkg/rhttp/datatx/manager/tus/tus.go b/pkg/rhttp/datatx/manager/tus/tus.go index ac0e037846e..59778934b2e 100644 --- a/pkg/rhttp/datatx/manager/tus/tus.go +++ b/pkg/rhttp/datatx/manager/tus/tus.go @@ -33,13 +33,13 @@ import ( "github.com/owncloud/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/owncloud/reva/v2/pkg/appctx" - "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/manager/registry" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" ) func init() { @@ -87,20 +87,9 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { - composable, ok := fs.(storage.ComposableFS) - if !ok { - return nil, errtypes.NotSupported("file system does not support the tus protocol") - } - - // A storage backend for tusd may consist of multiple different parts which - // handle upload creation, locking, termination and so on. The composer is a - // place where all those separated pieces are joined together. In this example - // we only use the file store but you may plug in multiple. +func (m *manager) Handler(coord pkgupload.Coordinator, _ storage.FS) (http.Handler, error) { composer := tusd.NewStoreComposer() - - // let the composable storage tell tus which extensions it supports - composable.UseIn(composer) + coord.UseIn(composer) config := tusd.Config{ StoreComposer: composer, @@ -130,33 +119,28 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return nil, err } - if usl, ok := fs.(storage.UploadSessionLister); ok { - // We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info - go func() { - for { - ev := <-handler.CompleteUploads - // We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files - // so we create a Progress instance here that is used to read the correct properties - ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) - if err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") - } else { - if len(ups) < 1 { - appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") - continue - } - up := ups[0] - executant := up.Executant() - ref := up.Reference() - if m.publisher != nil { - if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") - } - } + go func() { + for { + ev := <-handler.CompleteUploads + ups, err := coord.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) + if err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") + continue + } + if len(ups) < 1 { + appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") + continue + } + up := ups[0] + executant := up.Executant() + ref := up.Reference() + if m.publisher != nil { + if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") } } - }() - } + } + }() h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("uploadid", r.URL.Path).Logger() @@ -174,7 +158,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, w, r) handler.PostFile(w, r) case "HEAD": handler.HeadFile(w, r) @@ -184,7 +168,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, w, r) handler.PatchFile(w, r) case "DELETE": handler.DelFile(w, r) @@ -205,15 +189,10 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return h, nil } -func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { +func setHeaders(coord pkgupload.Coordinator, w http.ResponseWriter, r *http.Request) { ctx := r.Context() id := path.Base(r.URL.Path) - datastore, ok := fs.(tusd.DataStore) - if !ok { - appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore") - return - } - upload, err := datastore.GetUpload(ctx, id) + upload, err := coord.GetUpload(ctx, id) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage") return diff --git a/pkg/storage/fs/posix/posix.go b/pkg/storage/fs/posix/posix.go index 03cf89ce7ee..1c7d776f5a9 100644 --- a/pkg/storage/fs/posix/posix.go +++ b/pkg/storage/fs/posix/posix.go @@ -180,21 +180,23 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s } // ListUploadSessions returns the upload sessions matching the given filter +// TODO(OCISDEV-901): remove once UploadSessionLister is removed from storage.FS (coordinator responsibility). func (fs *posixFS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { return fs.FS.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) } // UseIn tells the tus upload middleware which extensions it supports. +// TODO(OCISDEV-901): coordinator now handles TUS registration globally — is per-driver UseIn still needed for +// capability variation, or can this be removed along with storage.ComposableFS? func (fs *posixFS) UseIn(composer *tusd.StoreComposer) { fs.FS.(storage.ComposableFS).UseIn(composer) } -// NewUpload returns a new tus Upload instance +// TODO(OCISDEV-901): remove NewUpload and GetUpload — TUS DataStore is the coordinator's responsibility, not the driver's. func (fs *posixFS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { return fs.FS.(tusd.DataStore).NewUpload(ctx, info) } -// NewUpload returns a new tus Upload instance func (fs *posixFS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { return fs.FS.(tusd.DataStore).GetUpload(ctx, id) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 6c8dc675850..9f7c216bffc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -236,6 +236,13 @@ type ComposableFS interface { UseIn(composer *tusd.StoreComposer) } +// TODO(OCISDEV-901): ComposableFS and all driver-side UseIn implementations +// (middleware.FS, posix.FS) are dead after the coordinator decoupling. The +// upload coordinator is the universal tusd DataStore for every driver, so TUS +// capabilities are global — no per-driver UseIn dispatch needed. Delete this +// interface and its implementations once OCISDEV-901 wires storage-system and +// removes the legacy decomposedfs upload path. + // Registry is the interface that storage registries implement // for discovering storage providers type Registry interface { diff --git a/pkg/storage/uploads.go b/pkg/storage/uploads.go index 59ff5088550..dd7e89fd5d7 100644 --- a/pkg/storage/uploads.go +++ b/pkg/storage/uploads.go @@ -46,6 +46,9 @@ type UploadsManager interface { } // UploadSessionLister defines the interface for FS implementations that allow listing and purging upload sessions +// TODO(OCISDEV-901): ListUploadSessions belongs on the coordinator, not the driver. Remove this interface and all +// driver-side implementations (decomposedfs, posix, middleware, ocm). The CLI (storage-users uploads list) currently +// instantiates the driver directly to call this — it needs to be rerouted through the coordinator before removal. type UploadSessionLister interface { // ListUploadSessions returns the upload sessions matching the given filter ListUploadSessions(ctx context.Context, filter UploadSessionFilter) ([]UploadSession, error) diff --git a/pkg/storage/utils/chunking/chunking.go b/pkg/storage/utils/chunking/chunking.go index 97c21e030e6..bc3bdea3f1c 100644 --- a/pkg/storage/utils/chunking/chunking.go +++ b/pkg/storage/utils/chunking/chunking.go @@ -201,6 +201,44 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (bool, string, er return true, assembledFileName, nil } +// Assemble saves an intermediate chunk and, once all chunks have arrived, +// assembles them into a single temp file. Returns (reader, size, true, nil) +// when the transfer is complete. Returns (nil, 0, false, nil) when more +// chunks are still expected (partial transfer — caller should return PartialContent). +func (c *ChunkHandler) Assemble(chunkName string, body io.ReadCloser) (io.ReadCloser, int64, bool, error) { + origPath, assembledPath, err := c.WriteChunk(chunkName, body) + if err != nil { + return nil, 0, false, err + } + if origPath == "" { + return nil, 0, false, nil // partial + } + f, err := os.Open(assembledPath) + if err != nil { + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + return &assembledFile{File: f, path: assembledPath}, fi.Size(), true, nil +} + +// assembledFile wraps an *os.File and removes the temp file on Close. +type assembledFile struct { + *os.File + path string +} + +func (a *assembledFile) Close() error { + err := a.File.Close() + _ = os.RemoveAll(a.path) + return err +} + // WriteChunk saves an intermediate chunk temporarily and assembles all chunks // once the final one is received. func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index f1e348f8c28..eadc31c35de 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -28,9 +28,7 @@ import ( "path/filepath" "strconv" "strings" - "time" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/jellydator/ttlcache/v2" @@ -43,13 +41,10 @@ import ( "golang.org/x/sync/errgroup" "github.com/owncloud/reva/v2/pkg/appctx" - "github.com/owncloud/reva/v2/pkg/autoprop" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" - "github.com/owncloud/reva/v2/pkg/logger" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" - "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/utils/chunking" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/aspects" @@ -80,14 +75,6 @@ const ( var ( tracer trace.Tracer - - _registeredEvents = []events.Unmarshaller{ - events.PostprocessingFinished{}, - events.PostprocessingStepFinished{}, - events.RestartPostprocessing{}, - events.CleanUpload{}, - events.RevertRevision{}, - } ) func init() { @@ -103,10 +90,10 @@ func init() { type Session interface { tusd.Upload storage.UploadSession - upload.Session LockID() string } +// SessionStore is the interface for upload session persistence. type SessionStore interface { New(ctx context.Context) *upload.OcisSession List(ctx context.Context) ([]*upload.OcisSession, error) @@ -254,359 +241,14 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor return nil, err } - if o.AsyncFileUploads { - if fs.stream == nil { - log.Error().Msg("need event stream for async file processing") - return nil, errors.New("need nats for async file processing") - } - - ch, err := events.Consume(fs.stream, o.Events.ConsumerGroup, _registeredEvents...) - if err != nil { - return nil, err - } - - if o.Events.NumConsumers <= 0 { - o.Events.NumConsumers = 1 - } - - for i := 0; i < o.Events.NumConsumers; i++ { - go fs.Postprocessing(ch) - } + if o.AsyncFileUploads && fs.stream == nil { + log.Error().Msg("need event stream for async file processing") + return nil, errors.New("need nats for async file processing") } return fs, nil } -// Postprocessing starts the postprocessing result collector -func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { - log := logger.New() - for event := range ch { - evCtx := context.Background() - fs.processEvent(evCtx, event, log) - } -} - -func (fs *Decomposedfs) processEvent(evCtx context.Context, event events.Event, log *zerolog.Logger) { - ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) - ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) - defer span.End() - - switch ev := event.Event.(type) { - case events.PostprocessingFinished: - sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - - ctx = session.Context(ctx) - - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - if !n.Exists { - sublog.Debug().Msg("node no longer exists") - session.Cleanup(false, true, true, false) - return - } - - var ( - failed bool - revertNodeMetadata bool - keepUpload bool - ) - unmarkPostprocessing := true - - switch ev.Outcome { - default: - sublog.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - revertNodeMetadata = true - keepUpload = true - metrics.UploadSessionsAborted.Inc() - case events.PPOutcomeContinue: - if err := session.Finalize(ctx); err != nil { - sublog.Error().Err(err).Msg("could not finalize upload") - failed = true - revertNodeMetadata = false - keepUpload = true - // keep postprocessing status so the upload is not deleted during housekeeping - unmarkPostprocessing = false - } else { - metrics.UploadSessionsFinalized.Inc() - } - case events.PPOutcomeDelete: - failed = true - revertNodeMetadata = true - metrics.UploadSessionsDeleted.Inc() - } - - getParent := func() *node.Node { - p, err := n.Parent(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read parent") - return nil - } - return p - } - - now := time.Now() - if failed { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - sublog.Error().Err(err).Msg("reading node for session failed") - } - if latestSession == session.ID() { - // propagate reverted sizeDiff after failed postprocessing - if err := fs.tp.Propagate(ctx, n, -session.SizeDiff()); err != nil { - sublog.Error().Err(err).Msg("could not propagate tree size change") - } - } - } else if p := getParent(); p != nil { - // update parent tmtime to propagate etag change after successful postprocessing - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - sublog.Error().Err(err).Msg("could not propagate etag change") - } - } - - session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) - - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionsPath"] != "" { - isVersion = true - } - } - - if err := events.Publish( - ctx, - fs.stream, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - }, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), - IsVersion: isVersion, - ImpersonatingUser: ev.ImpersonatingUser, - }, - ); err != nil { - sublog.Error().Err(err).Msg("Failed to publish UploadReady event") - } - case events.RestartPostprocessing: - sublog := log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - s, err := session.URL(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not create url") - return - } - - metrics.UploadSessionsRestarted.Inc() - - // restart postprocessing - if err := events.Publish(ctx, fs.stream, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(ctx), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - }); err != nil { - sublog.Error().Err(err).Msg("Failed to publish BytesReceived event") - } - case events.CleanUpload: - sublog := log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) - case events.RevertRevision: - sublog := log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node") - return - } - - if err := n.RevertCurrentRevision(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to revert revision") - return - } - case events.PostprocessingStepFinished: - sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - if ev.FinishedStep != events.PPStepAntivirus { - // atm we are only interested in antivirus results - return - } - - res := ev.Result.(events.VirusscanResult) - if res.ErrorMsg != "" { - // scan failed somehow - // Should we handle this here? - return - } - sublog = log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() - - var n *node.Node - switch ev.UploadID { - case "": - // uploadid is empty -> this was an on-demand scan - /* ON DEMAND SCANNING NOT SUPPORTED ATM - ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) - ref := &provider.Reference{ResourceId: ev.ResourceID} - - no, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") - continue - - } - n = no - if ev.Outcome == events.PPOutcomeDelete { - // antivir wants us to delete the file. We must obey and need to - - // check if there a previous versions existing - revs, err := fs.ListRevisions(ctx, ref) - if len(revs) == 0 { - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") - } - - // no versions -> trash file - err := fs.Delete(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") - continue - } - - // now purge it from the recycle bin - if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - - // we have versions - find the newest - versions := make(map[uint64]string) // remember all versions - we need them later - var nv uint64 - for _, v := range revs { - versions[v.Mtime] = v.Key - if v.Mtime > nv { - nv = v.Mtime - } - } - - // restore newest version - if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") - continue - } - - // now find infected version - revs, err = fs.ListRevisions(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") - } - - for _, v := range revs { - // we looking for a version that was previously not there - if _, ok := versions[v.Mtime]; ok { - continue - } - - if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") - } - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - */ - default: - // uploadid is not empty -> this is an async upload - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - - n, err = session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node after scan") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - - session.SetScanData(res.Description, res.Scandate) - if err := session.Persist(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to persist scan results") - } - } - - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - sublog.Error().Err(err).Msg("Failed to set scan results") - return - } - - metrics.UploadSessionsScanned.Inc() - default: - log.Error().Interface("event", ev).Msg("Unknown event") - } -} // Shutdown shuts down the storage func (fs *Decomposedfs) Shutdown(ctx context.Context) error { @@ -971,6 +613,10 @@ func (fs *Decomposedfs) GetMD(ctx context.Context, ref *provider.Reference, mdKe return } + if node.IsProcessing(ctx) { + return nil, errtypes.ResourceProcessing(ref.String()) + } + rp, err := fs.p.AssemblePermissions(ctx, node) switch { case err != nil: diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 3f3dadb7352..25fa308f519 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -1283,9 +1283,16 @@ func (n *Node) SetScanData(ctx context.Context, info string, date time.Time) err return n.SetXattrsWithContext(ctx, attribs, true) } -// ScanData returns scanning information of the node +// ScanData returns scanning information of the node. +// It first checks the new arbitrary-metadata keys (written by the coordinator +// via SetArbitraryMetadata), then falls back to the legacy xattr keys so that +// nodes scanned before this change still return correct results. func (n *Node) ScanData(ctx context.Context) (scanned bool, virus string, scantime time.Time) { - ti, _ := n.XattrString(ctx, prefixes.ScanDatePrefix) + ti, _ := n.XattrString(ctx, prefixes.MetadataPrefix+"scandate") + if ti == "" { + // fall back to legacy xattr key written by the old decomposedfs path + ti, _ = n.XattrString(ctx, prefixes.ScanDatePrefix) + } if ti == "" { return // not scanned yet } @@ -1295,9 +1302,9 @@ func (n *Node) ScanData(ctx context.Context) (scanned bool, virus string, scanti return } - i, err := n.XattrString(ctx, prefixes.ScanStatusPrefix) - if err != nil { - return + i, _ := n.XattrString(ctx, prefixes.MetadataPrefix+"scanstatus") + if i == "" { + i, _ = n.XattrString(ctx, prefixes.ScanStatusPrefix) } return true, i, t diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index abacbd25cd9..6c41810d2ff 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -559,6 +559,12 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") } + if p, err := n.Parent(ctx); err == nil && p != nil { + now := time.Now() + _ = p.SetTMTime(ctx, &now) + _ = fs.tp.Propagate(ctx, p, 0) + } + // etag is a best-effort, recomputable value; a failure here must not fail an // already-committed upload (matches the legacy Upload path). etag, _ := node.CalculateEtag(n.ID, mtime) diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index 1a42f20878b..0cc27b0a664 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -178,7 +178,6 @@ func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error { prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), } - // At this point we scope by the space to create the final file in the final location if session.store.um != nil && session.info.Storage["SpaceGid"] != "" { gid, err := strconv.Atoi(session.info.Storage["SpaceGid"]) diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 35827ec15c0..1be2fefb1f3 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -3,7 +3,6 @@ package decomposedfs import ( "bytes" "context" - "io" "os" "path/filepath" @@ -20,7 +19,6 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/aspects" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/permissions" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/permissions/mocks" @@ -29,6 +27,7 @@ import ( treemocks "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/tree/mocks" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/store" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" "github.com/owncloud/reva/v2/tests/helpers" "github.com/rs/zerolog" @@ -74,8 +73,9 @@ var _ = Describe("Async file uploads", Ordered, func() { con chan interface{} uploadID string - fs storage.FS - o *options.Options + fs storage.FS + coord pkgupload.Coordinator + o *options.Options lu *lookup.Lookup pmock *mocks.PermissionsChecker cs3permissionsclient *mocks.CS3PermissionsClient @@ -119,6 +119,19 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(item.Path).To(Equal(ref.Path)) return len(resources) == 1, utils.ReadPlainFromOpaque(item.Opaque, "status"), int(item.GetSize()) } + // tusUpload writes content via the coordinator TUS path (GetUpload → + // WriteChunk → FinishUpload) instead of the legacy fs.Upload path. + // This is needed because coordinator.InitiateUpload calls TouchFile, so + // the node already exists; FinishUploadDecomposed (legacy path) would + // try to create it again and fail with EEXIST. + tusUpload = func(id string, content []byte) { + up, err := coord.GetUpload(ctx, id) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(content)) + Expect(err).ToNot(HaveOccurred()) + Expect(up.FinishUpload(ctx)).To(Succeed()) + } + parentSize = func() int { parentInfo, err := fs.GetMD(ctx, rootRef, []string{}, []string{}) Expect(err).ToNot(HaveOccurred()) @@ -171,6 +184,7 @@ var _ = Describe("Async file uploads", Ordered, func() { InitiateFileUpload: true, ListContainer: true, ListFileVersions: true, + Delete: true, }, nil) // setup fs @@ -184,8 +198,16 @@ var _ = Describe("Async file uploads", Ordered, func() { EventStream: stream.Chan{pub, con}, Trashbin: &DecomposedfsTrashbin{}, } - fs, err = New(o, aspects, &zerolog.Logger{}) + dfs, err := New(o, aspects, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) + // Wire the coordinator so postprocessing events are consumed during tests. + d := dfs.(*Decomposedfs) + fileStore := pkgupload.NewFileStore(o.Root, pkgupload.TokenOptions{}, &zerolog.Logger{}) + var coordErr error + coord, coordErr = pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, true, "", "dcfs", 1, &zerolog.Logger{}, "") + Expect(coordErr).ToNot(HaveOccurred()) + Expect(coord.Start(aspects.EventStream)).To(Succeed()) + fs = d resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"}) Expect(err).ToNot(HaveOccurred()) @@ -194,39 +216,25 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(err).ToNot(HaveOccurred()) ref.ResourceId = &resID - bs.On("Upload", mock.AnythingOfType("*node.Node"), mock.AnythingOfType("string"), mock.Anything). - Return(nil). - Run(func(args mock.Arguments) { - n := args.Get(0).(*node.Node) - data, err := os.ReadFile(args.Get(1).(string)) - Expect(err).ToNot(HaveOccurred()) - Expect(len(data)).To(Equal(int(n.Blobsize))) - }) + bs.On("UploadFromReader", mock.AnythingOfType("*node.Node"), mock.Anything, mock.AnythingOfType("int64")). + Return(nil) // start upload of a file - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 10, map[string]string{}) Expect(err).ToNot(HaveOccurred()) Expect(len(uploadIds)).To(Equal(2)) Expect(uploadIds["simple"]).ToNot(BeEmpty()) Expect(uploadIds["tus"]).ToNot(BeEmpty()) - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(firstContent)), - Length: int64(len(firstContent)), - }, nil) - Expect(err).ToNot(HaveOccurred()) - uploadID = uploadIds["simple"] + tusUpload(uploadID, firstContent) // wait for bytes received event _, ok := (<-pub).(events.BytesReceived) Expect(ok).To(BeTrue()) // blobstore not called yet - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) }) AfterEach(func() { @@ -251,7 +259,7 @@ var _ = Describe("Async file uploads", Ordered, func() { succeedPostprocessing(uploadID) // blobstore called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) // node ready resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -281,7 +289,7 @@ var _ = Describe("Async file uploads", Ordered, func() { failPostprocessing(uploadID, events.PPOutcomeDelete) // blobstore still not called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) // node gone resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -310,7 +318,7 @@ var _ = Describe("Async file uploads", Ordered, func() { failPostprocessing(uploadID, events.PPOutcomeAbort) // blobstore still not called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) // node gone resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -333,34 +341,27 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(len(revs)).To(Equal(0)) // upload again - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 10, map[string]string{}) Expect(err).ToNot(HaveOccurred()) Expect(len(uploadIds)).To(Equal(2)) Expect(uploadIds["simple"]).ToNot(BeEmpty()) Expect(uploadIds["tus"]).ToNot(BeEmpty()) - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(firstContent)), - Length: int64(len(firstContent)), - }, nil) - Expect(err).ToNot(HaveOccurred()) - uploadID = uploadIds["simple"] + tusUpload(uploadID, firstContent) // wait for bytes received event _, ok := (<-pub).(events.BytesReceived) Expect(ok).To(BeTrue()) - // version already created + // version is not yet created at this point — it will be created when + // CommitUpload runs on PostprocessingFinished, not at InitiateUpload time. revs, err = fs.ListRevisions(ctx, ref) Expect(err).To(BeNil()) - Expect(len(revs)).To(Equal(1)) + Expect(len(revs)).To(Equal(0)) // at this stage: blobstore called once for the original file - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) }) @@ -373,7 +374,7 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(len(revs)).To(Equal(1)) // blobstore now called twice - for original file and new version - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 2) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 2) // bytes are gone from upload path _, err = os.Stat(filepath.Join(o.Root, "uploads", uploadID)) @@ -399,288 +400,98 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(err).ToNot(BeNil()) // blobstore still called only once for the original file - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) }) }) - When("Two uploads are processed in parallel", func() { - var secondUploadID string - - JustBeforeEach(func() { - // upload again - uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + When("a second upload is attempted while the first is still in postprocessing", func() { + // The coordinator serializes uploads via MarkProcessing: a second FinishUpload + // while the first session holds the processing slot returns ResourceProcessing. + // InitiateUpload itself succeeds — the conflict is detected when bytes arrive. + + It("rejects the second FinishUpload with ResourceProcessing", func() { + // First upload is in postprocessing (BytesReceived consumed in BeforeEach). + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) - - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} + secondUploadID := uploadIds["simple"] - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(secondContent)), - Length: int64(len(secondContent)), - }, nil) + up, err := coord.GetUpload(ctx, secondUploadID) Expect(err).ToNot(HaveOccurred()) - - secondUploadID = uploadIds["simple"] - - // wait for bytes received event - _, ok := (<-pub).(events.BytesReceived) - Expect(ok).To(BeTrue()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) + Expect(err).ToNot(HaveOccurred()) + err = up.FinishUpload(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("resource is processing")) }) - It("doesn't remove processing status when first upload is finished", func() { - succeedPostprocessing(uploadID) - - _, status, _ := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - }) + It("leaves no orphan session files after rejection", func() { + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID := uploadIds["simple"] - It("removes processing status when second upload is finished, even if first isn't", func() { - succeedPostprocessing(secondUploadID) + up, err := coord.GetUpload(ctx, secondUploadID) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) + Expect(err).ToNot(HaveOccurred()) + _ = up.FinishUpload(ctx) // expected to fail - _, status, _ := fileStatus() - Expect(status).To(Equal("")) + // touchAndMark rollback removes .bin and .info for the second session. + // Only the first upload's .bin and .info should remain. + entries, readErr := os.ReadDir(filepath.Join(o.Root, "uploads")) + Expect(readErr).ToNot(HaveOccurred()) + Expect(len(entries)).To(Equal(2)) }) + }) - It("correctly calculates the size when the second upload is finished, even if first is deleted", func() { - succeedPostprocessing(secondUploadID) - - _, status, size := fileStatus() - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(uploadID, events.PPOutcomeDelete) - - // check processing status - _, _, size = fileStatus() - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - }) + When("uploads are processed sequentially (second after first completes)", func() { + var secondUploadID string - It("the first can succeed before the second succeeds", func() { + JustBeforeEach(func() { + // Complete the first upload's postprocessing before starting the second. succeedPostprocessing(uploadID) - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal((len(secondContent)))) - - // parent size should match the second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(secondUploadID) - - // check processing status has been removed - _, status, size = fileStatus() - Expect(status).To(Equal("")) - - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) + // Second upload. + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID = uploadIds["simple"] + tusUpload(secondUploadID, secondContent) - // file should have one revision - Expect(revisionCount()).To(Equal(1)) + // wait for bytes received event + _, ok := (<-pub).(events.BytesReceived) + Expect(ok).To(BeTrue()) }) - It("the first can succeed after the second succeeds", func() { + It("succeeds and creates a revision", func() { succeedPostprocessing(secondUploadID) _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(uploadID) - - _, status, size = fileStatus() - // check processing status is still unset Expect(status).To(Equal("")) - // size should still match the second upload Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload Expect(parentSize()).To(Equal(len(secondContent))) - - // file should have one revision Expect(revisionCount()).To(Equal(1)) }) - It("the first can succeed before the second fails", func() { - succeedPostprocessing(uploadID) - - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match the second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - _, status, size = fileStatus() - // check processing status has been removed - Expect(status).To(Equal("")) - // size should match the first upload - Expect(size).To(Equal(len(firstContent))) - - // parent size should match first upload - Expect(parentSize()).To(Equal(len(firstContent))) - - // file should not have any revisions - Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can succeed after the second fails", func() { + It("reverts to previous content when second upload is deleted", func() { failPostprocessing(secondUploadID, events.PPOutcomeDelete) - _, _, size := fileStatus() - // check processing status has not been unset - // FIXME we need to fall back to the previous processing id - // Expect(status).To(Equal("processing")) - // size should match the first upload - Expect(size).To(Equal(len(firstContent))) - - // parent size should match first upload as well - Expect(parentSize()).To(Equal(len(firstContent))) - - succeedPostprocessing(uploadID) - _, status, size := fileStatus() - // check processing status is now unset Expect(status).To(Equal("")) - // size should still match the first upload Expect(size).To(Equal(len(firstContent))) - - // parent size should still match first upload Expect(parentSize()).To(Equal(len(firstContent))) - - // file should not have any revisions Expect(revisionCount()).To(Equal(0)) }) - It("the first can fail before the second succeeds", func() { - failPostprocessing(uploadID, events.PPOutcomeDelete) - - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(secondUploadID) - - _, status, size = fileStatus() - // check processing status has been removed - Expect(status).To(Equal("")) - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - // file should not have any revisions - // FIXME we need to delete the revision - // Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can fail after the second succeeds", func() { - succeedPostprocessing(secondUploadID) - - _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(uploadID, events.PPOutcomeDelete) - - _, status, size = fileStatus() - // check processing status is still unset - Expect(status).To(Equal("")) - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - // file should not have any revisions - // FIXME we need to delete the revision - // Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can fail before the second fails", func() { - failPostprocessing(uploadID, events.PPOutcomeDelete) + It("reverts to previous content when second upload is aborted and keeps bin", func() { + failPostprocessing(secondUploadID, events.PPOutcomeAbort) _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - // check file has been removed - // if all uploads have been processed with outcome delete -> delete the file - // exists, _, _ := fileStatus() - // FIXME this should be false, but we are not deleting the resource - // Expect(exists).To(BeFalse()) - - // parent size should be 0 - // FIXME we are not correctly reverting the sizediff - // Expect(parentSize()).To(Equal(0)) - }) - - It("the first can fail after the second fails", func() { - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded Expect(status).To(Equal("")) - // size should match the first upload Expect(size).To(Equal(len(firstContent))) - - // parent size should match second first as well Expect(parentSize()).To(Equal(len(firstContent))) - failPostprocessing(uploadID, events.PPOutcomeDelete) - - // check file has been removed - // if all uploads have been processed with outcome delete -> delete the file - // exists, _, _ := fileStatus() - // FIXME this should be false, but we are not deleting the resource - // Expect(exists).To(BeFalse()) - - // parent size should be 0 - // FIXME we are not correctly reverting the sizediff - // Expect(parentSize()).To(Equal(0)) + // bytes kept + _, statErr := os.Stat(filepath.Join(o.Root, "uploads", secondUploadID)) + Expect(statErr).To(BeNil()) }) }) }) diff --git a/pkg/storage/utils/middleware/middleware.go b/pkg/storage/utils/middleware/middleware.go index e589f645e8a..1f858b9005d 100644 --- a/pkg/storage/utils/middleware/middleware.go +++ b/pkg/storage/utils/middleware/middleware.go @@ -27,7 +27,6 @@ import ( tusd "github.com/tus/tusd/v2/pkg/handler" "github.com/owncloud/reva/v2/pkg/storage" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/owncloud/reva/v2/pkg/storagespace" ) @@ -51,21 +50,23 @@ func NewFS(next storage.FS, hooks ...Hook) *FS { } // ListUploadSessions returns the upload sessions matching the given filter +// TODO(OCISDEV-901): remove once UploadSessionLister is removed from storage.FS (coordinator responsibility). func (f *FS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { return f.next.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) } // UseIn tells the tus upload middleware which extensions it supports. +// TODO(OCISDEV-901): coordinator now handles TUS registration globally. Is per-driver UseIn still needed for +// capability variation, or can this be removed along with storage.ComposableFS? func (f *FS) UseIn(composer *tusd.StoreComposer) { f.next.(storage.ComposableFS).UseIn(composer) } -// NewUpload returns a new tus Upload instance +// TODO(OCISDEV-901): remove NewUpload and GetUpload. TUS DataStore is the coordinator's responsibility, not the driver's. func (f *FS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { return f.next.(tusd.DataStore).NewUpload(ctx, info) } -// NewUpload returns a new tus Upload instance func (f *FS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { return f.next.(tusd.DataStore).GetUpload(ctx, id) } @@ -74,21 +75,21 @@ func (f *FS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err // To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination // the storage needs to implement AsTerminatableUpload func (f *FS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.OcisSession) + return up.(tusd.TerminatableUpload) } // AsLengthDeclarableUpload returns a LengthDeclarableUpload // To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation // the storage needs to implement AsLengthDeclarableUpload func (f *FS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.OcisSession) + return up.(tusd.LengthDeclarableUpload) } // AsConcatableUpload returns a ConcatableUpload // To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation // the storage needs to implement AsConcatableUpload func (f *FS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.OcisSession) + return up.(tusd.ConcatableUpload) } func (f *FS) GetHome(ctx context.Context) (string, error) { diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go new file mode 100644 index 00000000000..381429627de --- /dev/null +++ b/pkg/upload/coordinated_upload.go @@ -0,0 +1,151 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/owncloud/reva/v2/pkg/errtypes" +) + +// coordinatedUpload is the TUS interface adapter between the tusd library and the coordinator. +// It implements tusd.Upload, TerminatableUpload, LengthDeclarableUpload, and ConcatableUpload. +// All real upload logic lives in coordinator; this type only translates tusd callbacks. +type coordinatedUpload struct { + session Session + coord *coordinator +} + +func (u *coordinatedUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return u.session.GetInfo(ctx) +} + +func (u *coordinatedUpload) GetReader(ctx context.Context) (io.ReadCloser, error) { + return u.session.GetReader(ctx) +} + +func (u *coordinatedUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + return u.session.WriteChunk(ctx, offset, src) +} + +func (u *coordinatedUpload) Terminate(ctx context.Context) error { + u.session.Cleanup(true, true) + // Node may not exist if Terminate is called before FinishUpload. + ref := u.session.Reference() + if ref.ResourceId.GetOpaqueId() != "" { + _ = u.coord.fs.MarkProcessing(ctx, &ref, false, u.session.ID()) + if !u.session.NodeExists() { + _, _ = u.coord.fs.Delete(ctx, &ref) + } + } + return nil +} + +func (u *coordinatedUpload) DeclareLength(ctx context.Context, length int64) error { + u.session.SetSize(length) + u.session.SetSizeIsDeferred(false) + return u.session.Persist(ctx) +} + +func (u *coordinatedUpload) ConcatUploads(ctx context.Context, partials []tusd.Upload) error { + file, err := os.OpenFile(u.session.BinPath(), os.O_WRONLY|os.O_APPEND, 0664) + if err != nil { + return err + } + defer file.Close() + for _, partial := range partials { + cu, ok := partial.(*coordinatedUpload) + if !ok { + return fmt.Errorf("coordinator: unexpected partial type %T", partial) + } + src, err := cu.session.GetReader(ctx) + if err != nil { + return err + } + _, copyErr := io.Copy(file, src) + src.Close() + if copyErr != nil { + return copyErr + } + } + return nil +} + +func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { + err := u.coord.finishUpload(u.session.Context(ctx), u.session) + switch err.(type) { + case nil: + return nil + case errtypes.ResourceProcessing, errtypes.TooEarly: + return tusd.NewError("ERR_TOO_EARLY", err.Error(), http.StatusTooEarly) + case errtypes.Aborted: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed) + case errtypes.PreconditionFailed: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusMethodNotAllowed) + case errtypes.Locked: + return tusd.NewError("ERR_LOCKED", err.Error(), http.StatusLocked) + case errtypes.BadRequest: + return tusd.NewError("ERR_BAD_REQUEST", err.Error(), http.StatusBadRequest) + case errtypes.ChecksumMismatch: + return tusd.NewError("ERR_CHECKSUM_MISMATCH", err.Error(), errtypes.StatusChecksumMismatch) + default: + return err + } +} + +// UseIn registers the coordinator as the TUS data store in the composer. +func (c *coordinator) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(c) + composer.UseTerminater(c) + composer.UseConcater(c) + composer.UseLengthDeferrer(c) +} + +// NewUpload is not supported; uploads are initiated via the CS3 API. +func (c *coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { + return nil, errNotImplemented +} + +// GetUpload returns the upload session wrapped in a coordinatedUpload so the +// TUS FinishUpload hook runs the coordinator path rather than the legacy one. +func (c *coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + return &coordinatedUpload{session: session, coord: c}, nil +} + +func (c *coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*coordinatedUpload) +} + +func (c *coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*coordinatedUpload) +} + +func (c *coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*coordinatedUpload) +} diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go new file mode 100644 index 00000000000..86a1c4f9161 --- /dev/null +++ b/pkg/upload/coordinator.go @@ -0,0 +1,643 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package upload provides the driver-agnostic upload coordinator: +// TUS session management, postprocessing event loop, lifecycle event +// publishing. +package upload + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/storage/utils/chunking" + "github.com/owncloud/reva/v2/pkg/utils" +) + +var tracer trace.Tracer + +func init() { + tracer = otel.Tracer("github.com/owncloud/reva/pkg/upload") +} + +// impersonatingUser extracts the impersonating user from the context user's opaque field. +// Returns nil when no impersonation is in effect. +func impersonatingUser(ctx context.Context) *user.User { + u, ok := ctxpkg.ContextGetUser(ctx) + if !ok || u == nil { + return nil + } + if !utils.ExistsInOpaque(u.Opaque, "impersonating-user") { + return nil + } + iu := &user.User{} + if err := utils.ReadJSONFromOpaque(u.Opaque, "impersonating-user", iu); err != nil { + return nil + } + return iu +} + +var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) + +// rewriteChunkedRef strips the chunk suffix from ref.Path and returns the chunk basename. +// Only called when chunking.IsChunked(ref.GetPath()) is true. +func rewriteChunkedRef(ref *provider.Reference) (*provider.Reference, string, error) { + ci, err := chunking.GetChunkBLOBInfo(ref.GetPath()) + if err != nil { + return nil, "", errtypes.BadRequest(err.Error()) + } + return &provider.Reference{ResourceId: ref.ResourceId, Path: ci.Path}, filepath.Base(ref.GetPath()), nil +} + +// rollback unmarks processing, cleans up session files, and deletes the placeholder +// node if it was created by this upload (NodeExists=false at initiation). +func (c *coordinator) rollback(ctx context.Context, session Session) { + ref := session.Reference() + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(true, true) + if !session.NodeExists() { + _, _ = c.fs.Delete(ctx, &ref) + } +} + +// finishSync commits the upload inline, without postprocessing. +// Used when async=false or the upload is empty (size==0). +func (c *coordinator) finishSync(ctx context.Context, session Session) error { + ref := session.Reference() + f, err := os.Open(session.BinPath()) + if err != nil { + c.rollback(ctx, session) + return err + } + cs := session.Checksums() + if _, err := c.fs.CommitUpload(ctx, &ref, storage.UploadSource{ + Body: f, + Length: session.Size(), + Metadata: session.Metadata(), + Checksums: cs, + }); err != nil { + c.rollback(ctx, session) + return err + } + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(true, true) + metrics.UploadSessionsFinalized.Inc() + return nil +} + +// triggerPostprocessing publishes BytesReceived to start async postprocessing. +func (c *coordinator) triggerPostprocessing(ctx context.Context, session Session) error { + s, err := session.URL(ctx) + if err != nil { + c.rollback(ctx, session) + return err + } + executingUser, _ := ctxpkg.ContextGetUser(ctx) + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: executingUser, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + ImpersonatingUser: impersonatingUser(ctx), + }); err != nil { + c.rollback(ctx, session) + return err + } + return nil +} + +// finishUpload is called after all bytes are received (TUS FinishUpload and simple PUT). +// It creates the node, validates checksums, then either commits inline or triggers postprocessing. +func (c *coordinator) finishUpload(ctx context.Context, session Session) error { + if err := c.touchAndMark(ctx, session); err != nil { + return err + } + if err := verifyAndStoreChecksums(ctx, session); err != nil { + c.rollback(ctx, session) + return err + } + if err := session.Persist(ctx); err != nil { + c.rollback(ctx, session) + return err + } + + metrics.UploadProcessing.Inc() + metrics.UploadSessionsBytesReceived.Inc() + + if !c.async || session.Size() == 0 { + return c.finishSync(ctx, session) + } + return c.triggerPostprocessing(ctx, session) +} + +// verifyAndStoreChecksums computes checksums from the staged binary, validates against +// any client-supplied checksum, and stores them on the session. +func verifyAndStoreChecksums(ctx context.Context, session Session) error { + sha1h, md5h, adler32h, err := calculateChecksums(ctx, session.BinPath()) + if err != nil { + return err + } + info, err := session.GetInfo(ctx) + if err != nil { + return err + } + if checksum := info.MetaData["checksum"]; checksum != "" { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + var checkErr error + switch parts[0] { + case "sha1": + checkErr = checkHash(parts[1], sha1h) + case "md5": + checkErr = checkHash(parts[1], md5h) + case "adler32": + checkErr = checkHash(parts[1], adler32h) + default: + checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if checkErr != nil { + session.Cleanup(true, true) + return checkErr + } + } + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + +// touchAndMark creates the node (new files only) and marks it as processing. +// Called from FinishUpload and Upload after all bytes have been received. +func (c *coordinator) touchAndMark(ctx context.Context, session Session) error { + if !session.NodeExists() { + pathRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeParentID(), + }, + Path: session.Filename(), + } + result, err := c.fs.TouchFile(ctx, pathRef, false, session.Metadata()["mtime"]) + if err != nil { + session.Cleanup(true, true) + if _, ok := err.(errtypes.IsNotFound); ok { + return errtypes.PreconditionFailed(err.Error()) + } + return err + } + session.SetStorageValue("NodeId", result.ResourceID.GetOpaqueId()) + session.SetStorageValue("SpaceRoot", result.SpaceID) + if result.SpaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", result.SpaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", result.SpaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(result.SpaceOwner.GetType())) + } + } + nodeRef := session.Reference() + if err := c.fs.MarkProcessing(ctx, &nodeRef, true, session.ID()); err != nil { + session.Cleanup(true, true) + if !session.NodeExists() { + _, _ = c.fs.Delete(ctx, &nodeRef) + } + return err + } + return session.Persist(ctx) +} + +// Coordinator owns the full upload lifecycle: session initiation, TUS data transfer, +// postprocessing event loop, and UploadReady publishing. +type Coordinator interface { + InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) + Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) + GetUpload(ctx context.Context, id string) (tusd.Upload, error) + UseIn(composer *tusd.StoreComposer) + ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) + Start(stream events.Consumer) error +} + +// coordinator is the concrete implementation of Coordinator. +type coordinator struct { + fs storage.FS + store SessionStore + pub events.Publisher + async bool + mountID string + numConc int + conGroup string + log *zerolog.Logger + chunkHandler *chunking.ChunkHandler // nil when legacy chunking v1 is not needed +} + +// NewCoordinator constructs a Coordinator. Call Start to begin consuming events. +// async=true requires a non-nil pub. +// chunkFolder enables legacy chunking v1 support; pass "" to disable it. +func NewCoordinator( + fs storage.FS, + store SessionStore, + pub events.Publisher, + async bool, + mountID string, + consumerGroup string, + numConsumers int, + log *zerolog.Logger, + chunkFolder string, +) (Coordinator, error) { + if async && pub == nil { + return nil, fmt.Errorf("need event stream for async upload processing") + } + if numConsumers <= 0 { + numConsumers = 1 + } + var ch *chunking.ChunkHandler + if chunkFolder != "" { + ch = chunking.NewChunkHandler(chunkFolder) + } + return &coordinator{ + fs: fs, + store: store, + pub: pub, + async: async, + mountID: mountID, + numConc: numConsumers, + conGroup: consumerGroup, + log: log, + chunkHandler: ch, + }, nil +} + +func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + var chunkName string + if chunking.IsChunked(ref.GetPath()) { // check legacy chunking v1 + var rerr error + ref, chunkName, rerr = rewriteChunkedRef(ref) + if rerr != nil { + return nil, rerr + } + } + + existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) + var nodeExists bool + switch err.(type) { + case nil: + nodeExists = true + case errtypes.IsNotFound: + nodeExists = false + default: + return nil, err + } + + var nodeID, spaceID, parentID, dir, nodeName string + var spaceOwner *user.UserId + + // check quota + if uploadLength >= 0 { + spaceRef := &provider.Reference{ResourceId: &provider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + }} + if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil { + var existingSize uint64 + if nodeExists { + existingSize = existing.GetSize() + } + netRequired := uint64(uploadLength) + if existingSize < netRequired { + netRequired -= existingSize + } else { + netRequired = 0 + } + if remaining < netRequired { + return nil, errtypes.InsufficientStorage("quota exceeded") + } + } + } + + if nodeExists { + nodeID = existing.GetId().GetOpaqueId() + spaceID = existing.GetId().GetSpaceId() + parentID = existing.GetParentId().GetOpaqueId() + dir = filepath.Dir(ref.GetPath()) + nodeName = existing.GetName() + spaceOwner = existing.GetOwner() + + diskLock, _ := c.fs.GetLock(ctx, ref) + contextLockID, _ := ctxpkg.ContextGetLockID(ctx) + if diskLock != nil { + switch contextLockID { + case "": + return nil, errtypes.Locked(diskLock.LockId) + case diskLock.LockId: + // ok + default: + return nil, errtypes.Aborted("mismatching lock") + } + } else if contextLockID != "" { + return nil, errtypes.Aborted("not locked") + } + } else { + spaceID = ref.GetResourceId().GetSpaceId() + dir = filepath.Dir(ref.GetPath()) + nodeName = filepath.Base(ref.GetPath()) + } + + if nodeExists { + if !existing.GetPermissionSet().GetInitiateFileUpload() { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } + if existing.GetType() == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + return nil, errtypes.PreconditionFailed("resource is not a file") + } + if metadata["if-none-match"] == "*" { + return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", parentID, nodeName, nodeID)) + } + } else { + parentRef := &provider.Reference{ + ResourceId: ref.GetResourceId(), + Path: dir, + } + parentMD, pErr := c.fs.GetMD(ctx, parentRef, []string{}, []string{}) + switch pErr.(type) { + case nil: + case errtypes.IsNotFound: + // RFC 4918: missing intermediate dir → 409, no permission → 404. + // GetMD returns NotFound for both (hides resources from unauthorized callers). + // Walk up the path: if an ancestor is visible, the dir is truly missing (409). + // If nothing is visible up to the root, caller has no access (404). + ancestor := dir + permDenied := true + for ancestor != "." && ancestor != "/" { + ancestor = filepath.Dir(ancestor) + ancestorRef := &provider.Reference{ResourceId: ref.GetResourceId(), Path: ancestor} + if _, aErr := c.fs.GetMD(ctx, ancestorRef, []string{}, []string{}); aErr == nil { + permDenied = false + break + } + } + if permDenied { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } + return nil, errtypes.PreconditionFailed(pErr.Error()) + default: + return nil, pErr + } + if !parentMD.GetPermissionSet().GetInitiateFileUpload() { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } + parentID = parentMD.GetId().GetOpaqueId() + spaceID = parentMD.GetId().GetSpaceId() + } + + if nodeName == "" { + return nil, errtypes.BadRequest("coordinator: missing filename in ref") + } + if dir == "" { + return nil, errtypes.BadRequest("coordinator: could not determine upload directory") + } + + session := c.store.New(ctx) + session.SetMetadata("filename", nodeName) + session.SetStorageValue("NodeName", nodeName) + session.SetMetadata("dir", dir) + session.SetStorageValue("Dir", dir) + session.SetStorageValue("SpaceRoot", spaceID) + if nodeExists { + session.SetStorageValue("NodeId", nodeID) + session.SetStorageValue("NodeExists", "true") + } + session.SetStorageValue("NodeParentId", parentID) + if spaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", spaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", spaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(spaceOwner.GetType())) + } + + usr := ctxpkg.ContextMustGetUser(ctx) + session.SetExecutant(usr) + + lockID, _ := ctxpkg.ContextGetLockID(ctx) + session.SetMetadata("lockid", lockID) + + iid, _ := ctxpkg.ContextGetInitiator(ctx) + session.SetMetadata("initiatorid", iid) + + session.SetSize(uploadLength) + + var mtimeSet bool + if metadata != nil { + session.SetMetadata("providerID", metadata["providerID"]) + if v, ok := metadata["mtime"]; ok && v != "null" { + session.SetMetadata("mtime", v) + mtimeSet = true + } + if v, ok := metadata["expires"]; ok && v != "null" { + session.SetMetadata("expires", v) + } + if _, ok := metadata["sizedeferred"]; ok { + session.SetSizeIsDeferred(true) + } + if checksum, ok := metadata["checksum"]; ok { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1", "md5", "adler32": + session.SetMetadata("checksum", checksum) + default: + return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + } + if v := metadata["if-match"]; v != "" { + session.SetMetadata("if-match", v) + } + if v := metadata["if-none-match"]; v != "" { + session.SetMetadata("if-none-match", v) + } + if v := metadata["if-unmodified-since"]; v != "" { + session.SetMetadata("if-unmodified-since", v) + } + } + + if !mtimeSet { + session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now())) + } + if chunkName != "" { // check legacy chunking v1 + session.SetStorageValue("Chunk", chunkName) + } + + if err := session.TouchBin(); err != nil { + return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) + } + if err := session.Persist(ctx); err != nil { + session.Cleanup(true, true) + return nil, fmt.Errorf("coordinator: could not persist session: %w", err) + } + + metrics.UploadSessionsInitiated.Inc() + + if uploadLength == 0 { + // Zero-length uploads complete immediately without postprocessing. + if err := c.finishUpload(ctx, session); err != nil { + return nil, err + } + return map[string]string{ + "simple": session.ID(), + "tus": session.ID(), + }, nil + } + + return map[string]string{ + "simple": session.ID(), + "tus": session.ID(), + }, nil +} + +// Upload handles the simple (single-PUT) upload path so the coordinator owns +// the complete upload lifecycle regardless of the datatx protocol used. +// simple.go calls fs.Upload(); when fs is a *Coordinator this method intercepts. +func (c *coordinator) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { + id := strings.TrimPrefix(req.Ref.GetPath(), "/") + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + ctx = session.Context(ctx) + + if session.Chunk() != "" { // check legacy chunking v1 + assembled, assembledSize, done, err := c.chunkHandler.Assemble(session.Chunk(), req.Body) + if err != nil { + return nil, err + } + if !done { + session.Cleanup(true, true) + return nil, errtypes.PartialContent(req.Ref.String()) + } + defer assembled.Close() + req.Body, req.Length = assembled, assembledSize + session.SetSize(assembledSize) + } + + size, err := session.WriteChunk(ctx, 0, req.Body) + if err != nil { + return nil, err + } + if size != req.Length { + return nil, errtypes.PartialContent(req.Ref.String()) + } + + if err := c.finishUpload(ctx, session); err != nil { + return nil, err + } + + executant := session.Executant() + uploadRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + } + if uff != nil { + uff(session.SpaceOwner(), &executant, uploadRef) + } + + ri := &provider.ResourceInfo{ + Id: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Name: session.Filename(), + } + if mt, ok := session.Metadata()["mtime"]; ok && mt != "" { + if t, err := utils.MTimeToTime(mt); err == nil { + ri.Etag, _ = utils.CalculateEtag(session.NodeID(), t) + ri.Mtime = utils.TimeToTS(t) + } + } + return ri, nil +} + +// ListUploadSessions returns upload sessions matching the given filter. +func (c *coordinator) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + if filter.ID != nil && *filter.ID != "" { + session, err := c.store.Get(ctx, *filter.ID) + if err != nil { + return nil, err + } + return []storage.UploadSession{session}, nil + } + sessions, err := c.store.List(ctx) + if err != nil { + return nil, err + } + result := []storage.UploadSession{} + now := time.Now() + for _, s := range sessions { + if filter.ID != nil && *filter.ID != "" && s.ID() != *filter.ID { + continue + } + if filter.Processing != nil && *filter.Processing != s.IsProcessing() { + continue + } + if filter.Expired != nil { + if *filter.Expired { + if now.Before(s.Expires()) { + continue + } + } else { + if now.After(s.Expires()) { + continue + } + } + } + if filter.HasVirus != nil { + sr, _ := s.ScanData() + infected := sr != "" + if *filter.HasVirus != infected { + continue + } + } + result = append(result, s) + } + return result, nil +} diff --git a/pkg/upload/coordinator_events_test.go b/pkg/upload/coordinator_events_test.go new file mode 100644 index 00000000000..9ec65da815e --- /dev/null +++ b/pkg/upload/coordinator_events_test.go @@ -0,0 +1,498 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "os" + "testing" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// TestHandlePostprocessingFinished_Continue covers PPOutcomeContinue. +func TestHandlePostprocessingFinished_Continue(t *testing.T) { + t.Run("happy path: CommitUpload, MarkProcessing(false), UploadReady{Failed:false}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.False(t, ev.Failed) + assert.Equal(t, session.ID(), ev.UploadID) + }) + + t.Run("CommitUpload fails: retryCommit=true, no MarkProcessing, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("disk full")) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + // MarkProcessing must NOT be called (retryCommit=true). + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + }) +} + +// TestHandlePostprocessingFinished_Abort covers PPOutcomeAbort. +func TestHandlePostprocessingFinished_Abort(t *testing.T) { + t.Run("new file: MarkProcessing(false), Delete, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + // Abort + new file: revertNodeMetadata=true → Delete called + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeAbort, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + }) + + t.Run("overwrite: MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeAbort, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + }) +} + +// TestHandlePostprocessingFinished_Delete covers PPOutcomeDelete. +func TestHandlePostprocessingFinished_Delete(t *testing.T) { + t.Run("new file: session cleaned, MarkProcessing(false), Delete, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeDelete, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("overwrite: session cleaned, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeDelete, + }) + + mockFs.AssertExpectations(t) + }) +} + +// TestHandlePostprocessingFinished_EdgeCases covers edge cases. +func TestHandlePostprocessingFinished_EdgeCases(t *testing.T) { + t.Run("different storageId ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingFinished(context.Background(), events.PostprocessingFinished{ + UploadID: "any", + ResourceID: &provider.ResourceId{ + StorageId: "different-mount", + OpaqueId: "node1", + }, + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertNotCalled(t, "GetMD", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + assert.Empty(t, pub.events) + }) + + t.Run("session not found: MarkProcessing via ResourceID ref", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + resID := &provider.ResourceId{ + StorageId: "test-mount", + OpaqueId: "node-orphan", + } + mockFs.On("MarkProcessing", mock.Anything, &provider.Reference{ResourceId: resID}, false, "orphan-id").Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(context.Background(), events.PostprocessingFinished{ + UploadID: "orphan-id", + ResourceID: resID, + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + assert.Empty(t, pub.events) + }) + + t.Run("GetMD fails for node: session cleaned, MarkProcessing, no UploadReady", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("n1")) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + assert.Empty(t, pub.events) + assert.NoFileExists(t, session.BinPath()) + }) +} + +// TestHandleRestartPostprocessing covers handleRestartPostprocessing. +func TestHandleRestartPostprocessing(t *testing.T) { + t.Run("happy path: BytesReceived with postprocessing-restart executant", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, _, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handleRestartPostprocessing(context.Background(), events.RestartPostprocessing{ + UploadID: session.ID(), + }) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, session.ID(), ev.UploadID) + assert.Equal(t, "postprocessing-restart", ev.ExecutingUser.GetId().GetOpaqueId()) + }) + + t.Run("session not found: no event published", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, _, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handleRestartPostprocessing(context.Background(), events.RestartPostprocessing{ + UploadID: "nonexistent", + }) + + assert.Empty(t, pub.events) + }) +} + +// TestHandleCleanUpload covers handleCleanUpload. +func TestHandleCleanUpload(t *testing.T) { + t.Run("KeepUpload=false, new file: Cleanup(true,true), MarkProcessing(false), Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + + mockFs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("KeepUpload=true, new file: Cleanup(false,false), MarkProcessing(false), Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: true, + }) + + mockFs.AssertExpectations(t) + // KeepUpload=true means Cleanup(false,false): bin and info stay. + assert.FileExists(t, session.BinPath()) + }) + + t.Run("KeepUpload=false, overwrite: Cleanup, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + + mockFs.AssertExpectations(t) + }) + + t.Run("Delete returns NotFound: silently ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), errtypes.NotFound("n1")) + + // Should not panic or return error to caller. + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + mockFs.AssertExpectations(t) + }) + + t.Run("session not found: no FS calls", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: "nonexistent", + KeepUpload: false, + }) + + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) +} + +// TestHandlePostprocessingStepFinished covers handlePostprocessingStepFinished. +func TestHandlePostprocessingStepFinished(t *testing.T) { + t.Run("antivirus clean scan: SetArbitraryMetadata called, session scan data persisted", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("SetArbitraryMetadata", mock.Anything, &ref, mock.MatchedBy(func(md *provider.ArbitraryMetadata) bool { + return md.Metadata["scanstatus"] == "clean" && md.Metadata["scandate"] != "" + })).Return(nil) + + scanDate := time.Now() + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + Infected: false, + Description: "clean", + Scandate: scanDate, + }, + }) + + mockFs.AssertExpectations(t) + + // Reload session from disk and check scan data was persisted. + reloaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + desc, _ := reloaded.ScanData() + assert.Equal(t, "clean", desc) + }) + + t.Run("antivirus result with ErrorMsg: SetArbitraryMetadata not called", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + ErrorMsg: "scanner unavailable", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("non-antivirus step: no side effects", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepDelay, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("result wrong type: no panic", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + // Should not panic; wrong result type is logged. + require.NotPanics(t, func() { + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: "not-a-virusscan-result", + }) + }) + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("empty UploadID (on-demand scan): return early, no session lookup", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: "", + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + Description: "clean", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("different storageId: ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: "any", + FinishedStep: events.PPStepAntivirus, + ResourceID: &provider.ResourceId{StorageId: "other-mount"}, + Result: events.VirusscanResult{ + Description: "clean", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) +} + +// Ensure userpb is referenced to satisfy the import check. +var _ = userpb.UserType_USER_TYPE_PRIMARY diff --git a/pkg/upload/coordinator_initiate_test.go b/pkg/upload/coordinator_initiate_test.go new file mode 100644 index 00000000000..5ece11dd832 --- /dev/null +++ b/pkg/upload/coordinator_initiate_test.go @@ -0,0 +1,339 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + tusd "github.com/tus/tusd/v2/pkg/handler" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// authedCtx returns a context carrying a user, as required by InitiateUpload. +func authedCtx() context.Context { + return ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) +} + +// touchFileResult is a convenience helper for a successful TouchFile result. +func touchFileResult(nodeID, spaceID string) *storage.TouchFileResult { + return &storage.TouchFileResult{ + ResourceID: &provider.ResourceId{OpaqueId: nodeID}, + SpaceID: spaceID, + SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + +// ref builds a simple path-only reference. +func ref(path string) *provider.Reference { + return &provider.Reference{Path: path} +} + +// existingNodeInfo builds a ResourceInfo as GetMD would return for an existing node. +func existingNodeInfo(nodeID, spaceID string, size uint64) *provider.ResourceInfo { + return &provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: nodeID, SpaceId: spaceID}, + Name: filepath.Base("/dir/file.txt"), + Path: "/dir/file.txt", + Size: size, + Owner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + +// TestInitiateUpload_NewFile covers new-file scenarios. +func TestInitiateUpload_NewFile(t *testing.T) { + t.Run("happy path creates session files and returns IDs", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + ids, err := coord.InitiateUpload(ctx, r, 10, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + assert.Equal(t, ids["simple"], ids["tus"]) + + sessionID := ids["simple"] + binPath := filepath.Join(store.root, "uploads", sessionID) + infoPath := filepath.Join(store.root, "uploads", sessionID+".info") + assert.FileExists(t, binPath) + assert.FileExists(t, infoPath) + }) + + t.Run("quota exceeded returns InsufficientStorage, no TouchFile", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(90), uint64(5), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + _, isInsufficient := err.(errtypes.InsufficientStorage) + assert.True(t, isInsufficient) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("GetQuota failure skips quota check and proceeds", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(0), uint64(0), uint64(0), errors.New("quota unavailable")) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.NoError(t, err) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) +} + +// TestInitiateUpload_Overwrite covers overwrite (existing node) scenarios. +func TestInitiateUpload_Overwrite(t *testing.T) { + t.Run("happy path overwrite creates session without touching node", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 20) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(50), uint64(50), nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) + + ids, err := coord.InitiateUpload(ctx, r, 30, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + fs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("quota exceeded for overwrite", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 5) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + // remaining=90, net_required=100-5=95 > 90 + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(200), uint64(110), uint64(90), nil) + + _, err := coord.InitiateUpload(ctx, r, 100, nil) + require.Error(t, err) + _, isInsufficient := err.(errtypes.InsufficientStorage) + assert.True(t, isInsufficient) + }) + + t.Run("upload smaller than existing: net_required=0, proceeds", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 50) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + // remaining=0 but net_required=0, should still succeed + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(100), uint64(0), nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, 30, nil) + require.NoError(t, err) + }) + + t.Run("size-deferred (uploadLength=-1): GetQuota not called", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 20) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, -1, map[string]string{"sizedeferred": "true"}) + require.NoError(t, err) + fs.AssertNotCalled(t, "GetQuota", mock.Anything, mock.Anything) + }) +} + +// TestInitiateUpload_ZeroLength covers the zero-length inline commit path. +func TestInitiateUpload_ZeroLength(t *testing.T) { + t.Run("zero-length: CommitUpload called inline, files cleaned", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResult("node1", "space1"), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + + ids, err := coord.InitiateUpload(ctx, r, 0, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + + sessionID := ids["simple"] + binPath := filepath.Join(store.root, "uploads", sessionID) + infoPath := filepath.Join(store.root, "uploads", sessionID+".info") + assert.NoFileExists(t, binPath) + assert.NoFileExists(t, infoPath) + }) + + t.Run("zero-length CommitUpload failure triggers rollback", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResult("node1", "space1"), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("commit failed")) + // rollback: MarkProcessing(false), Delete (ref is session.Reference(), ResourceId-based) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + mockFs.On("Delete", mock.Anything, mock.Anything).Return((*storage.DeleteResult)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, 0, nil) + require.Error(t, err) + }) +} + +// TestInitiateUpload_ErrorPaths covers failure cases during session setup. +func TestInitiateUpload_ErrorPaths(t *testing.T) { + t.Run("GetMD unexpected error returns error without TouchFile", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errors.New("unexpected")) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("TouchBin failure when uploads dir is missing: no node to delete", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + // Remove the uploads directory so TouchBin fails. + require.NoError(t, os.RemoveAll(filepath.Join(store.root, "uploads"))) + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + mockFs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockFs.AssertNotCalled(t, "Delete", mock.Anything, mock.Anything) + }) +} + +// TestInitiateUpload_Metadata covers metadata/checksum validation. +func TestInitiateUpload_Metadata(t *testing.T) { + t.Run("unsupported checksum algorithm returns BadRequest", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "crc32 abc123", + }) + require.Error(t, err) + _, isBad := err.(errtypes.BadRequest) + assert.True(t, isBad) + }) + + t.Run("malformed checksum (no space) returns BadRequest", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "nospace", + }) + require.Error(t, err) + _, isBad := err.(errtypes.BadRequest) + assert.True(t, isBad) + }) + + t.Run("valid sha1 checksum accepted", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "sha1 aabbccdd", + }) + require.NoError(t, err) + }) +} + +// noSuchPath returns true when the given path does not exist on disk. +func noSuchPath(path string) bool { + _, err := os.Stat(path) + return errors.Is(err, fs.ErrNotExist) +} + +// Ensure tusd is imported so it stays in go.mod. +var _ = tusd.ErrNotFound diff --git a/pkg/upload/coordinator_postprocessing.go b/pkg/upload/coordinator_postprocessing.go new file mode 100644 index 00000000000..c999a9448e5 --- /dev/null +++ b/pkg/upload/coordinator_postprocessing.go @@ -0,0 +1,321 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "os" + "path/filepath" + "time" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/owncloud/reva/v2/pkg/autoprop" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/utils" +) + +// Start subscribes to the event stream and launches numConsumers goroutines +// that process postprocessing events. +func (c *coordinator) Start(stream events.Consumer) error { + ch, err := events.Consume( + stream, + c.conGroup, + events.PostprocessingFinished{}, + events.PostprocessingStepFinished{}, + events.RestartPostprocessing{}, + events.CleanUpload{}, + ) + if err != nil { + return err + } + for i := 0; i < c.numConc; i++ { + go c.postprocessingLoop(ch) + } + return nil +} + +func (c *coordinator) postprocessingLoop(ch <-chan events.Event) { + for event := range ch { + c.processEvent(context.Background(), event) + } +} + +func (c *coordinator) processEvent(evCtx context.Context, event events.Event) { + ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) + ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) + defer span.End() + + switch ev := event.Event.(type) { + case events.PostprocessingFinished: + c.handlePostprocessingFinished(ctx, ev) + case events.PostprocessingStepFinished: + c.handlePostprocessingStepFinished(ctx, ev) + case events.RestartPostprocessing: + c.handleRestartPostprocessing(ctx, ev) + case events.CleanUpload: + c.handleCleanUpload(ctx, ev) + default: + c.log.Error().Interface("event", ev).Msg("coordinator: unknown event") + } +} + +func (c *coordinator) handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { + log := c.log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + // Session file gone (e.g. coordinator restarted mid-postprocessing). + // Clear the processing flag directly using the node ID from the event so + // the node does not stay stuck returning 429 Too Early forever. + if ev.ResourceID != nil && ev.ResourceID.GetOpaqueId() != "" { + ref := provider.Reference{ResourceId: ev.ResourceID} + if mpErr := c.fs.MarkProcessing(ctx, &ref, false, ev.UploadID); mpErr != nil { + log.Error().Err(mpErr).Msg("could not unmark processing after lost session") + } + } + return + } + + ctx = session.Context(ctx) + + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + ref := session.Reference() + if _, mdErr := c.fs.GetMD(ctx, &ref, []string{}, []string{}); mdErr != nil { + if _, notFound := mdErr.(errtypes.IsNotFound); notFound { + log.Debug().Err(mdErr).Msg("node deleted during postprocessing; cleaning up") + session.Cleanup(true, true) + if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during cleanup of deleted node") + } + return + } + } + + var ( + failed bool + revertNodeMetadata bool + keepUpload bool + retryCommit bool + ) + + switch ev.Outcome { + default: + log.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") + fallthrough + case events.PPOutcomeAbort: + failed = true + revertNodeMetadata = !session.NodeExists() + keepUpload = true + metrics.UploadSessionsAborted.Inc() + case events.PPOutcomeContinue: + f, fopenErr := os.Open(session.BinPath()) + if fopenErr != nil { + log.Error().Err(fopenErr).Msg("could not open staged binary for CommitUpload") + failed = true + keepUpload = true + retryCommit = true + } else { + defer f.Close() + commitRef := session.Reference() + _, commitErr := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ + Body: f, + Length: session.Size(), + Metadata: session.Metadata(), + Checksums: session.Checksums(), + }) + if commitErr != nil { + log.Error().Err(commitErr).Msg("could not commit upload") + failed = true + keepUpload = true + retryCommit = true + } else { + metrics.UploadSessionsFinalized.Inc() + } + } + case events.PPOutcomeDelete: + failed = true + revertNodeMetadata = !session.NodeExists() + metrics.UploadSessionsDeleted.Inc() + } + + now := time.Now() + + session.Cleanup(!keepUpload, !keepUpload) + + nodeRef := session.Reference() + if !retryCommit { + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing after postprocessing finished") + } + if revertNodeMetadata { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node on abort") + } + } + } + } + + isVersion := session.NodeExists() + + if err := events.Publish( + ctx, + c.pub, + events.UploadReady{ + UploadID: ev.UploadID, + Failed: failed, + ExecutingUser: ev.ExecutingUser, + Filename: ev.Filename, + FileRef: &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + }, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Timestamp: utils.TimeToTS(now), + SpaceOwner: session.SpaceOwner(), + IsVersion: isVersion, + ImpersonatingUser: ev.ImpersonatingUser, + }, + ); err != nil { + log.Error().Err(err).Msg("Failed to publish UploadReady event") + } +} + +func (c *coordinator) handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { + log := c.log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + ctx = session.Context(ctx) + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + s, err := session.URL(ctx) + if err != nil { + log.Error().Err(err).Msg("could not create url") + return + } + + metrics.UploadSessionsRestarted.Inc() + + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, + ResourceID: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + }); err != nil { + log.Error().Err(err).Msg("Failed to publish BytesReceived event") + } +} + +func (c *coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUpload) { + log := c.log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + ctx = session.Context(ctx) + session.Cleanup(!ev.KeepUpload, !ev.KeepUpload) + nodeRef := session.Reference() + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during CleanUpload") + } + if !session.NodeExists() { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node during CleanUpload") + } + } + } +} + +func (c *coordinator) handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { + log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + if ev.FinishedStep != events.PPStepAntivirus { + return + } + + res, ok := ev.Result.(events.VirusscanResult) + if !ok { + log.Error().Msgf("coordinator: unexpected antivirus result type %T", ev.Result) + return + } + if res.ErrorMsg != "" { + return + } + log = c.log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() + + if ev.UploadID == "" { + // on-demand scanning not supported + return + } + + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + + session.SetScanData(res.Description, res.Scandate) + if err := session.Persist(ctx); err != nil { + log.Error().Err(err).Msg("Failed to persist scan results") + } + + ctx = session.Context(ctx) + ref := session.Reference() + if err := c.fs.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + "scanstatus": res.Description, + "scandate": res.Scandate.Format(time.RFC3339Nano), + }, + }); err != nil { + log.Error().Err(err).Msg("Failed to write scan results to node") + } + + metrics.UploadSessionsScanned.Inc() +} diff --git a/pkg/upload/coordinator_test.go b/pkg/upload/coordinator_test.go new file mode 100644 index 00000000000..37fb5cc56d9 --- /dev/null +++ b/pkg/upload/coordinator_test.go @@ -0,0 +1,262 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/owncloud/reva/v2/pkg/storage" +) + +// TestNewCoordinator covers the constructor's validation logic. +func TestNewCoordinator(t *testing.T) { + root := t.TempDir() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + + t.Run("async without publisher returns error", func(t *testing.T) { + _, err := NewCoordinator(fs, store, nil, true, "m", "g", 1, &log, "") + require.Error(t, err) + }) + + t.Run("sync without publisher succeeds", func(t *testing.T) { + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 1, &log, "") + require.NoError(t, err) + require.NotNil(t, coord) + }) + + t.Run("async with publisher succeeds", func(t *testing.T) { + pub := &mockPublisher{} + coord, err := NewCoordinator(fs, store, pub, true, "m", "g", 1, &log, "") + require.NoError(t, err) + require.NotNil(t, coord) + }) + + t.Run("numConsumers zero defaults to 1", func(t *testing.T) { + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 0, &log, "") + require.NoError(t, err) + require.NotNil(t, coord) + c := coord.(*coordinator) + assert.Equal(t, 1, c.numConc) + }) +} + +// TestRollback verifies rollback behaviour: unmarks processing, removes session +// files, and (for new files) deletes the placeholder node. +func TestRollback(t *testing.T) { + t.Run("new file: unmarks processing, cleans files, deletes node", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).rollback(context.Background(), session) + + fs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + infoPath := fileSessionPath(store.root, session.ID()) + assert.NoFileExists(t, infoPath) + }) + + t.Run("overwrite: unmarks processing, cleans files, no delete", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", true) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).rollback(context.Background(), session) + + fs.AssertExpectations(t) + // Delete must NOT have been called — if it were, mock would record it as unexpected. + }) +} + +// TestFinishSync covers finishSync: happy path, missing .bin, and CommitUpload failure. +func TestFinishSync(t *testing.T) { + t.Run("happy path: commits, unmarks, removes bin and info", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + // Write content to the bin file. + require.NoError(t, os.WriteFile(session.BinPath(), []byte("hello"), 0600)) + // Reload session so offset is correct. + loaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + + ref := loaded.Reference() + fs.On("CommitUpload", mock.Anything, &ref, mock.AnythingOfType("storage.UploadSource")).Return((*provider.ResourceInfo)(nil), nil) + fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) + + err = coord.(*coordinator).finishSync(context.Background(), loaded) + require.NoError(t, err) + + fs.AssertExpectations(t) + assert.NoFileExists(t, loaded.BinPath()) + infoPath := fileSessionPath(store.root, loaded.ID()) + assert.NoFileExists(t, infoPath) + }) + + t.Run("missing bin triggers rollback and returns error", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + // Remove the bin file to simulate it being missing. + require.NoError(t, os.Remove(session.BinPath())) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + err := coord.(*coordinator).finishSync(context.Background(), session) + require.Error(t, err) + fs.AssertExpectations(t) + }) + + t.Run("CommitUpload failure triggers rollback and returns error", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + loaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + + ref := loaded.Reference() + fs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("commit error")) + // rollback calls MarkProcessing(false) and Delete (new file) + fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + err = coord.(*coordinator).finishSync(context.Background(), loaded) + require.Error(t, err) + fs.AssertExpectations(t) + }) +} + +// TestListUploadSessions covers filtering logic. +func TestListUploadSessions(t *testing.T) { + root := t.TempDir() + coord, _, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := context.Background() + + // Create three sessions with different characteristics. + s1 := newPopulatedSession(t, store, "/", "a.txt", "n1", "sp1", false) + s2 := newPopulatedSession(t, store, "/", "b.txt", "n2", "sp2", false) + s3 := newPopulatedSession(t, store, "/", "c.txt", "n3", "sp3", false) + + // Mark s1 as fully received (processing) by aligning offset with size. + // FileSession.IsProcessing() returns true when size==offset && scanResult=="". + // Since size defaults to 0, all sessions with empty bins are "processing" by that + // logic. Set s2 size to 10 so it is NOT processing (offset=0 != size=10). + { + s := s2 + s.SetSize(10) + require.NoError(t, s.Persist(ctx)) + // Touch the bin to keep the store happy (offset will be 0, != size 10). + } + + // Mark s3 as expired by setting expires in the past. + { + s := s3 + past := time.Now().Add(-time.Hour) + s.SetMetadata("expires", fmt.Sprintf("%d", past.Unix())) + require.NoError(t, s.Persist(ctx)) + } + + t.Run("by ID found", func(t *testing.T) { + id := s1.ID() + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{ID: &id}) + require.NoError(t, err) + require.Len(t, sessions, 1) + assert.Equal(t, s1.ID(), sessions[0].ID()) + }) + + t.Run("by ID not found returns error", func(t *testing.T) { + id := "nonexistent-id" + _, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{ID: &id}) + require.Error(t, err) + }) + + t.Run("filter Processing=true", func(t *testing.T) { + tr := true + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Processing: &tr}) + require.NoError(t, err) + // s1 and s3 have size==0==offset; s2 has size=10 != offset=0 + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s1.ID()) + assert.NotContains(t, ids, s2.ID()) + }) + + t.Run("filter Processing=false", func(t *testing.T) { + fl := false + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Processing: &fl}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s2.ID()) + assert.NotContains(t, ids, s1.ID()) + }) + + t.Run("filter Expired=true", func(t *testing.T) { + tr := true + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Expired: &tr}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s3.ID()) + // s1 and s2 have zero Expires() which is before now, so they may or may not be + // included depending on the zero-time semantics — document that s3 is included. + _ = s1 + }) + + t.Run("no filter returns all", func(t *testing.T) { + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s1.ID()) + assert.Contains(t, ids, s2.ID()) + assert.Contains(t, ids, s3.ID()) + }) +} diff --git a/pkg/upload/coordinator_upload_test.go b/pkg/upload/coordinator_upload_test.go new file mode 100644 index 00000000000..2183e72d238 --- /dev/null +++ b/pkg/upload/coordinator_upload_test.go @@ -0,0 +1,398 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "bytes" + "context" + "crypto/sha1" //nolint:gosec + "encoding/hex" + "io" + "strings" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// initiateAndGetID calls InitiateUpload on coord and returns the session ID. +// It sets up the expected FS mock calls for a new-file happy path. +// TouchFile and MarkProcessing(true) are NOT registered here — they happen in +// FinishUpload/Upload, not InitiateUpload. Callers must register them as needed. +func initiateAndGetID(t *testing.T, coord Coordinator, mockFs *mockFS, content string) string { + t.Helper() + ctx := ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) + r := &provider.Reference{Path: "/dir/file.txt"} + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + + ids, err := coord.InitiateUpload(ctx, r, int64(len(content)), nil) + require.NoError(t, err) + return ids["simple"] +} + +// touchFileResult returns the mock TouchFile result used by upload tests. +func touchFileResultForUpload() *storage.TouchFileResult { + return &storage.TouchFileResult{ + ResourceID: &provider.ResourceId{OpaqueId: "node1", SpaceId: "space1"}, + SpaceID: "space1", + SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + +// newUploadStore creates a new FileStore at the same root as used by the coordinator. +func newUploadStore(t *testing.T, root string) *FileStore { + t.Helper() + log := zerolog.Nop() + return NewFileStore(root, TokenOptions{ + DownloadEndpoint: "http://dl", + DataGatewayEndpoint: "http://gw", + TransferSharedSecret: "secret", + TransferExpires: 3600, + }, &log) +} + +// TestVerifyAndStoreChecksums tests the verifyAndStoreChecksums package-level function. +func TestVerifyAndStoreChecksums(t *testing.T) { + t.Run("no checksum in metadata: sets checksums on session", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello world" + n, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + require.Equal(t, int64(len(content)), n) + + err = verifyAndStoreChecksums(context.Background(), session) + require.NoError(t, err) + + cs := session.Checksums() + assert.NotEmpty(t, cs.SHA1) + assert.NotEmpty(t, cs.MD5) + assert.NotEmpty(t, cs.Adler32) + }) + + t.Run("correct sha1 checksum passes", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello" + _, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + + h := sha1.New() //nolint:gosec + h.Write([]byte(content)) + expected := hex.EncodeToString(h.Sum(nil)) + session.SetMetadata("checksum", "sha1 "+expected) + require.NoError(t, session.Persist(context.Background())) + + require.NoError(t, verifyAndStoreChecksums(context.Background(), session)) + }) + + t.Run("wrong sha1 checksum returns ChecksumMismatch and cleans bin", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello" + _, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + + session.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") + require.NoError(t, session.Persist(context.Background())) + + err = verifyAndStoreChecksums(context.Background(), session) + require.Error(t, err) + _, isMismatch := err.(errtypes.ChecksumMismatch) + assert.True(t, isMismatch) + assert.NoFileExists(t, session.BinPath()) + }) +} + +// TestUpload_Sync tests Upload in sync mode. +func TestUpload_Sync(t *testing.T) { + t.Run("happy path: CommitUpload called, uff invoked, ResourceInfo returned", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "hello" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + + ctx := context.Background() + var uffSpaceOwner, uffExecutant *userpb.UserId + var uffRef *provider.Reference + uff := func(so, ex *userpb.UserId, r *provider.Reference) { + uffSpaceOwner = so + uffExecutant = ex + uffRef = r + } + + ri, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, uff) + require.NoError(t, err) + require.NotNil(t, ri) + assert.Equal(t, "node1", ri.GetId().GetOpaqueId()) + assert.Equal(t, "space1", ri.GetId().GetSpaceId()) + assert.NotNil(t, uffSpaceOwner) + assert.NotNil(t, uffExecutant) + assert.NotNil(t, uffRef) + }) + + t.Run("size mismatch returns PartialContent", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "hello" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + _, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString("hi")), + Length: int64(len(content)), + }, nil) + require.Error(t, err) + _, isPartial := err.(errtypes.PartialContent) + assert.True(t, isPartial) + }) + + t.Run("session not found returns error", func(t *testing.T) { + root := t.TempDir() + coord, _, _ := newTestCoordinatorWithStore(t, root, false, nil) + + ctx := context.Background() + _, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/nonexistent-id"}, + Body: io.NopCloser(bytes.NewBufferString("data")), + Length: 4, + }, nil) + require.Error(t, err) + }) + + t.Run("verifyAndStoreChecksums failure triggers rollback", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "test" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + // Inject a bad checksum into the persisted session. + secondStore := newUploadStore(t, root) + sess, err := secondStore.Get(context.Background(), sessionID) + require.NoError(t, err) + sess.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") + require.NoError(t, sess.Persist(context.Background())) + + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) + mockFs.On("Delete", mock.Anything, mock.Anything).Return((*storage.DeleteResult)(nil), nil) + + ctx := context.Background() + _, err = coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, nil) + require.Error(t, err) + }) +} + +// TestUpload_Async tests Upload in async mode. +func TestUpload_Async(t *testing.T) { + t.Run("happy path: BytesReceived published, uff invoked", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + content := "asyncdata" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + ctx := context.Background() + var uffCalled bool + uff := func(_, _ *userpb.UserId, _ *provider.Reference) { uffCalled = true } + + ri, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, uff) + require.NoError(t, err) + require.NotNil(t, ri) + assert.True(t, uffCalled) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, sessionID, ev.UploadID) + assert.Equal(t, "file.txt", ev.Filename) + assert.Equal(t, uint64(len(content)), ev.Filesize) + }) +} + +// TestCoordinatedUpload_FinishUpload tests the TUS FinishUpload path. +func TestCoordinatedUpload_FinishUpload(t *testing.T) { + t.Run("sync: CommitUpload called, bin removed", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "finishme" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, sessionID) + require.NoError(t, err) + _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) + require.NoError(t, err) + + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + mockFs.AssertExpectations(t) + }) + + t.Run("async: BytesReceived published for non-zero size", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + content := "asyncfinish" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, sessionID) + require.NoError(t, err) + _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) + require.NoError(t, err) + + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, sessionID, ev.UploadID) + }) + + t.Run("async: size=0 no BytesReceived published", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + // Create a session with NodeExists=true and size=0 (overwrite). + session := newPopulatedSession(t, store, "/dir", "zero.txt", "n2", "sp2", true) + session.SetSize(0) + require.NoError(t, session.Persist(context.Background())) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, session.ID()) + require.NoError(t, err) + + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, session.ID()).Return(nil) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + assert.Empty(t, pub.events) + }) +} + +// TestCoordinatedUpload_Terminate tests the TUS Terminate path. +func TestCoordinatedUpload_Terminate(t *testing.T) { + t.Run("new file: Cleanup, MarkProcessing(false), Delete called", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "term.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.Terminate(context.Background())) + + mockFs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("overwrite: Cleanup, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "term.txt", "n1", "sp1", true) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.Terminate(context.Background())) + + mockFs.AssertExpectations(t) + }) +} + +// TestCoordinatedUpload_DeclareLength tests deferred-length declaration. +func TestCoordinatedUpload_DeclareLength(t *testing.T) { + root := t.TempDir() + coord, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "dl.txt", "n1", "sp1", false) + session.SetSizeIsDeferred(true) + require.NoError(t, session.Persist(context.Background())) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.DeclareLength(context.Background(), 42)) + + reloaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + assert.Equal(t, int64(42), reloaded.Size()) + info, err := reloaded.GetInfo(context.Background()) + require.NoError(t, err) + assert.False(t, info.SizeIsDeferred) +} diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go new file mode 100644 index 00000000000..d96ef138a13 --- /dev/null +++ b/pkg/upload/filestore.go @@ -0,0 +1,208 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "encoding/json" + iofs "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/google/uuid" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +// TokenOptions carries the JWT-signing configuration needed to produce transfer +// URLs for the postprocessing service. +type TokenOptions struct { + DownloadEndpoint string + DataGatewayEndpoint string + TransferSharedSecret string + TransferExpires int64 +} + +// SessionStore abstracts upload-session persistence for the Coordinator. +type SessionStore interface { + New(ctx context.Context) Session + Get(ctx context.Context, id string) (Session, error) + List(ctx context.Context) ([]Session, error) +} + +// FileStore is a filesystem-backed SessionStore. Sessions are stored as a pair +// of files under /uploads/: +// +// - .info — JSON-encoded tusd.FileInfo +// - — staged binary bytes +// +// This is the same on-disk format used by OcisStore so existing sessions +// survive a rolling deploy that switches to FileStore. +type FileStore struct { + root string + opts TokenOptions + log *zerolog.Logger +} + +// FileStoreFromDriverConf builds a FileStore from a reva driver config map. +// Returns nil if the config carries no root path (driver does not support +// coordinated uploads). Each service that mounts the same driver calls this +// independently. +func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if driverConf == nil { + return nil + } + + type driverRootConf struct { + Root string `mapstructure:"root"` + UploadDirectory string `mapstructure:"upload_directory"` + } + var rc driverRootConf + _ = mapstructure.Decode(driverConf, &rc) + + root := rc.UploadDirectory + if root == "" { + root = rc.Root + } + if root == "" { + return nil + } + + return newFileStoreWithTokens(root, driverConf, log) +} + +// NewFileStoreFromConfig builds a FileStore using uploadDir when set, falling +// back to root/upload_directory from the active driver config. This allows +// drivers that have no local root (e.g. KW) to still get a coordinator by +// setting upload_directory at the service level rather than inside the driver. +// Returns nil only when neither source resolves to a non-empty path. +func NewFileStoreFromConfig(uploadDir string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if uploadDir != "" { + return NewFileStore(uploadDir, TokenOptions{}, log) + } + return FileStoreFromDriverConf(driverConf, log) +} + +func newFileStoreWithTokens(root string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + type tokenConf struct { + DownloadEndpoint string `mapstructure:"download_endpoint"` + DataGatewayEndpoint string `mapstructure:"datagateway_endpoint"` + TransferSharedSecret string `mapstructure:"transfer_shared_secret"` + TransferExpires int64 `mapstructure:"transfer_expires"` + } + var tc tokenConf + if tokens, ok := driverConf["tokens"]; ok { + _ = mapstructure.Decode(tokens, &tc) + } + return NewFileStore(root, TokenOptions{ + DownloadEndpoint: tc.DownloadEndpoint, + DataGatewayEndpoint: tc.DataGatewayEndpoint, + TransferSharedSecret: tc.TransferSharedSecret, + TransferExpires: tc.TransferExpires, + }, log) +} + +// NewFileStore creates a FileStore rooted at root. +// root must be on a shared filesystem when multiple pods handle the same space. +func NewFileStore(root string, opts TokenOptions, log *zerolog.Logger) *FileStore { + return &FileStore{root: root, opts: opts, log: log} +} + +// Root returns the base directory of this FileStore. +func (fs *FileStore) Root() string { + return fs.root +} + +// Setup creates the uploads directory eagerly so permission problems are caught +// at startup rather than on the first upload. +func (fs *FileStore) Setup() error { + return os.MkdirAll(filepath.Join(fs.root, "uploads"), 0700) +} + +// New allocates a fresh session with a new UUID. +func (fs *FileStore) New(_ context.Context) Session { + return &FileSession{ + store: fs, + info: tusd.FileInfo{ + ID: uuid.New().String(), + Storage: map[string]string{ + "Type": "FileStore", + }, + MetaData: tusd.MetaData{}, + }, + } +} + +// Get loads the session with the given id from disk. +func (fs *FileStore) Get(ctx context.Context, id string) (Session, error) { + infoPath := fileSessionPath(fs.root, id) + + data, err := os.ReadFile(infoPath) + if err != nil { + if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE { + return nil, tusd.ErrNotFound + } + if errors.Is(err, iofs.ErrNotExist) { + return nil, tusd.ErrNotFound + } + return nil, err + } + + var info tusd.FileInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + session := &FileSession{store: fs, info: info} + + stat, err := os.Stat(session.binPath()) + if err != nil { + if os.IsNotExist(err) { + return nil, tusd.ErrNotFound + } + return nil, err + } + session.info.Offset = stat.Size() + + return session, nil +} + +// List returns all sessions found under /uploads/*.info. +func (fs *FileStore) List(ctx context.Context) ([]Session, error) { + infoFiles, err := filepath.Glob(filepath.Join(fs.root, "uploads", "*.info")) + if err != nil { + return nil, err + } + + sessions := make([]Session, 0, len(infoFiles)) + for _, path := range infoFiles { + id := strings.TrimSuffix(filepath.Base(path), ".info") + session, err := fs.Get(ctx, id) + if err != nil { + fs.log.Error().Str("path", path).Err(err).Msg("filestore: could not load session") + continue + } + sessions = append(sessions, session) + } + return sessions, nil +} diff --git a/pkg/upload/filestore_test.go b/pkg/upload/filestore_test.go new file mode 100644 index 00000000000..8ad5c5b9141 --- /dev/null +++ b/pkg/upload/filestore_test.go @@ -0,0 +1,312 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +func nopLog() *zerolog.Logger { + l := zerolog.Nop() + return &l +} + +// FileStore.Setup + +func TestFileStoreSetup_CreatesUploadsDir(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + + err := fs.Setup() + + require.NoError(t, err) + info, err := os.Stat(filepath.Join(root, "uploads")) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func TestFileStoreSetup_Idempotent(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + + require.NoError(t, fs.Setup()) + assert.NoError(t, fs.Setup()) +} + +// FileStore.New + +func TestFileStoreNew_NonEmptyID(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + assert.NotEmpty(t, s.ID()) +} + +func TestFileStoreNew_UniqueIDs(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s1 := fs.New(context.Background()) + s2 := fs.New(context.Background()) + + assert.NotEqual(t, s1.ID(), s2.ID()) +} + +func TestFileStoreNew_StorageTypeIsFileStore(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + info, err := s.GetInfo(context.Background()) + require.NoError(t, err) + assert.Equal(t, "FileStore", info.Storage["Type"]) +} + +func TestFileStoreNew_MetaDataIsNotNil(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + info, err := s.GetInfo(context.Background()) + require.NoError(t, err) + assert.NotNil(t, info.MetaData) +} + +// FileStore.Get + +func TestFileStoreGet_HappyPath(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + got, err := fs.Get(ctx, s.ID()) + + require.NoError(t, err) + assert.Equal(t, s.ID(), got.ID()) + info, err := got.GetInfo(ctx) + require.NoError(t, err) + assert.Equal(t, "FileStore", info.Storage["Type"]) +} + +func TestFileStoreGet_OffsetFromBinSize(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + payload := []byte("hello world") + require.NoError(t, os.WriteFile(s.binPath(), payload, 0600)) + + got, err := fs.Get(ctx, s.ID()) + + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), got.Offset()) +} + +func TestFileStoreGet_MissingInfoReturnsErrNotFound(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + _, err := fs.Get(context.Background(), "no-such-id") + + assert.ErrorIs(t, err, tusd.ErrNotFound) +} + +func TestFileStoreGet_CorruptInfoReturnsError(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + // Overwrite the .info with garbage JSON. + require.NoError(t, os.WriteFile(s.infoPath(), []byte("{not valid json"), 0600)) + + _, err := fs.Get(ctx, s.ID()) + + require.Error(t, err) + assert.NotErrorIs(t, err, tusd.ErrNotFound) +} + +func TestFileStoreGet_MissingBinReturnsErrNotFound(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + // Write .info but do NOT create the .bin file. + require.NoError(t, s.Persist(ctx)) + + _, err := fs.Get(ctx, s.ID()) + + assert.ErrorIs(t, err, tusd.ErrNotFound) +} + +// FileStore.List + +func TestFileStoreList_EmptyDir(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + sessions, err := fs.List(context.Background()) + + require.NoError(t, err) + assert.Empty(t, sessions) +} + +func TestFileStoreList_ReturnsBothSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s1 := fs.New(ctx).(*FileSession) + require.NoError(t, s1.TouchBin()) + require.NoError(t, s1.Persist(ctx)) + + s2 := fs.New(ctx).(*FileSession) + require.NoError(t, s2.TouchBin()) + require.NoError(t, s2.Persist(ctx)) + + sessions, err := fs.List(ctx) + + require.NoError(t, err) + ids := make([]string, 0, len(sessions)) + for _, s := range sessions { + ids = append(ids, s.ID()) + } + assert.ElementsMatch(t, []string{s1.ID(), s2.ID()}, ids) +} + +func TestFileStoreList_SkipsSessionWithMissingBin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + good := fs.New(ctx).(*FileSession) + require.NoError(t, good.TouchBin()) + require.NoError(t, good.Persist(ctx)) + + bad := fs.New(ctx).(*FileSession) + require.NoError(t, bad.TouchBin()) + require.NoError(t, bad.Persist(ctx)) + require.NoError(t, os.Remove(bad.binPath())) + + sessions, err := fs.List(ctx) + + require.NoError(t, err) + require.Len(t, sessions, 1) + assert.Equal(t, good.ID(), sessions[0].ID()) +} + +// FileStoreFromDriverConf + +func TestFileStoreFromDriverConf_NilConfigReturnsNil(t *testing.T) { + assert.Nil(t, FileStoreFromDriverConf(nil, nopLog())) +} + +func TestFileStoreFromDriverConf_RootKey(t *testing.T) { + root := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{"root": root}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, root, fs.root) +} + +func TestFileStoreFromDriverConf_UploadDirectoryKey(t *testing.T) { + dir := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{"upload_directory": dir}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, dir, fs.root) +} + +func TestFileStoreFromDriverConf_UploadDirectoryWinsOverRoot(t *testing.T) { + root := t.TempDir() + uploadDir := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{ + "root": root, + "upload_directory": uploadDir, + }, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, uploadDir, fs.root) +} + +func TestFileStoreFromDriverConf_NeitherKeyReturnsNil(t *testing.T) { + fs := FileStoreFromDriverConf(map[string]interface{}{"some_other_key": "value"}, nopLog()) + + assert.Nil(t, fs) +} + +// NewFileStoreFromConfig + +func TestNewFileStoreFromConfig_UploadDirUsed(t *testing.T) { + uploadDir := t.TempDir() + fs := NewFileStoreFromConfig(uploadDir, map[string]interface{}{"root": "/ignored"}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, uploadDir, fs.root) +} + +func TestNewFileStoreFromConfig_FallsBackToDriverConf(t *testing.T) { + root := t.TempDir() + fs := NewFileStoreFromConfig("", map[string]interface{}{"root": root}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, root, fs.root) +} + +func TestNewFileStoreFromConfig_BothEmptyReturnsNil(t *testing.T) { + fs := NewFileStoreFromConfig("", nil, nopLog()) + + assert.Nil(t, fs) +} diff --git a/pkg/upload/mock_fs_test.go b/pkg/upload/mock_fs_test.go new file mode 100644 index 00000000000..533daf46a4d --- /dev/null +++ b/pkg/upload/mock_fs_test.go @@ -0,0 +1,277 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "io" + "net/url" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + goevents "go-micro.dev/v4/events" + + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// mockFS implements storage.FS for the coordinator tests. +// Only the methods actually exercised by the coordinator are wired to testify/mock; +// all others panic so accidental calls are caught immediately. +type mockFS struct { + mock.Mock +} + +func (m *mockFS) GetMD(ctx context.Context, ref *provider.Reference, mdKeys, fieldMask []string) (*provider.ResourceInfo, error) { + args := m.Called(ctx, ref, mdKeys, fieldMask) + ri, _ := args.Get(0).(*provider.ResourceInfo) + return ri, args.Error(1) +} + +func (m *mockFS) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, uint64, error) { + args := m.Called(ctx, ref) + return args.Get(0).(uint64), args.Get(1).(uint64), args.Get(2).(uint64), args.Error(3) +} + +func (m *mockFS) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) (*storage.TouchFileResult, error) { + args := m.Called(ctx, ref, markprocessing, mtime) + r, _ := args.Get(0).(*storage.TouchFileResult) + return r, args.Error(1) +} + +func (m *mockFS) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { + args := m.Called(ctx, ref, processing, sessionID) + return args.Error(0) +} + +func (m *mockFS) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { + args := m.Called(ctx, ref, source) + ri, _ := args.Get(0).(*provider.ResourceInfo) + return ri, args.Error(1) +} + +func (m *mockFS) Delete(ctx context.Context, ref *provider.Reference) (*storage.DeleteResult, error) { + args := m.Called(ctx, ref) + dr, _ := args.Get(0).(*storage.DeleteResult) + return dr, args.Error(1) +} + +func (m *mockFS) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { + args := m.Called(ctx, ref, md) + return args.Error(0) +} + +// unimplemented methods panic to catch accidental calls + +func (m *mockFS) Shutdown(_ context.Context) error { panic("not implemented: Shutdown") } + +func (m *mockFS) ListStorageSpaces(_ context.Context, _ []*provider.ListStorageSpacesRequest_Filter, _ bool) ([]*provider.StorageSpace, error) { + panic("not implemented: ListStorageSpaces") +} + +func (m *mockFS) ListFolder(_ context.Context, _ *provider.Reference, _, _ []string) ([]*provider.ResourceInfo, error) { + panic("not implemented: ListFolder") +} + +func (m *mockFS) Download(_ context.Context, _ *provider.Reference, _ func(*provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { + panic("not implemented: Download") +} + +func (m *mockFS) GetPathByID(_ context.Context, _ *provider.ResourceId) (string, error) { + panic("not implemented: GetPathByID") +} + +func (m *mockFS) CreateReference(_ context.Context, _ string, _ *url.URL) error { + panic("not implemented: CreateReference") +} + +func (m *mockFS) CreateDir(_ context.Context, _ *provider.Reference) (*storage.CreateDirResult, error) { + panic("not implemented: CreateDir") +} + +func (m *mockFS) Move(_ context.Context, _, _ *provider.Reference) (*storage.MoveResult, error) { + panic("not implemented: Move") +} + +func (m *mockFS) InitiateUpload(_ context.Context, _ *provider.Reference, _ int64, _ map[string]string) (map[string]string, error) { + panic("not implemented: InitiateUpload") +} + +func (m *mockFS) Upload(_ context.Context, _ storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { + panic("not implemented: Upload") +} + +func (m *mockFS) ListRevisions(_ context.Context, _ *provider.Reference) ([]*provider.FileVersion, error) { + panic("not implemented: ListRevisions") +} + +func (m *mockFS) DownloadRevision(_ context.Context, _ *provider.Reference, _ string, _ func(*provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { + panic("not implemented: DownloadRevision") +} + +func (m *mockFS) RestoreRevision(_ context.Context, _ *provider.Reference, _ string) (*storage.RestoreRevisionResult, error) { + panic("not implemented: RestoreRevision") +} + +func (m *mockFS) ListRecycle(_ context.Context, _ *provider.Reference, _, _ string) ([]*provider.RecycleItem, error) { + panic("not implemented: ListRecycle") +} + +func (m *mockFS) RestoreRecycleItem(_ context.Context, _ *provider.Reference, _, _ string, _ *provider.Reference) (*storage.RestoreRecycleItemResult, error) { + panic("not implemented: RestoreRecycleItem") +} + +func (m *mockFS) PurgeRecycleItem(_ context.Context, _ *provider.Reference, _, _ string) error { + panic("not implemented: PurgeRecycleItem") +} + +func (m *mockFS) EmptyRecycle(_ context.Context, _ *provider.Reference) error { + panic("not implemented: EmptyRecycle") +} + +func (m *mockFS) AddGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: AddGrant") +} + +func (m *mockFS) DenyGrant(_ context.Context, _ *provider.Reference, _ *provider.Grantee) error { + panic("not implemented: DenyGrant") +} + +func (m *mockFS) RemoveGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: RemoveGrant") +} + +func (m *mockFS) UpdateGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: UpdateGrant") +} + +func (m *mockFS) ListGrants(_ context.Context, _ *provider.Reference) ([]*provider.Grant, error) { + panic("not implemented: ListGrants") +} + +func (m *mockFS) UnsetArbitraryMetadata(_ context.Context, _ *provider.Reference, _ []string) error { + panic("not implemented: UnsetArbitraryMetadata") +} + +func (m *mockFS) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + args := m.Called(ctx, ref) + return args.Get(0).(*provider.Lock), args.Error(1) +} + +func (m *mockFS) SetLock(_ context.Context, _ *provider.Reference, _ *provider.Lock) (*storage.SetLockResult, error) { + panic("not implemented: SetLock") +} + +func (m *mockFS) RefreshLock(_ context.Context, _ *provider.Reference, _ *provider.Lock, _ string) error { + panic("not implemented: RefreshLock") +} + +func (m *mockFS) Unlock(_ context.Context, _ *provider.Reference, _ *provider.Lock) (*storage.UnlockResult, error) { + panic("not implemented: Unlock") +} + +func (m *mockFS) CreateStorageSpace(_ context.Context, _ *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { + panic("not implemented: CreateStorageSpace") +} + +func (m *mockFS) UpdateStorageSpace(_ context.Context, _ *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { + panic("not implemented: UpdateStorageSpace") +} + +func (m *mockFS) DeleteStorageSpace(_ context.Context, _ *provider.DeleteStorageSpaceRequest) (*storage.DeleteStorageSpaceResult, error) { + panic("not implemented: DeleteStorageSpace") +} + +func (m *mockFS) CreateHome(_ context.Context) error { panic("not implemented: CreateHome") } +func (m *mockFS) GetHome(_ context.Context) (string, error) { panic("not implemented: GetHome") } + +// mockPublisher captures published events for inspection in tests. +type mockPublisher struct { + events []interface{} +} + +func (p *mockPublisher) Publish(_ string, msg interface{}, _ ...goevents.PublishOption) error { + p.events = append(p.events, msg) + return nil +} + +// newTestStore creates a FileStore with JWT transfer options rooted at root. +func newTestStore(t *testing.T, root string) *FileStore { + t.Helper() + log := zerolog.Nop() + store := NewFileStore(root, TokenOptions{ + DownloadEndpoint: "http://dl", + DataGatewayEndpoint: "http://gw", + TransferSharedSecret: "secret", + TransferExpires: 3600, + }, &log) + require.NoError(t, store.Setup()) + return store +} + +// newTestCoordinator builds a coordinator backed by a real FileStore at root. +// Returns the coordinator and the mockFS so callers can set up expectations. +func newTestCoordinator(t *testing.T, root string, async bool, pub events.Publisher) (Coordinator, *mockFS) { + t.Helper() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log, "") + require.NoError(t, err) + return coord, fs +} + +// newTestCoordinatorWithStore is like newTestCoordinator but also returns the FileStore. +func newTestCoordinatorWithStore(t *testing.T, root string, async bool, pub events.Publisher) (Coordinator, *mockFS, *FileStore) { + t.Helper() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log, "") + require.NoError(t, err) + return coord, fs, store +} + +// newPopulatedSession creates a persisted session with .bin and .info files on disk. +func newPopulatedSession(t *testing.T, store *FileStore, dir, filename, nodeID, spaceID string, nodeExists bool) Session { + t.Helper() + ctx := context.Background() + s := store.New(ctx) + s.SetMetadata("filename", filename) + s.SetStorageValue("NodeName", filename) + s.SetMetadata("dir", dir) + s.SetStorageValue("Dir", dir) + s.SetStorageValue("NodeId", nodeID) + s.SetStorageValue("SpaceRoot", spaceID) + s.SetMetadata("providerID", "test-provider") + s.SetStorageValue("SpaceOwnerOrManager", "owner1") + if nodeExists { + s.SetStorageValue("NodeExists", "true") + } + s.SetExecutant(&userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + return s +} diff --git a/pkg/upload/session.go b/pkg/upload/session.go new file mode 100644 index 00000000000..4db4d665fec --- /dev/null +++ b/pkg/upload/session.go @@ -0,0 +1,473 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "crypto/md5" //nolint:gosec + "crypto/sha1" //nolint:gosec + "hash/adler32" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "strings" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/golang-jwt/jwt/v5" + "github.com/google/renameio/v2" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/owncloud/reva/v2/pkg/appctx" + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/utils" +) + +const defaultFilePerm = os.FileMode(0664) + +// FileSession is the Session implementation for disk-backed uploads. While an +// upload is in progress, incoming bytes are staged in a .bin file and upload +// metadata (size, owner, checksums, etc.) is persisted in a .info file. Both +// survive process restarts, allowing TUS resumption. +// +// In scope: read/write the staged .bin file, persist/load upload metadata in the .info file. +// Out of scope: TUS protocol, checksums, event publishing, postprocessing — those live in coordinatedUpload and coordinator. +type FileSession struct { + store *FileStore + info tusd.FileInfo +} + +// Session is the driver-agnostic view of an upload session the Coordinator +// needs. Implementations must be pure state (CRUD): protocol orchestration +// belongs to coordinatedUpload or the coordinator itself. +type Session interface { + storage.UploadSession + + // Data access — delegated to by coordinatedUpload for TUS reads/writes. + GetInfo(ctx context.Context) (tusd.FileInfo, error) + GetReader(ctx context.Context) (io.ReadCloser, error) + WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) + + // Internal coordinator plumbing. + Chunk() string + BinPath() string + ProviderID() string + SpaceID() string + NodeID() string + NodeParentID() string + NodeExists() bool + Dir() string + URL(ctx context.Context) (string, error) + SetScanData(result string, date time.Time) + Checksums() storage.UploadChecksums + SetChecksums(sha1, md5, adler32 []byte) + Metadata() map[string]string + Persist(ctx context.Context) error + Cleanup(cleanBin, cleanInfo bool) + Context(ctx context.Context) context.Context + + // Typed setters used by Coordinator.InitiateUpload to populate a new session + // without knowing internal storage key names. + SetStorageValue(key, value string) + SetMetadata(key, value string) + SetSize(size int64) + SetSizeIsDeferred(value bool) + SetExecutant(u *userpb.User) + TouchBin() error +} + +func (s *FileSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { + return s.info, nil +} + +func (s *FileSession) GetReader(_ context.Context) (io.ReadCloser, error) { + return os.Open(s.binPath()) +} + +func (s *FileSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + if err != nil && err != io.ErrUnexpectedEOF { + return n, err + } + s.info.Offset += n + return n, nil +} + + +// Purge removes all on-disk state for this session. +func (s *FileSession) Purge(ctx context.Context) { + s.Cleanup(true, true) +} + +// ScanData returns the AV scan result and scan date stored on the session. +func (s *FileSession) ScanData() (string, time.Time) { + date := s.info.MetaData["scanDate"] + if date == "" { + return "", time.Time{} + } + d, _ := time.Parse(time.RFC3339, date) + return s.info.MetaData["scanResult"], d +} + +// ID returns the upload session ID. +func (s *FileSession) ID() string { + return s.info.ID +} + +// Filename returns the filename stored in the session. +func (s *FileSession) Filename() string { + return s.info.Storage["NodeName"] +} + +// Size returns the declared upload size. +func (s *FileSession) Size() int64 { + return s.info.Size +} + +// Offset returns the current upload offset. +func (s *FileSession) Offset() int64 { + return s.info.Offset +} + +// Chunk returns the chunk basename stored in the session, or "" for non-chunked uploads. +func (s *FileSession) Chunk() string { + return s.info.Storage["Chunk"] +} + +// BinPath returns the path to the staged binary file. +func (s *FileSession) BinPath() string { + return s.binPath() +} + +// SpaceGid returns the numeric GID of the space owner, or "" if not set. +func (s *FileSession) SpaceGid() string { + return s.info.Storage["SpaceGid"] +} + +// ProviderID returns the storage provider ID stored in the session. +func (s *FileSession) ProviderID() string { + return s.info.MetaData["providerID"] +} + +// SpaceID returns the space (root) ID. +func (s *FileSession) SpaceID() string { + return s.info.Storage["SpaceRoot"] +} + +// NodeID returns the node ID for this upload. +func (s *FileSession) NodeID() string { + return s.info.Storage["NodeId"] +} + +// NodeParentID returns the parent node ID for this upload. +func (s *FileSession) NodeParentID() string { + return s.info.Storage["NodeParentId"] +} + +// NodeExists returns whether the target node existed when the upload was initiated. +func (s *FileSession) NodeExists() bool { + return s.info.Storage["NodeExists"] == "true" +} + +// Dir returns the directory portion of the upload path. +func (s *FileSession) Dir() string { + return s.info.Storage["Dir"] +} + +// IsProcessing returns true if all bytes are received but postprocessing has not finished. +func (s *FileSession) IsProcessing() bool { + return s.info.Size == s.info.Offset && s.info.MetaData["scanResult"] == "" +} + +// SpaceOwner returns the space owner user ID. +func (s *FileSession) SpaceOwner() *userpb.UserId { + return &userpb.UserId{ + OpaqueId: s.info.Storage["SpaceOwnerOrManager"], + Idp: s.info.Storage["SpaceOwnerIdp"], + Type: userpb.UserType(userpb.UserType_value[s.info.Storage["SpaceOwnerType"]]), + } +} + +// Executant returns the user ID of the user who initiated this upload. +func (s *FileSession) Executant() userpb.UserId { + return userpb.UserId{ + Type: utils.UserTypeMap(s.info.Storage["UserType"]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + } +} + +// Expires returns the upload expiry time. +func (s *FileSession) Expires() time.Time { + var t time.Time + if value, ok := s.info.MetaData["expires"]; ok { + t, _ = utils.MTimeToTime(value) + } + return t +} + +// Reference returns a CS3 reference for the resource being uploaded. +func (s *FileSession) Reference() provider.Reference { + return provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: s.info.MetaData["providerID"], + SpaceId: s.info.Storage["SpaceRoot"], + OpaqueId: s.info.Storage["NodeId"], + }, + } +} + +// Checksums returns the pre-computed checksums stored on the session. +func (s *FileSession) Checksums() storage.UploadChecksums { + decode := func(key string) []byte { + b, _ := hex.DecodeString(s.info.MetaData[key]) + return b + } + return storage.UploadChecksums{ + SHA1: decode("checksumSHA1"), + MD5: decode("checksumMD5"), + Adler32: decode("checksumAdler32"), + } +} + +// Metadata returns the upload metadata map passed to CommitUpload. +func (s *FileSession) Metadata() map[string]string { + return map[string]string{ + "providerID": s.info.MetaData["providerID"], + "mtime": s.info.MetaData["mtime"], + "nodeExists": s.info.Storage["NodeExists"], + "versionsPath": s.info.MetaData["versionsPath"], + "sessionID": s.info.ID, + } +} + +// SetScanData stores AV scan results on the session. +func (s *FileSession) SetScanData(result string, date time.Time) { + s.info.MetaData["scanResult"] = result + s.info.MetaData["scanDate"] = date.Format(time.RFC3339) +} + +// SetChecksums stores pre-computed checksums so CommitUpload can use them without +// re-reading the binary file. +func (s *FileSession) SetChecksums(sha1Sum, md5Sum, adler32Sum []byte) { + s.info.MetaData["checksumSHA1"] = hex.EncodeToString(sha1Sum) + s.info.MetaData["checksumMD5"] = hex.EncodeToString(md5Sum) + s.info.MetaData["checksumAdler32"] = hex.EncodeToString(adler32Sum) +} + +// SetMetadata sets a user-visible upload metadata field. +func (s *FileSession) SetMetadata(key, value string) { + s.info.MetaData[key] = value +} + +// SetStorageValue sets an internal storage field on the session. +func (s *FileSession) SetStorageValue(key, value string) { + s.info.Storage[key] = value +} + +// SetSize updates the declared upload size. +func (s *FileSession) SetSize(size int64) { + s.info.Size = size +} + +// SetSizeIsDeferred marks the upload size as not yet known. +func (s *FileSession) SetSizeIsDeferred(value bool) { + s.info.SizeIsDeferred = value +} + +// SetExecutant stores the identity of the user who initiated the upload. +func (s *FileSession) SetExecutant(u *userpb.User) { + s.info.Storage["Idp"] = u.GetId().GetIdp() + s.info.Storage["UserId"] = u.GetId().GetOpaqueId() + s.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type) + s.info.Storage["UserName"] = u.GetUsername() + s.info.Storage["UserDisplayName"] = u.GetDisplayName() + b, _ := json.Marshal(u.GetOpaque()) + s.info.Storage["UserOpaque"] = string(b) + g, _ := json.Marshal(u.GetGroups()) + s.info.Storage["UserGroups"] = string(g) +} + +// TouchBin creates the empty staging file. +func (s *FileSession) TouchBin() error { + f, err := os.OpenFile(s.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return err + } + return f.Close() +} + +// Persist writes the session metadata atomically to disk. +func (s *FileSession) Persist(ctx context.Context) error { + infoPath := s.infoPath() + if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil { + return err + } + d, err := json.Marshal(s.info) + if err != nil { + return err + } + return renameio.WriteFile(infoPath, d, 0600) +} + +// Cleanup removes the staged binary and/or info file. +// Node deletion and processing flag changes are the coordinator's responsibility. +func (s *FileSession) Cleanup(cleanBin, cleanInfo bool) { + log := s.store.log + if cleanBin { + if err := os.Remove(s.binPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Str("path", s.binPath()).Err(err).Msg("filestore: removing staged binary failed") + } + } + if cleanInfo { + if err := os.Remove(s.infoPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Str("path", s.infoPath()).Err(err).Msg("filestore: removing session info failed") + } + } +} + +// Context reconstructs a context carrying the user, logger, lock ID, and +// initiator ID that were recorded when the upload was initiated. +func (s *FileSession) Context(ctx context.Context) context.Context { + sub := s.store.log.With().Int("pid", os.Getpid()).Logger() + ctx = appctx.WithLogger(ctx, &sub) + ctx = ctxpkg.ContextSetLockID(ctx, s.info.MetaData["lockid"]) + ctx = ctxpkg.ContextSetUser(ctx, s.executantUser()) + return ctxpkg.ContextSetInitiator(ctx, s.info.MetaData["initiatorid"]) +} + +// URL returns a signed JWT URL that the postprocessing service can use to +// download the staged binary via the data gateway. +func (s *FileSession) URL(_ context.Context) (string, error) { + type transferClaims struct { + jwt.RegisteredClaims + Target string `json:"target"` + } + + target := joinURLParts(s.store.opts.DownloadEndpoint, "tus/", s.info.ID) + ttl := time.Duration(s.store.opts.TransferExpires) * time.Second + claims := transferClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), + Audience: jwt.ClaimStrings{"reva"}, + IssuedAt: jwt.NewNumericDate(time.Now()), + }, + Target: target, + } + t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) + tkn, err := t.SignedString([]byte(s.store.opts.TransferSharedSecret)) + if err != nil { + return "", errors.Wrapf(err, "filestore: error signing transfer token with claims %+v", claims) + } + return joinURLParts(s.store.opts.DataGatewayEndpoint, tkn), nil +} + +func (s *FileSession) ToFileInfo() tusd.FileInfo { + return s.info +} + +func (s *FileSession) InitiatorID() string { + return s.info.MetaData["initiatorid"] +} + +func (s *FileSession) binPath() string { + return filepath.Join(s.store.root, "uploads", s.info.ID) +} + +func (s *FileSession) infoPath() string { + return fileSessionPath(s.store.root, s.info.ID) +} + +func (s *FileSession) executantUser() *userpb.User { + var o *typespb.Opaque + _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) + var groups []string + _ = json.Unmarshal([]byte(s.info.Storage["UserGroups"]), &groups) + return &userpb.User{ + Id: &userpb.UserId{ + Type: utils.UserTypeMap(s.info.Storage["UserType"]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + }, + Username: s.info.Storage["UserName"], + DisplayName: s.info.Storage["UserDisplayName"], + Opaque: o, + Groups: groups, + } +} + +// fileSessionPath returns the path to the .info file for the given session ID. +func fileSessionPath(root, id string) string { + return filepath.Join(root, "uploads", id+".info") +} + +// calculateChecksums computes sha1, md5, and adler32 in a single pass over path. +func calculateChecksums(_ context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { + sha1h := sha1.New() //nolint:gosec + md5h := md5.New() //nolint:gosec + adler32h := adler32.New() + + f, err := os.Open(path) + if err != nil { + return nil, nil, nil, err + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + if _, err = io.Copy(adler32h, r2); err != nil { + return nil, nil, nil, err + } + return sha1h, md5h, adler32h, nil +} + +func checkHash(expected string, h hash.Hash) error { + got := hex.EncodeToString(h.Sum(nil)) + if expected != got { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %s", expected, got)) + } + return nil +} + +// joinURLParts concatenates URL path segments, inserting "/" between them if needed. +func joinURLParts(parts ...string) string { + var b strings.Builder + for i, p := range parts { + b.WriteString(p) + if i < len(parts)-1 && !strings.HasSuffix(p, "/") { + b.WriteByte('/') + } + } + return b.String() +} diff --git a/pkg/upload/session_test.go b/pkg/upload/session_test.go new file mode 100644 index 00000000000..8bcf1167a98 --- /dev/null +++ b/pkg/upload/session_test.go @@ -0,0 +1,456 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "crypto/sha1" //nolint:gosec + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/utils" +) + +func newTestSession(t *testing.T) (*FileSession, *FileStore) { + t.Helper() + log := zerolog.Nop() + fs := NewFileStore(t.TempDir(), TokenOptions{}, &log) + sess := fs.New(context.Background()).(*FileSession) + return sess, fs +} + +// SetMetadata / SetStorageValue / SetSize / SetSizeIsDeferred + +func TestSetMetadata(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "storage-1") + assert.Equal(t, "storage-1", sess.ProviderID()) +} + +func TestSetStorageValue(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetStorageValue("SpaceRoot", "space-abc") + assert.Equal(t, "space-abc", sess.SpaceID()) +} + +func TestSetSize(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSize(1234) + assert.Equal(t, int64(1234), sess.Size()) +} + +func TestSetSizeIsDeferred(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSizeIsDeferred(true) + assert.True(t, sess.info.SizeIsDeferred) + sess.SetSizeIsDeferred(false) + assert.False(t, sess.info.SizeIsDeferred) +} + +// SetExecutant / Executant + +func TestSetExecutant_Executant(t *testing.T) { + sess, _ := newTestSession(t) + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp.example.com", + OpaqueId: "user-42", + Type: userpb.UserType_USER_TYPE_PRIMARY, + }, + Username: "alice", + DisplayName: "Alice", + } + sess.SetExecutant(u) + got := sess.Executant() + assert.Equal(t, "idp.example.com", got.Idp) + assert.Equal(t, "user-42", got.OpaqueId) + assert.Equal(t, userpb.UserType_USER_TYPE_PRIMARY, got.Type) +} + +// NodeExists + +func TestNodeExists(t *testing.T) { + sess, _ := newTestSession(t) + + assert.False(t, sess.NodeExists(), "absent key should return false") + + sess.SetStorageValue("NodeExists", "true") + assert.True(t, sess.NodeExists()) + + sess.SetStorageValue("NodeExists", "false") + assert.False(t, sess.NodeExists()) + + sess.SetStorageValue("NodeExists", "yes") + assert.False(t, sess.NodeExists(), "non-'true' value should return false") +} + +// Checksums / SetChecksums + +func TestChecksums_SetChecksums(t *testing.T) { + sess, _ := newTestSession(t) + sha1bytes := []byte{0x01, 0x02, 0x03, 0x04} + md5bytes := []byte{0x05, 0x06, 0x07, 0x08} + adlerBytes := []byte{0x09, 0x0a, 0x0b, 0x0c} + + sess.SetChecksums(sha1bytes, md5bytes, adlerBytes) + got := sess.Checksums() + assert.Equal(t, sha1bytes, got.SHA1) + assert.Equal(t, md5bytes, got.MD5) + assert.Equal(t, adlerBytes, got.Adler32) +} + +// ScanData / SetScanData + +func TestScanData_SetScanData(t *testing.T) { + sess, _ := newTestSession(t) + + result, date := sess.ScanData() + assert.Empty(t, result, "fresh session should have no scan result") + assert.True(t, date.IsZero(), "fresh session should have zero scan date") + + now := time.Now().Truncate(time.Second) + sess.SetScanData("clean", now) + result, date = sess.ScanData() + assert.Equal(t, "clean", result) + assert.WithinDuration(t, now, date, time.Second) +} + +// Expires + +func TestExpires(t *testing.T) { + sess, _ := newTestSession(t) + assert.True(t, sess.Expires().IsZero(), "absent expires should be zero time") + + want := time.Now().Add(time.Hour).Truncate(time.Second) + sess.SetMetadata("expires", utils.TimeToOCMtime(want)) + got := sess.Expires() + assert.WithinDuration(t, want, got, time.Second) +} + +// IsProcessing + +func TestIsProcessing(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSize(100) + + assert.False(t, sess.IsProcessing(), "size != offset should not be processing") + + sess.info.Offset = 100 + assert.True(t, sess.IsProcessing(), "size == offset with no scan result should be processing") + + sess.SetScanData("clean", time.Now()) + assert.False(t, sess.IsProcessing(), "scan result set means processing finished") +} + +// Reference + +func TestReference(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "prov-1") + sess.SetStorageValue("SpaceRoot", "space-2") + sess.SetStorageValue("NodeId", "node-3") + + ref := sess.Reference() + require.NotNil(t, ref.ResourceId) + assert.Equal(t, "prov-1", ref.ResourceId.StorageId) + assert.Equal(t, "space-2", ref.ResourceId.SpaceId) + assert.Equal(t, "node-3", ref.ResourceId.OpaqueId) +} + +// Metadata + +func TestMetadata(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "p1") + sess.SetMetadata("mtime", "12345.0") + sess.SetStorageValue("NodeExists", "true") + sess.SetMetadata("versionsPath", "/v/path") + + m := sess.Metadata() + assert.Equal(t, "p1", m["providerID"]) + assert.Equal(t, "12345.0", m["mtime"]) + assert.Equal(t, "true", m["nodeExists"]) + assert.Equal(t, "/v/path", m["versionsPath"]) + assert.Equal(t, sess.ID(), m["sessionID"]) +} + +// Persist + Get round-trip + +func TestPersist_GetRoundtrip(t *testing.T) { + log := zerolog.Nop() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, &log) + require.NoError(t, fs.Setup()) + + sess := fs.New(context.Background()).(*FileSession) + sess.SetSize(512) + sess.SetMetadata("providerID", "prov-rt") + sess.SetStorageValue("NodeId", "nd-rt") + sess.SetStorageValue("NodeExists", "true") + + require.NoError(t, sess.TouchBin()) + require.NoError(t, sess.Persist(context.Background())) + + loaded, err := fs.Get(context.Background(), sess.ID()) + require.NoError(t, err) + ls := loaded.(*FileSession) + + assert.Equal(t, int64(512), ls.Size()) + assert.Equal(t, "prov-rt", ls.ProviderID()) + assert.Equal(t, "nd-rt", ls.NodeID()) + assert.True(t, ls.NodeExists()) +} + +func TestPersist_CreatesIntermediateDirs(t *testing.T) { + log := zerolog.Nop() + root := filepath.Join(t.TempDir(), "sub", "nested") + fs := NewFileStore(root, TokenOptions{}, &log) + + sess := fs.New(context.Background()).(*FileSession) + sess.SetSize(1) + + err := sess.Persist(context.Background()) + assert.NoError(t, err) + _, statErr := os.Stat(sess.infoPath()) + assert.NoError(t, statErr) +} + +// WriteChunk + +func TestWriteChunk(t *testing.T) { + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + require.NoError(t, sess.TouchBin()) + + n, err := sess.WriteChunk(context.Background(), 0, strings.NewReader("hello")) + require.NoError(t, err) + assert.Equal(t, int64(5), n) + assert.Equal(t, int64(5), sess.Offset()) + + n2, err := sess.WriteChunk(context.Background(), 5, strings.NewReader(" world")) + require.NoError(t, err) + assert.Equal(t, int64(6), n2) + assert.Equal(t, int64(11), sess.Offset()) + + data, err := os.ReadFile(sess.binPath()) + require.NoError(t, err) + assert.Equal(t, "hello world", string(data)) +} + +func TestWriteChunk_NoBinFile(t *testing.T) { + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + + _, err := sess.WriteChunk(context.Background(), 0, strings.NewReader("data")) + assert.Error(t, err) +} + +// Cleanup + +func setupCleanupSession(t *testing.T) *FileSession { + t.Helper() + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + require.NoError(t, sess.TouchBin()) + require.NoError(t, sess.Persist(context.Background())) + return sess +} + +func TestCleanup_BinOnly(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(true, false) + _, err := os.Stat(sess.binPath()) + assert.True(t, os.IsNotExist(err), "bin should be removed") + _, err = os.Stat(sess.infoPath()) + assert.NoError(t, err, "info should survive") +} + +func TestCleanup_InfoOnly(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(false, true) + _, err := os.Stat(sess.binPath()) + assert.NoError(t, err, "bin should survive") + _, err = os.Stat(sess.infoPath()) + assert.True(t, os.IsNotExist(err), "info should be removed") +} + +func TestCleanup_Both(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(true, true) + _, err := os.Stat(sess.binPath()) + assert.True(t, os.IsNotExist(err), "bin should be removed") + _, err = os.Stat(sess.infoPath()) + assert.True(t, os.IsNotExist(err), "info should be removed") +} + +func TestCleanup_Neither(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(false, false) + _, err := os.Stat(sess.binPath()) + assert.NoError(t, err, "bin should survive") + _, err = os.Stat(sess.infoPath()) + assert.NoError(t, err, "info should survive") +} + +func TestCleanup_MissingFiles_NoError(t *testing.T) { + sess, _ := newTestSession(t) + assert.NotPanics(t, func() { + sess.Cleanup(true, true) + }) +} + +// calculateChecksums + +func TestCalculateChecksums(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "testfile") + require.NoError(t, os.WriteFile(path, []byte("hello"), 0600)) + + sha1h, md5h, adler32h, err := calculateChecksums(context.Background(), path) + require.NoError(t, err) + + assert.Equal(t, "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", hex.EncodeToString(sha1h.Sum(nil))) + assert.Equal(t, "5d41402abc4b2a76b9719d911017c592", hex.EncodeToString(md5h.Sum(nil))) + assert.Equal(t, "062c0215", hex.EncodeToString(adler32h.Sum(nil))) +} + +func TestCalculateChecksums_MissingFile(t *testing.T) { + _, _, _, err := calculateChecksums(context.Background(), "/nonexistent/path/file.bin") + assert.Error(t, err) +} + +// checkHash + +func TestCheckHash_Correct(t *testing.T) { + h := sha1.New() //nolint:gosec + h.Write([]byte("hello")) + err := checkHash("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", h) + assert.NoError(t, err) +} + +func TestCheckHash_Mismatch(t *testing.T) { + h := sha1.New() //nolint:gosec + h.Write([]byte("hello")) + err := checkHash("0000000000000000000000000000000000000000", h) + require.Error(t, err) + assert.ErrorAs(t, err, new(errtypes.ChecksumMismatch)) +} + +// joinURLParts + +func TestJoinURLParts(t *testing.T) { + tests := []struct { + parts []string + want string + }{ + {[]string{"http://host/", "path"}, "http://host/path"}, + {[]string{"http://host", "path"}, "http://host/path"}, + {[]string{"http://host"}, "http://host"}, + {[]string{"http://host", "a", "b"}, "http://host/a/b"}, + {[]string{"http://host/", "a/", "b"}, "http://host/a/b"}, + } + for _, tc := range tests { + got := joinURLParts(tc.parts...) + assert.Equal(t, tc.want, got, "joinURLParts(%v)", tc.parts) + } +} + +// Context + +func TestContext(t *testing.T) { + log := zerolog.Nop() + fs := NewFileStore(t.TempDir(), TokenOptions{}, &log) + sess := fs.New(context.Background()).(*FileSession) + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp.test", + OpaqueId: "ctx-user", + Type: userpb.UserType_USER_TYPE_PRIMARY, + }, + Username: "ctxuser", + } + sess.SetExecutant(u) + sess.SetMetadata("lockid", "lock-xyz") + sess.SetMetadata("initiatorid", "initiator-abc") + + ctx := sess.Context(context.Background()) + + gotUser, ok := ctxpkg.ContextGetUser(ctx) + require.True(t, ok) + assert.Equal(t, "ctx-user", gotUser.GetId().GetOpaqueId()) + assert.Equal(t, "idp.test", gotUser.GetId().GetIdp()) + + lockID, ok := ctxpkg.ContextGetLockID(ctx) + require.True(t, ok) + assert.Equal(t, "lock-xyz", lockID) + + initiator, ok := ctxpkg.ContextGetInitiator(ctx) + require.True(t, ok) + assert.Equal(t, "initiator-abc", initiator) +} + +// URL + +func TestURL(t *testing.T) { + log := zerolog.Nop() + opts := TokenOptions{ + DownloadEndpoint: "http://download.example.com", + DataGatewayEndpoint: "http://gateway.example.com", + TransferSharedSecret: "s3cr3t", + TransferExpires: 3600, + } + fs := NewFileStore(t.TempDir(), opts, &log) + sess := fs.New(context.Background()).(*FileSession) + + url, err := sess.URL(context.Background()) + require.NoError(t, err) + assert.NotEmpty(t, url) + assert.True(t, strings.HasPrefix(url, "http://gateway.example.com"), "URL should start with DataGatewayEndpoint") +} + +func TestURL_NonEmpty(t *testing.T) { + log := zerolog.Nop() + opts := TokenOptions{ + DataGatewayEndpoint: "http://gw.example.com", + TransferSharedSecret: "secret", + TransferExpires: 60, + } + fs := NewFileStore(t.TempDir(), opts, &log) + sess := fs.New(context.Background()).(*FileSession) + + url1, err := sess.URL(context.Background()) + require.NoError(t, err) + url2, err := sess.URL(context.Background()) + require.NoError(t, err) + + assert.NotEmpty(t, url1) + assert.NotEmpty(t, url2) +} diff --git a/pkg/utils/etag.go b/pkg/utils/etag.go new file mode 100644 index 00000000000..9293b463451 --- /dev/null +++ b/pkg/utils/etag.go @@ -0,0 +1,42 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package utils + +import ( + "crypto/md5" + "fmt" + "io" + "time" +) + +// CalculateEtag returns a hash of fileid + tmtime (or mtime) +func CalculateEtag(id string, t time.Time) (string, error) { + h := md5.New() + if _, err := io.WriteString(h, id); err != nil { + return "", err + } + if tb, err := t.UTC().MarshalBinary(); err == nil { + if _, err := h.Write(tb); err != nil { + return "", err + } + } else { + return "", err + } + return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil +}