From 18f6f21f1e493f320917af6ea7811cdf72863203 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Tue, 30 Jun 2026 15:38:39 +0200 Subject: [PATCH 01/20] feat(OCISDEV-900): extract upload coordinator out of decomposedfs --- .../storageprovider/storageprovider.go | 29 +- .../services/dataprovider/dataprovider.go | 20 +- .../utils/decomposedfs/decomposedfs.go | 393 ++------------- pkg/storage/utils/decomposedfs/upload.go | 21 + .../utils/decomposedfs/upload/session.go | 10 + .../utils/decomposedfs/upload_async_test.go | 5 +- pkg/upload/coordinator.go | 472 ++++++++++++++++++ 7 files changed, 587 insertions(+), 363 deletions(-) create mode 100644 pkg/upload/coordinator.go diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 01fd4045b7f..0a121fc3efe 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -103,9 +103,18 @@ func (c *config) init() { } } +// uploadCoordinatorProvider is satisfied by storage drivers that can vend a +// ready-to-use upload coordinator (e.g. decomposedfs). Using a local interface +// avoids importing the decomposedfs or pkg/upload packages here, which would +// create an import cycle through decomposedfs/node → storageprovider. +type uploadCoordinatorProvider interface { + UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) +} + type Service struct { conf *config Storage storage.FS + Coordinator storage.FS // nil when driver does not support coordinated uploads dataServerURL *url.URL availableXS []*provider.ResourceChecksumPriority } @@ -202,9 +211,23 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } + // Build and start the upload coordinator if the driver supports it. + var coordinator storage.FS + if cp, ok := fs.(uploadCoordinatorProvider); ok { + evstream, err := estreamFromConfig(c.Events) + if err != nil { + return nil, err + } + coordinator, err = cp.UploadCoordinator(evstream, log) + if err != nil { + return nil, err + } + } + service := &Service{ conf: c, Storage: fs, + Coordinator: coordinator, dataServerURL: u, availableXS: xsTypes, } @@ -427,7 +450,11 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate metadata["expires"] = strconv.Itoa(int(expirationTimestamp.Seconds)) } - uploadIDs, err := s.Storage.InitiateUpload(ctx, req.Ref, uploadLength, metadata) + var initiator storage.FS = s.Storage + if s.Coordinator != nil { + initiator = s.Coordinator + } + uploadIDs, err := initiator.InitiateUpload(ctx, req.Ref, uploadLength, metadata) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index bffe73bebe1..774a1a1e4d1 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -35,6 +35,13 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/fs/registry" ) +// uploadCoordinatorProvider is satisfied by storage drivers that can vend a +// ready-to-use upload coordinator. Using a local interface avoids importing +// decomposedfs, which would create an import cycle. +type uploadCoordinatorProvider interface { + UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) +} + func init() { global.Register("dataprovider", New) } @@ -104,7 +111,18 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - dataTXs, err := getDataTXs(conf, fs, evstream, log) + // Wrap the FS in the upload coordinator if the driver supports it. + // The coordinator IS-A storage.FS, so it passes unchanged to getDataTXs. + txFS := storage.FS(fs) + if cp, ok := fs.(uploadCoordinatorProvider); ok { + coordinator, err := cp.UploadCoordinator(evstream, log) + if err != nil { + return nil, err + } + txFS = coordinator + } + + dataTXs, err := getDataTXs(conf, txFS, evstream, log) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index f1e348f8c28..8d610563cb5 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -28,9 +28,7 @@ import ( "path/filepath" "strconv" "strings" - "time" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/jellydator/ttlcache/v2" @@ -43,13 +41,10 @@ import ( "golang.org/x/sync/errgroup" "github.com/owncloud/reva/v2/pkg/appctx" - "github.com/owncloud/reva/v2/pkg/autoprop" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" - "github.com/owncloud/reva/v2/pkg/logger" "github.com/owncloud/reva/v2/pkg/rgrpc/todo/pool" - "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/utils/chunking" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/aspects" @@ -63,6 +58,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/timemanager" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/trashbin" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/tree" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/usermapper" "github.com/owncloud/reva/v2/pkg/storage/utils/filelocks" @@ -80,14 +76,6 @@ const ( var ( tracer trace.Tracer - - _registeredEvents = []events.Unmarshaller{ - events.PostprocessingFinished{}, - events.PostprocessingStepFinished{}, - events.RestartPostprocessing{}, - events.CleanUpload{}, - events.RevertRevision{}, - } ) func init() { @@ -107,6 +95,7 @@ type Session interface { LockID() string } +// SessionStore is the interface for upload session persistence. type SessionStore interface { New(ctx context.Context) *upload.OcisSession List(ctx context.Context) ([]*upload.OcisSession, error) @@ -254,363 +243,47 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor return nil, err } - if o.AsyncFileUploads { - if fs.stream == nil { - log.Error().Msg("need event stream for async file processing") - return nil, errors.New("need nats for async file processing") - } - - ch, err := events.Consume(fs.stream, o.Events.ConsumerGroup, _registeredEvents...) - if err != nil { - return nil, err - } - - if o.Events.NumConsumers <= 0 { - o.Events.NumConsumers = 1 - } - - for i := 0; i < o.Events.NumConsumers; i++ { - go fs.Postprocessing(ch) - } + if o.AsyncFileUploads && fs.stream == nil { + log.Error().Msg("need event stream for async file processing") + return nil, errors.New("need nats for async file processing") } return fs, nil } -// Postprocessing starts the postprocessing result collector -func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { - log := logger.New() - for event := range ch { - evCtx := context.Background() - fs.processEvent(evCtx, event, log) - } -} - -func (fs *Decomposedfs) processEvent(evCtx context.Context, event events.Event, log *zerolog.Logger) { - ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) - ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) - defer span.End() - - switch ev := event.Event.(type) { - case events.PostprocessingFinished: - sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - - ctx = session.Context(ctx) - - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - if !n.Exists { - sublog.Debug().Msg("node no longer exists") - session.Cleanup(false, true, true, false) - return - } - - var ( - failed bool - revertNodeMetadata bool - keepUpload bool - ) - unmarkPostprocessing := true - - switch ev.Outcome { - default: - sublog.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - revertNodeMetadata = true - keepUpload = true - metrics.UploadSessionsAborted.Inc() - case events.PPOutcomeContinue: - if err := session.Finalize(ctx); err != nil { - sublog.Error().Err(err).Msg("could not finalize upload") - failed = true - revertNodeMetadata = false - keepUpload = true - // keep postprocessing status so the upload is not deleted during housekeeping - unmarkPostprocessing = false - } else { - metrics.UploadSessionsFinalized.Inc() - } - case events.PPOutcomeDelete: - failed = true - revertNodeMetadata = true - metrics.UploadSessionsDeleted.Inc() - } - getParent := func() *node.Node { - p, err := n.Parent(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read parent") - return nil - } - return p - } - - now := time.Now() - if failed { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - sublog.Error().Err(err).Msg("reading node for session failed") - } - if latestSession == session.ID() { - // propagate reverted sizeDiff after failed postprocessing - if err := fs.tp.Propagate(ctx, n, -session.SizeDiff()); err != nil { - sublog.Error().Err(err).Msg("could not propagate tree size change") - } - } - } else if p := getParent(); p != nil { - // update parent tmtime to propagate etag change after successful postprocessing - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - sublog.Error().Err(err).Msg("could not propagate etag change") - } - } - - session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) - - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionsPath"] != "" { - isVersion = true - } - } - - if err := events.Publish( - ctx, - fs.stream, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - }, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), - IsVersion: isVersion, - ImpersonatingUser: ev.ImpersonatingUser, - }, - ); err != nil { - sublog.Error().Err(err).Msg("Failed to publish UploadReady event") - } - case events.RestartPostprocessing: - sublog := log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - s, err := session.URL(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not create url") - return - } - - metrics.UploadSessionsRestarted.Inc() - - // restart postprocessing - if err := events.Publish(ctx, fs.stream, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(ctx), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - }); err != nil { - sublog.Error().Err(err).Msg("Failed to publish BytesReceived event") - } - case events.CleanUpload: - sublog := log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) - case events.RevertRevision: - sublog := log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node") - return - } - - if err := n.RevertCurrentRevision(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to revert revision") - return - } - case events.PostprocessingStepFinished: - sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != fs.o.MountID { - sublog.Debug().Msg("ignoring event for different storage") - return - } - if ev.FinishedStep != events.PPStepAntivirus { - // atm we are only interested in antivirus results - return - } - - res := ev.Result.(events.VirusscanResult) - if res.ErrorMsg != "" { - // scan failed somehow - // Should we handle this here? - return - } - sublog = log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() - - var n *node.Node - switch ev.UploadID { - case "": - // uploadid is empty -> this was an on-demand scan - /* ON DEMAND SCANNING NOT SUPPORTED ATM - ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) - ref := &provider.Reference{ResourceId: ev.ResourceID} - - no, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") - continue - - } - n = no - if ev.Outcome == events.PPOutcomeDelete { - // antivir wants us to delete the file. We must obey and need to - - // check if there a previous versions existing - revs, err := fs.ListRevisions(ctx, ref) - if len(revs) == 0 { - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") - } - - // no versions -> trash file - err := fs.Delete(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") - continue - } - - // now purge it from the recycle bin - if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - - // we have versions - find the newest - versions := make(map[uint64]string) // remember all versions - we need them later - var nv uint64 - for _, v := range revs { - versions[v.Mtime] = v.Key - if v.Mtime > nv { - nv = v.Mtime - } - } - - // restore newest version - if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") - continue - } - - // now find infected version - revs, err = fs.ListRevisions(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") - } - - for _, v := range revs { - // we looking for a version that was previously not there - if _, ok := versions[v.Mtime]; ok { - continue - } - - if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") - } - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - */ - default: - // uploadid is not empty -> this is an async upload - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - - n, err = session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node after scan") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - - session.SetScanData(res.Description, res.Scandate) - if err := session.Persist(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to persist scan results") - } - } - - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - sublog.Error().Err(err).Msg("Failed to set scan results") - return - } +// Shutdown shuts down the storage +func (fs *Decomposedfs) Shutdown(ctx context.Context) error { + return nil +} - metrics.UploadSessionsScanned.Inc() - default: - log.Error().Interface("event", ev).Msg("Unknown event") +// RevertUploadRevision reverts an in-flight upload by calling RevertCurrentRevision on the node. +// It implements upload.RevisionReverter so the coordinator can delegate RevertRevision events. +func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error { + n, err := fs.lu.NodeFromID(ctx, id) + if err != nil { + return err } + return n.RevertCurrentRevision(ctx) } -// Shutdown shuts down the storage -func (fs *Decomposedfs) Shutdown(ctx context.Context) error { - return nil +// UploadCoordinator constructs, wires, and starts a driver-agnostic upload +// coordinator for this decomposedfs instance. The returned storage.FS embeds +// decomposedfs and overrides the upload-related methods. Callers (storageprovider, +// dataprovider) only interact with storage.FS — they need not import pkg/upload +// or decomposedfs directly. +func (fs *Decomposedfs) UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) { + var store pkgupload.SessionStore + if ocisStore, ok := fs.sessionStore.(*upload.OcisStore); ok { + store = ocisStore + } + c := pkgupload.NewCoordinator(fs, store, fs.stream, fs.o.MountID, fs.o.Events.ConsumerGroup, fs.o.Events.NumConsumers, log) + if stream != nil { + if err := c.Start(stream); err != nil { + return nil, err + } + } + return c, nil } // GetQuota returns the quota available diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index abacbd25cd9..5192077f69b 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -385,6 +385,18 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere }, false) // acquireLock=false, because outer lock already held } +// PropagateRevertedSize updates the tree size counters after a failed upload +// that was previously counted. Called by the upload coordinator when a +// postprocessing outcome of PPOutcomeAbort or PPOutcomeDelete is received and +// the session still owns the processing slot (latestSession == session.ID()). +func (fs *Decomposedfs) PropagateRevertedSize(ctx context.Context, id *provider.ResourceId, sizeDiff int64) error { + n, err := fs.lu.NodeFromID(ctx, id) + if err != nil { + return err + } + return fs.tp.Propagate(ctx, n, sizeDiff) +} + // CommitUpload writes the staged bytes from source to the resource at ref. func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { if source.Body == nil { @@ -559,6 +571,15 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") } + // Update parent tmtime so that etag changes propagate to folder listings + // after a successful commit (previously done in processEvent; absorbed here + // so that all drivers inheriting CommitUpload get it for free). + if p, err := n.Parent(ctx); err == nil && p != nil { + now := time.Now() + _ = p.SetTMTime(ctx, &now) + _ = fs.tp.Propagate(ctx, p, 0) + } + // etag is a best-effort, recomputable value; a failure here must not fail an // already-committed upload (matches the legacy Upload path). etag, _ := node.CalculateEtag(n.ID, mtime) diff --git a/pkg/storage/utils/decomposedfs/upload/session.go b/pkg/storage/utils/decomposedfs/upload/session.go index 8d7a58ce68a..e65d2a1fa44 100644 --- a/pkg/storage/utils/decomposedfs/upload/session.go +++ b/pkg/storage/utils/decomposedfs/upload/session.go @@ -225,6 +225,11 @@ func (s *OcisSession) Dir() string { return s.info.Storage["Dir"] } +// SpaceGid returns the numeric GID of the space owner, or "" if not set. +func (s *OcisSession) SpaceGid() string { + return s.info.Storage["SpaceGid"] +} + // Size returns the upload size func (s *OcisSession) Size() int64 { return s.info.Size @@ -311,6 +316,11 @@ func (s *OcisSession) binPath() string { return filepath.Join(s.store.root, "uploads", s.info.ID) } +// BinPath returns the path to the staged binary file. +func (s *OcisSession) BinPath() string { + return s.binPath() +} + // infoPath returns the path to the .info file storing the file's info. func (s *OcisSession) infoPath() string { return sessionPath(s.store.root, s.info.ID) diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 35827ec15c0..0187e795242 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -184,7 +184,10 @@ var _ = Describe("Async file uploads", Ordered, func() { EventStream: stream.Chan{pub, con}, Trashbin: &DecomposedfsTrashbin{}, } - fs, err = New(o, aspects, &zerolog.Logger{}) + dfs, err := New(o, aspects, &zerolog.Logger{}) + Expect(err).ToNot(HaveOccurred()) + // Wire the coordinator so postprocessing events are consumed during tests. + fs, err = dfs.(*Decomposedfs).UploadCoordinator(aspects.EventStream, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"}) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go new file mode 100644 index 00000000000..041ff76ce0e --- /dev/null +++ b/pkg/upload/coordinator.go @@ -0,0 +1,472 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package upload provides the driver-agnostic upload coordinator: +// TUS session management, postprocessing event loop, lifecycle event +// publishing. Any storage driver can embed the Coordinator to inherit +// the full oCIS upload pipeline. +package upload + +import ( + "context" + "path/filepath" + "time" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" + + "github.com/owncloud/reva/v2/pkg/autoprop" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/owncloud/reva/v2/pkg/storage" + decomposedupload "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" + "github.com/owncloud/reva/v2/pkg/utils" +) + +var tracer trace.Tracer + +func init() { + tracer = otel.Tracer("github.com/owncloud/reva/pkg/upload") +} + +var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) + +// SessionStore abstracts upload-session persistence for the Coordinator. +// The concrete implementation during OCISDEV-900 is *decomposedupload.OcisStore. +type SessionStore interface { + New(ctx context.Context) *decomposedupload.OcisSession + Get(ctx context.Context, id string) (*decomposedupload.OcisSession, error) + List(ctx context.Context) ([]*decomposedupload.OcisSession, error) +} + +// RevisionReverter is an optional interface a storage driver may implement +// to handle RevertRevision postprocessing events. Decomposedfs implements it. +type RevisionReverter interface { + RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error +} + +// SizePropagator is an optional interface a storage driver may implement to +// propagate a reverted size delta up the directory tree after a failed upload. +// Decomposedfs implements it. Drivers whose tree accounting is server-side do +// not need to implement this. +type SizePropagator interface { + PropagateRevertedSize(ctx context.Context, id *provider.ResourceId, sizeDiff int64) error +} + +// Coordinator owns the upload state machine: +// - TUS HTTP layer (InitiateUpload, UseIn, GetUpload, ListUploadSessions) +// - postprocessing event loop (PostprocessingFinished, PostprocessingStepFinished, +// RestartPostprocessing, CleanUpload, RevertRevision) +// +// It embeds storage.FS so it IS-A storage.FS and can be passed wherever the +// underlying driver is expected. All methods not overridden here delegate to +// the embedded FS. +type Coordinator struct { + storage.FS + store SessionStore + pub events.Publisher + mountID string + numConc int + conGroup string + log *zerolog.Logger +} + +// NewCoordinator constructs a Coordinator. Call Start to begin consuming events. +func NewCoordinator( + fs storage.FS, + store SessionStore, + pub events.Publisher, + mountID string, + consumerGroup string, + numConsumers int, + log *zerolog.Logger, +) *Coordinator { + if numConsumers <= 0 { + numConsumers = 1 + } + return &Coordinator{ + FS: fs, + store: store, + pub: pub, + mountID: mountID, + numConc: numConsumers, + conGroup: consumerGroup, + log: log, + } +} + +// Start subscribes to the event stream and launches numConsumers goroutines +// that process postprocessing events. +func (c *Coordinator) Start(stream events.Consumer) error { + ch, err := events.Consume( + stream, + c.conGroup, + events.PostprocessingFinished{}, + events.PostprocessingStepFinished{}, + events.RestartPostprocessing{}, + events.CleanUpload{}, + events.RevertRevision{}, + ) + if err != nil { + return err + } + for i := 0; i < c.numConc; i++ { + go c.postprocessingLoop(ch) + } + return nil +} + +func (c *Coordinator) postprocessingLoop(ch <-chan events.Event) { + for event := range ch { + c.processEvent(context.Background(), event) + } +} + +func (c *Coordinator) processEvent(evCtx context.Context, event events.Event) { + ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) + ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) + defer span.End() + + switch ev := event.Event.(type) { + case events.PostprocessingFinished: + c.handlePostprocessingFinished(ctx, ev) + case events.PostprocessingStepFinished: + c.handlePostprocessingStepFinished(ctx, ev) + case events.RestartPostprocessing: + c.handleRestartPostprocessing(ctx, ev) + case events.CleanUpload: + c.handleCleanUpload(ctx, ev) + case events.RevertRevision: + c.handleRevertRevision(ctx, ev) + default: + c.log.Error().Interface("event", ev).Msg("coordinator: unknown event") + } +} + +func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { + log := c.log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + + ctx = session.Context(ctx) + + n, err := session.Node(ctx) + if err != nil { + log.Error().Err(err).Msg("could not read node") + return + } + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + if !n.Exists { + log.Debug().Msg("node no longer exists") + session.Cleanup(false, true, true, false) + return + } + + var ( + failed bool + revertNodeMetadata bool + keepUpload bool + ) + unmarkPostprocessing := true + + switch ev.Outcome { + default: + log.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") + fallthrough + case events.PPOutcomeAbort: + failed = true + revertNodeMetadata = true + keepUpload = true + metrics.UploadSessionsAborted.Inc() + case events.PPOutcomeContinue: + if err := session.Finalize(ctx); err != nil { + log.Error().Err(err).Msg("could not finalize upload") + failed = true + revertNodeMetadata = false + keepUpload = true + unmarkPostprocessing = false + } else { + metrics.UploadSessionsFinalized.Inc() + } + case events.PPOutcomeDelete: + failed = true + revertNodeMetadata = true + metrics.UploadSessionsDeleted.Inc() + } + + now := time.Now() + if failed { + // Propagate the reverted size so parent tree counters stay correct. + // This is only needed when the session still owns the processing slot + // (i.e. no later upload has taken over). The check and the propagation + // are both delegated to the driver via SizePropagator so that external + // drivers (which do tree accounting server-side) can no-op this. + latestSession, err := n.ProcessingID(ctx) + if err != nil { + log.Error().Err(err).Msg("reading node processingID failed") + } else if latestSession == session.ID() { + if sp, ok := c.FS.(SizePropagator); ok { + if err := sp.PropagateRevertedSize(ctx, session.Reference().ResourceId, -session.SizeDiff()); err != nil { + log.Error().Err(err).Msg("could not propagate reverted size") + } + } + } + } else { + // Bump parent tmtime so etag propagates to folder listings. + // CommitUpload handles this for the sync path; session.Finalize (async + // path) does not go through CommitUpload, so we do it here. + p, perr := n.Parent(ctx) + if perr == nil && p != nil { + _ = p.SetTMTime(ctx, &now) + } + } + + session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) + + var isVersion bool + if session.NodeExists() { + info, err := session.GetInfo(ctx) + if err == nil && info.MetaData["versionsPath"] != "" { + isVersion = true + } + } + + if err := events.Publish( + ctx, + c.pub, + events.UploadReady{ + UploadID: ev.UploadID, + Failed: failed, + ExecutingUser: ev.ExecutingUser, + Filename: ev.Filename, + FileRef: &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + }, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Timestamp: utils.TimeToTS(now), + SpaceOwner: n.SpaceOwnerOrManager(ctx), + IsVersion: isVersion, + ImpersonatingUser: ev.ImpersonatingUser, + }, + ); err != nil { + log.Error().Err(err).Msg("Failed to publish UploadReady event") + } +} + +func (c *Coordinator) handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { + log := c.log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + n, err := session.Node(ctx) + if err != nil { + log.Error().Err(err).Msg("could not read node") + return + } + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + s, err := session.URL(ctx) + if err != nil { + log.Error().Err(err).Msg("could not create url") + return + } + + metrics.UploadSessionsRestarted.Inc() + + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: n.SpaceOwnerOrManager(ctx), + ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, + ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + }); err != nil { + log.Error().Err(err).Msg("Failed to publish BytesReceived event") + } +} + +func (c *Coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUpload) { + log := c.log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) +} + +func (c *Coordinator) handleRevertRevision(ctx context.Context, ev events.RevertRevision) { + log := c.log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + rr, ok := c.FS.(RevisionReverter) + if !ok { + log.Error().Msg("storage driver does not implement RevisionReverter") + return + } + if err := rr.RevertUploadRevision(ctx, ev.ResourceID); err != nil { + log.Error().Err(err).Msg("Failed to revert revision") + } +} + +func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { + log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + if ev.FinishedStep != events.PPStepAntivirus { + return + } + + res := ev.Result.(events.VirusscanResult) + if res.ErrorMsg != "" { + return + } + log = c.log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() + + if ev.UploadID == "" { + // on-demand scanning not supported + return + } + + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + + n, err := session.Node(ctx) + if err != nil { + log.Error().Err(err).Msg("Failed to get node after scan") + return + } + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + + session.SetScanData(res.Description, res.Scandate) + if err := session.Persist(ctx); err != nil { + log.Error().Err(err).Msg("Failed to persist scan results") + } + + if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { + log.Error().Err(err).Msg("Failed to set scan results") + return + } + + metrics.UploadSessionsScanned.Inc() +} + +// InitiateUpload delegates to the underlying storage.FS. +func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + return c.FS.InitiateUpload(ctx, ref, uploadLength, metadata) +} + +// UseIn registers the coordinator as the TUS data store in the composer. +func (c *Coordinator) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(c) + composer.UseTerminater(c) + composer.UseConcater(c) + composer.UseLengthDeferrer(c) +} + +// NewUpload is not supported; uploads are initiated via the CS3 API. +func (c *Coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { + return nil, errNotImplemented +} + +// GetUpload returns the upload session for the given id. +func (c *Coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + return c.store.Get(ctx, id) +} + +// ListUploadSessions returns upload sessions matching the given filter. +func (c *Coordinator) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + sessions, err := c.store.List(ctx) + if err != nil { + return nil, err + } + result := []storage.UploadSession{} + now := time.Now() + for _, s := range sessions { + if filter.ID != nil && *filter.ID != "" && s.ID() != *filter.ID { + continue + } + if filter.Processing != nil && *filter.Processing != s.IsProcessing() { + continue + } + if filter.Expired != nil { + if *filter.Expired { + if now.Before(s.Expires()) { + continue + } + } else { + if now.After(s.Expires()) { + continue + } + } + } + if filter.HasVirus != nil { + sr, _ := s.ScanData() + infected := sr != "" + if *filter.HasVirus != infected { + continue + } + } + result = append(result, s) + } + return result, nil +} + +// AsTerminatableUpload returns the upload as a TerminatableUpload. +func (c *Coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*decomposedupload.OcisSession) +} + +// AsLengthDeclarableUpload returns the upload as a LengthDeclarableUpload. +func (c *Coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*decomposedupload.OcisSession) +} + +// AsConcatableUpload returns the upload as a ConcatableUpload. +func (c *Coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*decomposedupload.OcisSession) +} From 6df2d1da32c0c3c4a2aedba34c854db766e5d726 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Tue, 30 Jun 2026 16:39:38 +0200 Subject: [PATCH 02/20] feat(OCISDEV-900): define generic Session interface; remove OcisSession concrete dep from coordinator --- .../utils/decomposedfs/decomposedfs.go | 24 ++++++++- .../utils/decomposedfs/upload/session.go | 31 +++++++++++ .../utils/decomposedfs/upload/upload.go | 2 + pkg/upload/coordinator.go | 51 +++++++++++++++---- 4 files changed, 96 insertions(+), 12 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 8d610563cb5..2e39e3f75ed 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -267,6 +267,28 @@ func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.R return n.RevertCurrentRevision(ctx) } +// ocisSessionStoreAdapter adapts *upload.OcisStore to pkgupload.SessionStore, +// bridging the concrete *OcisSession return types to the generic Session interface. +type ocisSessionStoreAdapter struct{ s *upload.OcisStore } + +func (a ocisSessionStoreAdapter) New(ctx context.Context) pkgupload.Session { + return a.s.New(ctx) +} +func (a ocisSessionStoreAdapter) Get(ctx context.Context, id string) (pkgupload.Session, error) { + return a.s.Get(ctx, id) +} +func (a ocisSessionStoreAdapter) List(ctx context.Context) ([]pkgupload.Session, error) { + sessions, err := a.s.List(ctx) + if err != nil { + return nil, err + } + result := make([]pkgupload.Session, len(sessions)) + for i, s := range sessions { + result[i] = s + } + return result, nil +} + // UploadCoordinator constructs, wires, and starts a driver-agnostic upload // coordinator for this decomposedfs instance. The returned storage.FS embeds // decomposedfs and overrides the upload-related methods. Callers (storageprovider, @@ -275,7 +297,7 @@ func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.R func (fs *Decomposedfs) UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) { var store pkgupload.SessionStore if ocisStore, ok := fs.sessionStore.(*upload.OcisStore); ok { - store = ocisStore + store = ocisSessionStoreAdapter{s: ocisStore} } c := pkgupload.NewCoordinator(fs, store, fs.stream, fs.o.MountID, fs.o.Events.ConsumerGroup, fs.o.Events.NumConsumers, log) if stream != nil { diff --git a/pkg/storage/utils/decomposedfs/upload/session.go b/pkg/storage/utils/decomposedfs/upload/session.go index e65d2a1fa44..55f3ff02c32 100644 --- a/pkg/storage/utils/decomposedfs/upload/session.go +++ b/pkg/storage/utils/decomposedfs/upload/session.go @@ -20,6 +20,7 @@ package upload import ( "context" + "encoding/hex" "encoding/json" "os" "path/filepath" @@ -34,6 +35,7 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/owncloud/reva/v2/pkg/appctx" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -337,6 +339,35 @@ func (s *OcisSession) SetScanData(result string, date time.Time) { s.info.MetaData["scanDate"] = date.Format(time.RFC3339) } +// SetChecksums stores pre-computed checksums so the coordinator can pass them +// to CommitUpload without re-reading the bin file. +func (s *OcisSession) SetChecksums(sha1, md5, adler32 []byte) { + s.info.MetaData["checksumSHA1"] = hex.EncodeToString(sha1) + s.info.MetaData["checksumMD5"] = hex.EncodeToString(md5) + s.info.MetaData["checksumAdler32"] = hex.EncodeToString(adler32) +} + +// Checksums returns the pre-computed checksums stored on the session. +func (s *OcisSession) Checksums() storage.UploadChecksums { + decode := func(key string) []byte { + b, _ := hex.DecodeString(s.info.MetaData[key]) + return b + } + return storage.UploadChecksums{ + SHA1: decode("checksumSHA1"), + MD5: decode("checksumMD5"), + Adler32: decode("checksumAdler32"), + } +} + +// Metadata returns a map of upload metadata needed to call CommitUpload. +func (s *OcisSession) Metadata() map[string]string { + return map[string]string{ + "providerID": s.info.MetaData["providerID"], + "mtime": s.info.MetaData["mtime"], + } +} + // ScanData returns the virus scan data func (s *OcisSession) ScanData() (string, time.Time) { date := s.info.MetaData["scanDate"] diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index 1a42f20878b..f2e2a011337 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -178,6 +178,8 @@ func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error { prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), } + // store on session so the coordinator can pass them to CommitUpload without re-reading the bin + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) // At this point we scope by the space to create the final file in the final location if session.store.um != nil && session.info.Storage["SpaceGid"] != "" { diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 041ff76ce0e..9098231e364 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -38,7 +38,7 @@ import ( "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" - decomposedupload "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" + "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -50,12 +50,42 @@ func init() { var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) +// Session is the driver-agnostic view of an upload session the Coordinator +// needs. OcisSession satisfies this interface. +type Session interface { + tusd.Upload + storage.UploadSession + ID() string + Filename() string + Size() int64 + SizeDiff() int64 + BinPath() string + SpaceGid() string + ProviderID() string + SpaceID() string + NodeID() string + NodeExists() bool + Dir() string + IsProcessing() bool + SpaceOwner() *user.UserId + Executant() user.UserId + Reference() provider.Reference + URL(ctx context.Context) (string, error) + SetScanData(result string, date time.Time) + Checksums() storage.UploadChecksums + Metadata() map[string]string + Persist(ctx context.Context) error + Finalize(ctx context.Context) error + Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) + Context(ctx context.Context) context.Context + Node(ctx context.Context) (*node.Node, error) +} + // SessionStore abstracts upload-session persistence for the Coordinator. -// The concrete implementation during OCISDEV-900 is *decomposedupload.OcisStore. type SessionStore interface { - New(ctx context.Context) *decomposedupload.OcisSession - Get(ctx context.Context, id string) (*decomposedupload.OcisSession, error) - List(ctx context.Context) ([]*decomposedupload.OcisSession, error) + New(ctx context.Context) Session + Get(ctx context.Context, id string) (Session, error) + List(ctx context.Context) ([]Session, error) } // RevisionReverter is an optional interface a storage driver may implement @@ -238,9 +268,8 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event } } } else { - // Bump parent tmtime so etag propagates to folder listings. - // CommitUpload handles this for the sync path; session.Finalize (async - // path) does not go through CommitUpload, so we do it here. + // Finalize writes the blob but does not bump parent tmtime; do it here + // so etag changes propagate to folder listings after async uploads. p, perr := n.Parent(ctx) if perr == nil && p != nil { _ = p.SetTMTime(ctx, &now) @@ -458,15 +487,15 @@ func (c *Coordinator) ListUploadSessions(ctx context.Context, filter storage.Upl // AsTerminatableUpload returns the upload as a TerminatableUpload. func (c *Coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*decomposedupload.OcisSession) + return up.(tusd.TerminatableUpload) } // AsLengthDeclarableUpload returns the upload as a LengthDeclarableUpload. func (c *Coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*decomposedupload.OcisSession) + return up.(tusd.LengthDeclarableUpload) } // AsConcatableUpload returns the upload as a ConcatableUpload. func (c *Coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*decomposedupload.OcisSession) + return up.(tusd.ConcatableUpload) } From 009117195eb7b5df4ea96ee2edf1ed4b887af30d Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 1 Jul 2026 00:23:15 +0200 Subject: [PATCH 03/20] feat(OCISDEV-900): coordinator owns full upload lifecycle --- pkg/storage/utils/decomposedfs/upload.go | 187 ++++++--- .../utils/decomposedfs/upload/session.go | 13 +- .../utils/decomposedfs/upload/upload.go | 37 ++ .../utils/decomposedfs/upload_async_test.go | 361 +++++------------ pkg/upload/coordinator.go | 368 ++++++++++++++---- 5 files changed, 558 insertions(+), 408 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 5192077f69b..3ef8e694356 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -385,18 +385,6 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere }, false) // acquireLock=false, because outer lock already held } -// PropagateRevertedSize updates the tree size counters after a failed upload -// that was previously counted. Called by the upload coordinator when a -// postprocessing outcome of PPOutcomeAbort or PPOutcomeDelete is received and -// the session still owns the processing slot (latestSession == session.ID()). -func (fs *Decomposedfs) PropagateRevertedSize(ctx context.Context, id *provider.ResourceId, sizeDiff int64) error { - n, err := fs.lu.NodeFromID(ctx, id) - if err != nil { - return err - } - return fs.tp.Propagate(ctx, n, sizeDiff) -} - // CommitUpload writes the staged bytes from source to the resource at ref. func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { if source.Body == nil { @@ -480,7 +468,12 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc if err != nil { return nil, errors.Wrap(err, "Decomposedfs: failed to read existing node") } - if _, err := node.CheckQuota(ctx, n.SpaceRoot, old.BlobID != "", uint64(old.Blobsize), uint64(source.Length)); err != nil { + // nodeExists reports whether the target file existed *before* this upload + // was initiated. When false, the current BlobID on the node is a + // placeholder written by CreateNodeForUpload, not real prior content, so + // no version should be archived and the quota check must treat it as new. + nodeExisted := source.Metadata["nodeExists"] == "true" + if _, err := node.CheckQuota(ctx, n.SpaceRoot, nodeExisted, uint64(old.Blobsize), uint64(source.Length)); err != nil { return nil, err } @@ -489,49 +482,80 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc return nil, errors.Wrap(err, "Decomposedfs: failed to read old mtime") } - if !fs.o.DisableVersioning && old.BlobID != "" { - versionPath := fs.lu.InternalPath(n.SpaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) + if !fs.o.DisableVersioning && nodeExisted { + // TODO(OCISDEV-900): remove versionsPath reuse once CreateNodeForUpload is retired. + // CreateNodeForUpload pre-creates the version archive during FinishUploadDecomposed, + // so CommitUpload reuses it to avoid a duplicate. The coordinator never calls + // CreateNodeForUpload, so versionsPath is always empty on the coordinator path + // and this branch is already dead there. + versionPath := source.Metadata["versionsPath"] + if versionPath == "" { + versionPath = fs.lu.InternalPath(n.SpaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) + } - revFile, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - if !errors.Is(err, os.ErrExist) { - return nil, errors.Wrap(err, "Decomposedfs: failed to create revision file") - } - // EEXIST: a revision archive at this mtime already exists from a - // prior CommitUpload run. If the archive is byte-identical to the - // live node, it is a leftover from an idempotent retry and can be - // safely reset; otherwise we refuse rather than clobber history. - if err := validateRevisionChecksums(ctx, fs.lu, old, versionPath); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: existing revision archive does not match current node") - } - bID, _, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) + _, errStat := os.Stat(versionPath) + if errStat != nil { + // Version file does not exist yet; create it. + revFile, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to read blob id of existing revision") - } - if err := fs.tp.DeleteBlob(&node.Node{BlobID: bID, SpaceID: n.SpaceID}); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to delete stale revision blob") - } - revFile, err = os.Create(versionPath) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to truncate revision file") - } - } - revFile.Close() + if !errors.Is(err, os.ErrExist) { + return nil, errors.Wrap(err, "Decomposedfs: failed to create revision file") + } + // EEXIST: a revision archive at this mtime already exists from a + // prior CommitUpload run. If the archive is byte-identical to the + // live node, it is a leftover from an idempotent retry and can be + // safely reset; otherwise we refuse rather than clobber history. + if err := validateRevisionChecksums(ctx, fs.lu, old, versionPath); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: existing revision archive does not match current node") + } + bID, _, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) + if err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to read blob id of existing revision") + } + if err := fs.tp.DeleteBlob(&node.Node{BlobID: bID, SpaceID: n.SpaceID}); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to delete stale revision blob") + } + revFile, err = os.Create(versionPath) + if err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to truncate revision file") + } + revFile.Close() + + if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, + func(name string, value []byte) ([]byte, bool) { + return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || + name == prefixes.TypeAttr || + name == prefixes.BlobIDAttr || + name == prefixes.BlobsizeAttr || + name == prefixes.MTimeAttr + }, f, true); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") + } - if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, - func(name string, value []byte) ([]byte, bool) { - return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || - name == prefixes.TypeAttr || - name == prefixes.BlobIDAttr || - name == prefixes.BlobsizeAttr || - name == prefixes.MTimeAttr - }, f, true); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") - } + if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") + } + } else { + revFile.Close() + + if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, + func(name string, value []byte) ([]byte, bool) { + return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || + name == prefixes.TypeAttr || + name == prefixes.BlobIDAttr || + name == prefixes.BlobsizeAttr || + name == prefixes.MTimeAttr + }, f, true); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") + } - if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") + if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") + } + } } + // If the version file already exists (created by CreateNodeForUpload), + // its metadata was already copied there; nothing more to do. } sizeDiff = source.Length - old.Blobsize @@ -555,21 +579,56 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc } }() - if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") - } + // Determine whether to stomp existing node metadata or only update the + // blob pointer. We should NOT overwrite Blobsize/mtime when: + // (a) a newer in-flight session currently owns the processing slot, OR + // (b) this session's upload was for a new file (nodeExists=false) but the + // node has already been fully committed by a concurrent session. + // TODO(OCISDEV-900): remove newerSessionOwnsNode once CreateNodeForUpload is + // retired. The old path allowed two sessions for the same file to race through + // postprocessing simultaneously. The coordinator serializes this via + // MarkProcessing, so only one session can hold the slot at a time and the race + // described by cases (a) and (b) cannot occur. + sessionID := source.Metadata["sessionID"] + n.ResetXattrsCache() + curProcessingID, _ := n.ProcessingID(ctx) + // Case (a): another session is actively processing. + newerSessionInFlight := curProcessingID != "" && sessionID != "" && curProcessingID != sessionID + // Case (b): node was new at initiation but a concurrent session already + // committed a real blob (processing finished before us). + concurrentCommitWon := !nodeExisted && curProcessingID == "" && old.BlobID != "" && old.BlobID != sessionID + newerSessionOwnsNode := newerSessionInFlight || concurrentCommitWon + if newerSessionOwnsNode { + // Only update the blob pointer; preserve size/mtime set by the newer session. + limitedAttrs := node.Attributes{ + prefixes.BlobIDAttr: []byte(n.BlobID), + } + for k, v := range attrs { + if strings.HasPrefix(k, prefixes.ChecksumPrefix) { + limitedAttrs[k] = v + } + } + if err := n.SetXattrsWithContext(ctx, limitedAttrs, false); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to write blob reference metadata") + } + committed = true + } else { + if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") + } - if err := n.SetXattrsWithContext(ctx, attrs, false); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to write metadata") - } - // Durable commit point: the node metadata now references the new blob. - // Past here the file is committed, so the orphaned-blob cleanup must no - // longer run - a failure in the post-commit steps below leaves the file - // intact and must not delete the referenced blob. - committed = true + if err := n.SetXattrsWithContext(ctx, attrs, false); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to write metadata") + } + // Durable commit point: the node metadata now references the new blob. + // Past here the file is committed, so the orphaned-blob cleanup must no + // longer run - a failure in the post-commit steps below leaves the file + // intact and must not delete the referenced blob. + committed = true - if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") + if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") + } } // Update parent tmtime so that etag changes propagate to folder listings // after a successful commit (previously done in processEvent; absorbed here diff --git a/pkg/storage/utils/decomposedfs/upload/session.go b/pkg/storage/utils/decomposedfs/upload/session.go index 55f3ff02c32..4f8fa854f80 100644 --- a/pkg/storage/utils/decomposedfs/upload/session.go +++ b/pkg/storage/utils/decomposedfs/upload/session.go @@ -361,10 +361,19 @@ func (s *OcisSession) Checksums() storage.UploadChecksums { } // Metadata returns a map of upload metadata needed to call CommitUpload. +// "nodeExists" carries whether the target node existed at InitiateUpload time. +// "versionsPath" carries the version archive path already created by +// CreateNodeForUpload so CommitUpload reuses it rather than creating a duplicate. +// "sessionID" is included so CommitUpload can tell whether this session still +// owns the processing slot; if a newer session took over, node size metadata +// is left intact so the in-progress upload's expected size is preserved. func (s *OcisSession) Metadata() map[string]string { return map[string]string{ - "providerID": s.info.MetaData["providerID"], - "mtime": s.info.MetaData["mtime"], + "providerID": s.info.MetaData["providerID"], + "mtime": s.info.MetaData["mtime"], + "nodeExists": s.info.Storage["NodeExists"], + "versionsPath": s.info.MetaData["versionsPath"], + "sessionID": s.info.ID, } } diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index f2e2a011337..d59ce5e1ff1 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -110,6 +110,43 @@ func (session *OcisSession) GetReader(ctx context.Context) (io.ReadCloser, error return os.Open(session.binPath()) } +// FinishBytesOnly computes and validates checksums and stores them on the session. +// It does NOT call CreateNodeForUpload, publish any event, or propagate size. +// The coordinator calls this from its coordinatedUpload.FinishUpload. +func (session *OcisSession) FinishBytesOnly(ctx context.Context) error { + ctx, span := tracer.Start(session.Context(ctx), "FinishBytesOnly") + defer span.End() + + sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) + if err != nil { + return err + } + + if session.info.MetaData["checksum"] != "" { + parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1": + err = checkHash(parts[1], sha1h) + case "md5": + err = checkHash(parts[1], md5h) + case "adler32": + err = checkHash(parts[1], adler32h) + default: + err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if err != nil { + session.Cleanup(false, true, true, false) + return err + } + } + + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + // FinishUpload finishes an upload and moves the file to the internal destination // implements tusd.DataStore interface // returns tusd errors diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 0187e795242..3e4d9eede26 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -3,7 +3,6 @@ package decomposedfs import ( "bytes" "context" - "io" "os" "path/filepath" @@ -11,6 +10,7 @@ import ( cs3permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + tusd "github.com/tus/tusd/v2/pkg/handler" ruser "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/events/stream" @@ -20,7 +20,6 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/aspects" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/permissions" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/permissions/mocks" @@ -119,6 +118,21 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(item.Path).To(Equal(ref.Path)) return len(resources) == 1, utils.ReadPlainFromOpaque(item.Opaque, "status"), int(item.GetSize()) } + // tusUpload writes content via the coordinator TUS path (GetUpload → + // WriteChunk → FinishUpload) instead of the legacy fs.Upload path. + // This is needed because coordinator.InitiateUpload calls TouchFile, so + // the node already exists; FinishUploadDecomposed (legacy path) would + // try to create it again and fail with EEXIST. + tusUpload = func(id string, content []byte) { + ds, ok := fs.(tusd.DataStore) + Expect(ok).To(BeTrue(), "fs must implement tusd.DataStore") + up, err := ds.GetUpload(ctx, id) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(content)) + Expect(err).ToNot(HaveOccurred()) + Expect(up.FinishUpload(ctx)).To(Succeed()) + } + parentSize = func() int { parentInfo, err := fs.GetMD(ctx, rootRef, []string{}, []string{}) Expect(err).ToNot(HaveOccurred()) @@ -197,14 +211,8 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(err).ToNot(HaveOccurred()) ref.ResourceId = &resID - bs.On("Upload", mock.AnythingOfType("*node.Node"), mock.AnythingOfType("string"), mock.Anything). - Return(nil). - Run(func(args mock.Arguments) { - n := args.Get(0).(*node.Node) - data, err := os.ReadFile(args.Get(1).(string)) - Expect(err).ToNot(HaveOccurred()) - Expect(len(data)).To(Equal(int(n.Blobsize))) - }) + bs.On("UploadFromReader", mock.AnythingOfType("*node.Node"), mock.Anything, mock.AnythingOfType("int64")). + Return(nil) // start upload of a file uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) @@ -213,23 +221,15 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(uploadIds["simple"]).ToNot(BeEmpty()) Expect(uploadIds["tus"]).ToNot(BeEmpty()) - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(firstContent)), - Length: int64(len(firstContent)), - }, nil) - Expect(err).ToNot(HaveOccurred()) - uploadID = uploadIds["simple"] + tusUpload(uploadID, firstContent) // wait for bytes received event _, ok := (<-pub).(events.BytesReceived) Expect(ok).To(BeTrue()) // blobstore not called yet - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) }) AfterEach(func() { @@ -254,7 +254,7 @@ var _ = Describe("Async file uploads", Ordered, func() { succeedPostprocessing(uploadID) // blobstore called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) // node ready resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -284,7 +284,7 @@ var _ = Describe("Async file uploads", Ordered, func() { failPostprocessing(uploadID, events.PPOutcomeDelete) // blobstore still not called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) // node gone resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -313,7 +313,7 @@ var _ = Describe("Async file uploads", Ordered, func() { failPostprocessing(uploadID, events.PPOutcomeAbort) // blobstore still not called now - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 0) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 0) // node gone resources, err = fs.ListFolder(ctx, rootRef, []string{}, []string{}) @@ -342,28 +342,21 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(uploadIds["simple"]).ToNot(BeEmpty()) Expect(uploadIds["tus"]).ToNot(BeEmpty()) - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(firstContent)), - Length: int64(len(firstContent)), - }, nil) - Expect(err).ToNot(HaveOccurred()) - uploadID = uploadIds["simple"] + tusUpload(uploadID, firstContent) // wait for bytes received event _, ok := (<-pub).(events.BytesReceived) Expect(ok).To(BeTrue()) - // version already created + // version is not yet created at this point — it will be created when + // CommitUpload runs on PostprocessingFinished, not at InitiateUpload time. revs, err = fs.ListRevisions(ctx, ref) Expect(err).To(BeNil()) - Expect(len(revs)).To(Equal(1)) + Expect(len(revs)).To(Equal(0)) // at this stage: blobstore called once for the original file - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) }) @@ -376,7 +369,7 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(len(revs)).To(Equal(1)) // blobstore now called twice - for original file and new version - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 2) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 2) // bytes are gone from upload path _, err = os.Stat(filepath.Join(o.Root, "uploads", uploadID)) @@ -402,288 +395,110 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(err).ToNot(BeNil()) // blobstore still called only once for the original file - bs.AssertNumberOfCalls(GinkgoT(), "Upload", 1) + bs.AssertNumberOfCalls(GinkgoT(), "UploadFromReader", 1) }) }) - When("Two uploads are processed in parallel", func() { - var secondUploadID string - - JustBeforeEach(func() { - // upload again + When("a second upload is attempted while the first is still in postprocessing", func() { + // The coordinator serializes uploads via MarkProcessing: a second FinishUpload + // call while the first session holds the processing slot returns ResourceProcessing + // and the second session is cleaned up immediately. + + It("rejects the second FinishUpload with ResourceProcessing", func() { + // First upload is in postprocessing (BytesReceived consumed in BeforeEach). + // Initiate a second upload. uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).ToNot(HaveOccurred()) - Expect(len(uploadIds)).To(Equal(2)) - Expect(uploadIds["simple"]).ToNot(BeEmpty()) - Expect(uploadIds["tus"]).ToNot(BeEmpty()) + secondUploadID := uploadIds["simple"] - uploadRef := &provider.Reference{Path: "/" + uploadIds["simple"]} - - _, err = fs.Upload(ctx, storage.UploadRequest{ - Ref: uploadRef, - Body: io.NopCloser(bytes.NewReader(secondContent)), - Length: int64(len(secondContent)), - }, nil) + // Write bytes for the second upload. + ds, ok := fs.(tusd.DataStore) + Expect(ok).To(BeTrue()) + up, err := ds.GetUpload(ctx, secondUploadID) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) Expect(err).ToNot(HaveOccurred()) - secondUploadID = uploadIds["simple"] - - // wait for bytes received event - _, ok := (<-pub).(events.BytesReceived) - Expect(ok).To(BeTrue()) + // FinishUpload must fail because the node is already processing. + err = up.FinishUpload(ctx) + Expect(err).To(HaveOccurred()) + _, isProcessing := err.(interface{ IsResourceProcessing() bool }) + _ = isProcessing + Expect(err.Error()).To(ContainSubstring("resource is processing")) }) - It("doesn't remove processing status when first upload is finished", func() { - succeedPostprocessing(uploadID) + It("cleans up the rejected session's bin and info files", func() { + uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID := uploadIds["simple"] - _, status, _ := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - }) + ds, ok := fs.(tusd.DataStore) + Expect(ok).To(BeTrue()) + up, err := ds.GetUpload(ctx, secondUploadID) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) + Expect(err).ToNot(HaveOccurred()) + _ = up.FinishUpload(ctx) // expect failure; error checked in other test - It("removes processing status when second upload is finished, even if first isn't", func() { - succeedPostprocessing(secondUploadID) + // Bin file must be removed. + _, statErr := os.Stat(filepath.Join(o.Root, "uploads", secondUploadID)) + Expect(statErr).ToNot(BeNil(), "bin file should be cleaned up after rejection") - _, status, _ := fileStatus() - Expect(status).To(Equal("")) + // Info file must be removed. + _, statErr = os.Stat(filepath.Join(o.Root, "uploads", secondUploadID+".info")) + Expect(statErr).ToNot(BeNil(), "info file should be cleaned up after rejection") }) + }) - It("correctly calculates the size when the second upload is finished, even if first is deleted", func() { - succeedPostprocessing(secondUploadID) - - _, status, size := fileStatus() - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(uploadID, events.PPOutcomeDelete) - - // check processing status - _, _, size = fileStatus() - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - }) + When("uploads are processed sequentially (second after first completes)", func() { + var secondUploadID string - It("the first can succeed before the second succeeds", func() { + JustBeforeEach(func() { + // Complete the first upload's postprocessing before starting the second. succeedPostprocessing(uploadID) - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal((len(secondContent)))) - - // parent size should match the second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(secondUploadID) - - // check processing status has been removed - _, status, size = fileStatus() - Expect(status).To(Equal("")) - - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) + // Second upload. + uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID = uploadIds["simple"] + tusUpload(secondUploadID, secondContent) - // file should have one revision - Expect(revisionCount()).To(Equal(1)) + // wait for bytes received event + _, ok := (<-pub).(events.BytesReceived) + Expect(ok).To(BeTrue()) }) - It("the first can succeed after the second succeeds", func() { + It("succeeds and creates a revision", func() { succeedPostprocessing(secondUploadID) _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(uploadID) - - _, status, size = fileStatus() - // check processing status is still unset Expect(status).To(Equal("")) - // size should still match the second upload Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload Expect(parentSize()).To(Equal(len(secondContent))) - - // file should have one revision Expect(revisionCount()).To(Equal(1)) }) - It("the first can succeed before the second fails", func() { - succeedPostprocessing(uploadID) - - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match the second upload - Expect(parentSize()).To(Equal(len(secondContent))) - + It("reverts to previous content when second upload is deleted", func() { failPostprocessing(secondUploadID, events.PPOutcomeDelete) - _, status, size = fileStatus() - // check processing status has been removed - Expect(status).To(Equal("")) - // size should match the first upload - Expect(size).To(Equal(len(firstContent))) - - // parent size should match first upload - Expect(parentSize()).To(Equal(len(firstContent))) - - // file should not have any revisions - Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can succeed after the second fails", func() { - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - _, _, size := fileStatus() - // check processing status has not been unset - // FIXME we need to fall back to the previous processing id - // Expect(status).To(Equal("processing")) - // size should match the first upload - Expect(size).To(Equal(len(firstContent))) - - // parent size should match first upload as well - Expect(parentSize()).To(Equal(len(firstContent))) - - succeedPostprocessing(uploadID) - _, status, size := fileStatus() - // check processing status is now unset Expect(status).To(Equal("")) - // size should still match the first upload Expect(size).To(Equal(len(firstContent))) - - // parent size should still match first upload Expect(parentSize()).To(Equal(len(firstContent))) - - // file should not have any revisions Expect(revisionCount()).To(Equal(0)) }) - It("the first can fail before the second succeeds", func() { - failPostprocessing(uploadID, events.PPOutcomeDelete) + It("reverts to previous content when second upload is aborted and keeps bin", func() { + failPostprocessing(secondUploadID, events.PPOutcomeAbort) _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - succeedPostprocessing(secondUploadID) - - _, status, size = fileStatus() - // check processing status has been removed Expect(status).To(Equal("")) - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - // file should not have any revisions - // FIXME we need to delete the revision - // Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can fail after the second succeeds", func() { - succeedPostprocessing(secondUploadID) - - _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded - Expect(status).To(Equal("")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(uploadID, events.PPOutcomeDelete) - - _, status, size = fileStatus() - // check processing status is still unset - Expect(status).To(Equal("")) - // size should still match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should still match second upload - Expect(parentSize()).To(Equal(len(secondContent))) - - // file should not have any revisions - // FIXME we need to delete the revision - // Expect(revisionCount()).To(Equal(0)) - }) - - It("the first can fail before the second fails", func() { - failPostprocessing(uploadID, events.PPOutcomeDelete) - - _, status, size := fileStatus() - // check processing status - Expect(status).To(Equal("processing")) - // size should match the second upload - Expect(size).To(Equal(len(secondContent))) - - // parent size should match second upload as well - Expect(parentSize()).To(Equal(len(secondContent))) - - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - // check file has been removed - // if all uploads have been processed with outcome delete -> delete the file - // exists, _, _ := fileStatus() - // FIXME this should be false, but we are not deleting the resource - // Expect(exists).To(BeFalse()) - - // parent size should be 0 - // FIXME we are not correctly reverting the sizediff - // Expect(parentSize()).To(Equal(0)) - }) - - It("the first can fail after the second fails", func() { - failPostprocessing(secondUploadID, events.PPOutcomeDelete) - - _, status, size := fileStatus() - // check processing status has been removed because the most recent upload finished and can be downloaded - Expect(status).To(Equal("")) - // size should match the first upload Expect(size).To(Equal(len(firstContent))) - - // parent size should match second first as well Expect(parentSize()).To(Equal(len(firstContent))) - failPostprocessing(uploadID, events.PPOutcomeDelete) - - // check file has been removed - // if all uploads have been processed with outcome delete -> delete the file - // exists, _, _ := fileStatus() - // FIXME this should be false, but we are not deleting the resource - // Expect(exists).To(BeFalse()) - - // parent size should be 0 - // FIXME we are not correctly reverting the sizediff - // Expect(parentSize()).To(Equal(0)) + // bytes kept + _, statErr := os.Stat(filepath.Join(o.Root, "uploads", secondUploadID)) + Expect(statErr).To(BeNil()) }) }) }) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 9098231e364..87b16b675f3 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -23,8 +23,13 @@ package upload import ( + "bytes" "context" + "fmt" + "io" + "os" "path/filepath" + "strings" "time" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -35,10 +40,12 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/owncloud/reva/v2/pkg/autoprop" + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -75,10 +82,85 @@ type Session interface { Checksums() storage.UploadChecksums Metadata() map[string]string Persist(ctx context.Context) error - Finalize(ctx context.Context) error + FinishBytesOnly(ctx context.Context) error Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) Context(ctx context.Context) context.Context - Node(ctx context.Context) (*node.Node, error) + // Typed setters used by Coordinator.InitiateUpload to populate a new session + // without knowing internal storage key names. + SetStorageValue(key, value string) + SetMetadata(key, value string) + SetSize(size int64) + SetSizeIsDeferred(value bool) + SetExecutant(u *user.User) + TouchBin() error +} + + +// bytesFinisher is implemented by OcisSession to compute checksums without +// touching the node. The coordinator uses it from coordinatedUpload.FinishUpload. +type bytesFinisher interface { + FinishBytesOnly(ctx context.Context) error +} + +// coordinatedUpload wraps a Session so the TUS FinishUpload hook runs the +// coordinator path (checksums + MarkProcessing + BytesReceived) instead of +// the legacy FinishUploadDecomposed path. +type coordinatedUpload struct { + tusd.Upload + session Session + coord *Coordinator +} + +func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { + if bf, ok := u.session.(bytesFinisher); ok { + if err := bf.FinishBytesOnly(ctx); err != nil { + return err + } + // Persist checksums to disk so the postprocessing handler can read them + // from the session store after the BytesReceived event is processed. + if err := u.session.Persist(ctx); err != nil { + return err + } + } + ref := u.session.Reference() + if err := u.coord.FS.MarkProcessing(ctx, &ref, true, u.session.ID()); err != nil { + // Slot already owned by another session. Clean up this session's bin and + // info files — they will never reach postprocessing. + u.session.Cleanup(false, true, true, false) + return err + } + + metrics.UploadProcessing.Inc() + metrics.UploadSessionsBytesReceived.Inc() + + if u.coord.pub != nil && u.session.Size() > 0 { + s, err := u.session.URL(ctx) + if err != nil { + return err + } + if err := events.Publish(ctx, u.coord.pub, events.BytesReceived{ + UploadID: u.session.ID(), + URL: s, + SpaceOwner: u.session.SpaceOwner(), + ExecutingUser: &user.User{ + Id: &user.UserId{ + Type: u.session.Executant().Type, + Idp: u.session.Executant().Idp, + OpaqueId: u.session.Executant().OpaqueId, + }, + }, + ResourceID: &provider.ResourceId{ + StorageId: u.session.ProviderID(), + SpaceId: u.session.SpaceID(), + OpaqueId: u.session.NodeID(), + }, + Filename: u.session.Filename(), + Filesize: uint64(u.session.Size()), + }); err != nil { + return err + } + } + return nil } // SessionStore abstracts upload-session persistence for the Coordinator. @@ -94,14 +176,6 @@ type RevisionReverter interface { RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error } -// SizePropagator is an optional interface a storage driver may implement to -// propagate a reverted size delta up the directory tree after a failed upload. -// Decomposedfs implements it. Drivers whose tree accounting is server-side do -// not need to implement this. -type SizePropagator interface { - PropagateRevertedSize(ctx context.Context, id *provider.ResourceId, sizeDiff int64) error -} - // Coordinator owns the upload state machine: // - TUS HTTP layer (InitiateUpload, UseIn, GetUpload, ListUploadSessions) // - postprocessing event loop (PostprocessingFinished, PostprocessingStepFinished, @@ -206,14 +280,10 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event ctx = session.Context(ctx) - n, err := session.Node(ctx) - if err != nil { - log.Error().Err(err).Msg("could not read node") - return - } log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - if !n.Exists { - log.Debug().Msg("node no longer exists") + ref := session.Reference() + if _, err := c.FS.GetMD(ctx, &ref, []string{}, []string{}); err != nil { + log.Debug().Err(err).Msg("node no longer exists or not accessible; cleaning up") session.Cleanup(false, true, true, false) return } @@ -231,50 +301,46 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event fallthrough case events.PPOutcomeAbort: failed = true - revertNodeMetadata = true + // Only revert node metadata for new files: for overwrites, CommitUpload + // never ran so the node still holds the previous content — nothing to undo. + revertNodeMetadata = !session.NodeExists() keepUpload = true metrics.UploadSessionsAborted.Inc() case events.PPOutcomeContinue: - if err := session.Finalize(ctx); err != nil { - log.Error().Err(err).Msg("could not finalize upload") + f, fopenErr := os.Open(session.BinPath()) + if fopenErr != nil { + log.Error().Err(fopenErr).Msg("could not open staged binary for CommitUpload") failed = true revertNodeMetadata = false keepUpload = true unmarkPostprocessing = false } else { - metrics.UploadSessionsFinalized.Inc() + ref := session.Reference() + _, commitErr := c.FS.CommitUpload(ctx, &ref, storage.UploadSource{ + Body: f, + Length: session.Size(), + Metadata: session.Metadata(), + Checksums: session.Checksums(), + }) + if commitErr != nil { + log.Error().Err(commitErr).Msg("could not commit upload") + failed = true + revertNodeMetadata = false + keepUpload = true + unmarkPostprocessing = false + } else { + metrics.UploadSessionsFinalized.Inc() + } } case events.PPOutcomeDelete: failed = true - revertNodeMetadata = true + // Only revert node metadata for new files: for overwrites, CommitUpload + // never ran so the node still holds the previous content — nothing to undo. + revertNodeMetadata = !session.NodeExists() metrics.UploadSessionsDeleted.Inc() } now := time.Now() - if failed { - // Propagate the reverted size so parent tree counters stay correct. - // This is only needed when the session still owns the processing slot - // (i.e. no later upload has taken over). The check and the propagation - // are both delegated to the driver via SizePropagator so that external - // drivers (which do tree accounting server-side) can no-op this. - latestSession, err := n.ProcessingID(ctx) - if err != nil { - log.Error().Err(err).Msg("reading node processingID failed") - } else if latestSession == session.ID() { - if sp, ok := c.FS.(SizePropagator); ok { - if err := sp.PropagateRevertedSize(ctx, session.Reference().ResourceId, -session.SizeDiff()); err != nil { - log.Error().Err(err).Msg("could not propagate reverted size") - } - } - } - } else { - // Finalize writes the blob but does not bump parent tmtime; do it here - // so etag changes propagate to folder listings after async uploads. - p, perr := n.Parent(ctx) - if perr == nil && p != nil { - _ = p.SetTMTime(ctx, &now) - } - } session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) @@ -308,7 +374,7 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event OpaqueId: session.NodeID(), }, Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), + SpaceOwner: session.SpaceOwner(), IsVersion: isVersion, ImpersonatingUser: ev.ImpersonatingUser, }, @@ -324,11 +390,6 @@ func (c *Coordinator) handleRestartPostprocessing(ctx context.Context, ev events log.Error().Err(err).Msg("Failed to get upload") return } - n, err := session.Node(ctx) - if err != nil { - log.Error().Err(err).Msg("could not read node") - return - } log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() s, err := session.URL(ctx) if err != nil { @@ -341,11 +402,14 @@ func (c *Coordinator) handleRestartPostprocessing(ctx context.Context, ev events if err := events.Publish(ctx, c.pub, events.BytesReceived{ UploadID: session.ID(), URL: s, - SpaceOwner: n.SpaceOwnerOrManager(ctx), + SpaceOwner: session.SpaceOwner(), ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), + ResourceID: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), }); err != nil { log.Error().Err(err).Msg("Failed to publish BytesReceived event") } @@ -403,12 +467,6 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e log.Error().Err(err).Msg("Failed to get upload") return } - - n, err := session.Node(ctx) - if err != nil { - log.Error().Err(err).Msg("Failed to get node after scan") - return - } log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() session.SetScanData(res.Description, res.Scandate) @@ -416,17 +474,184 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e log.Error().Err(err).Msg("Failed to persist scan results") } - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - log.Error().Err(err).Msg("Failed to set scan results") + // TODO(OCISDEV-900): other callers of node.SetScanData may need the same + // refactor — scanning concerns should likely move out of decomposedfs entirely. + ref := session.Reference() + if err := c.FS.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + prefixes.ScanStatusPrefix: res.Description, + prefixes.ScanDatePrefix: res.Scandate.Format(time.RFC3339Nano), + }, + }); err != nil { + log.Error().Err(err).Msg("Failed to set scan results on node") return } metrics.UploadSessionsScanned.Inc() } -// InitiateUpload delegates to the underlying storage.FS. +// InitiateUpload creates a node placeholder via TouchFile and builds an upload +// session. For new files TouchFile creates the node; for overwrites it already +// exists and we skip it. All session fields are populated via typed setters so +// the coordinator has no knowledge of internal storage key names. func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - return c.FS.InitiateUpload(ctx, ref, uploadLength, metadata) + // Resolve node metadata: determine whether the target exists, get its ID, + // parent ID, space ID, space owner, and the path for Dir. + existing, err := c.FS.GetMD(ctx, ref, []string{}, []string{}) + nodeExists := err == nil + + mtime := "" + if m, ok := metadata["mtime"]; ok && m != "null" { + mtime = m + } + + var nodeID, spaceID, parentID, dir, nodeName string + var spaceOwner *user.UserId + + if nodeExists { + // Overwrite: node is already there; TouchFile is a no-op for existing nodes. + nodeID = existing.GetId().GetOpaqueId() + spaceID = existing.GetId().GetSpaceId() + parentID = existing.GetParentId().GetOpaqueId() + dir = filepath.Dir(existing.GetPath()) + nodeName = existing.GetName() + // SpaceOwner is not on ResourceInfo directly; read from a space listing + // or fall back to the owner field (which is the node owner, not space owner). + // The session field is used to populate BytesReceived / UploadReady events. + spaceOwner = existing.GetOwner() + } else { + // New file: create the placeholder node via TouchFile. + result, tfErr := c.FS.TouchFile(ctx, ref, false, mtime) + if tfErr != nil { + return nil, tfErr + } + nodeID = result.ResourceID.GetOpaqueId() + spaceID = result.SpaceID + spaceOwner = result.SpaceOwner + // Derive dir and name from the ref path — ref must carry a path for new files. + dir = filepath.Dir(ref.GetPath()) + nodeName = filepath.Base(ref.GetPath()) + // Parent ResourceId is not returned by TouchFile; derive from GetMD on the parent. + parentRef := &provider.Reference{ + ResourceId: ref.ResourceId, + Path: filepath.Dir(ref.GetPath()), + } + if parentInfo, pErr := c.FS.GetMD(ctx, parentRef, []string{}, []string{}); pErr == nil { + parentID = parentInfo.GetId().GetOpaqueId() + } + } + + if nodeName == "" { + return nil, errtypes.BadRequest("coordinator: missing filename in ref") + } + if dir == "" { + return nil, errtypes.BadRequest("coordinator: could not determine upload directory") + } + + session := c.store.New(ctx) + session.SetMetadata("filename", nodeName) + session.SetStorageValue("NodeName", nodeName) + session.SetMetadata("dir", dir) + session.SetStorageValue("Dir", dir) + session.SetStorageValue("NodeId", nodeID) + session.SetStorageValue("SpaceRoot", spaceID) + if nodeExists { + session.SetStorageValue("NodeExists", "true") + } + session.SetStorageValue("NodeParentId", parentID) + if spaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", spaceOwner.GetOpaqueId()) + } + + usr := ctxpkg.ContextMustGetUser(ctx) + session.SetExecutant(usr) + + lockID, _ := ctxpkg.ContextGetLockID(ctx) + session.SetMetadata("lockid", lockID) + + iid, _ := ctxpkg.ContextGetInitiator(ctx) + session.SetMetadata("initiatorid", iid) + + session.SetSize(uploadLength) + + var mtimeSet bool + if metadata != nil { + session.SetMetadata("providerID", metadata["providerID"]) + if v, ok := metadata["mtime"]; ok && v != "null" { + session.SetMetadata("mtime", v) + mtimeSet = true + } + if v, ok := metadata["expires"]; ok && v != "null" { + session.SetMetadata("expires", v) + } + if _, ok := metadata["sizedeferred"]; ok { + session.SetSizeIsDeferred(true) + } + if checksum, ok := metadata["checksum"]; ok { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1", "md5", "adler32": + session.SetMetadata("checksum", checksum) + default: + return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + } + if v := metadata["if-match"]; v != "" { + session.SetMetadata("if-match", v) + } + if v := metadata["if-none-match"]; v != "" { + session.SetMetadata("if-none-match", v) + } + if v := metadata["if-unmodified-since"]; v != "" { + session.SetMetadata("if-unmodified-since", v) + } + } + + if !mtimeSet { + session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now())) + } + + if err := session.TouchBin(); err != nil { + return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) + } + if err := session.Persist(ctx); err != nil { + return nil, fmt.Errorf("coordinator: could not persist session: %w", err) + } + + metrics.UploadSessionsInitiated.Inc() + + if uploadLength == 0 { + // Zero-length uploads complete immediately: compute checksums on the empty + // bin, commit, and clean up — no postprocessing needed. + if err := session.FinishBytesOnly(ctx); err != nil { + session.Cleanup(false, true, true, false) + return nil, fmt.Errorf("coordinator: zero-length FinishBytesOnly: %w", err) + } + commitRef := session.Reference() + if _, err := c.FS.CommitUpload(ctx, &commitRef, storage.UploadSource{ + Body: io.NopCloser(bytes.NewReader(nil)), + Length: 0, + Metadata: session.Metadata(), + Checksums: session.Checksums(), + }); err != nil { + session.Cleanup(false, true, true, false) + return nil, fmt.Errorf("coordinator: zero-length CommitUpload: %w", err) + } + session.Cleanup(false, true, true, false) + metrics.UploadSessionsFinalized.Inc() + return map[string]string{ + "simple": session.ID(), + "tus": session.ID(), + }, nil + } + + return map[string]string{ + "simple": session.ID(), + "tus": session.ID(), + }, nil } // UseIn registers the coordinator as the TUS data store in the composer. @@ -442,9 +667,14 @@ func (c *Coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload return nil, errNotImplemented } -// GetUpload returns the upload session for the given id. +// GetUpload returns the upload session wrapped in a coordinatedUpload so the +// TUS FinishUpload hook runs the coordinator path rather than the legacy one. func (c *Coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - return c.store.Get(ctx, id) + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + return &coordinatedUpload{Upload: session, session: session, coord: c}, nil } // ListUploadSessions returns upload sessions matching the given filter. From 65727c51442a5471096d883840046d8a7677e777 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 1 Jul 2026 11:32:39 +0200 Subject: [PATCH 04/20] feat(OCISDEV-900): wire coordinator from storageprovider/dataprovider, not decomposedfs --- .../storageprovider/storageprovider.go | 41 +++++++++++++------ .../services/dataprovider/dataprovider.go | 32 +++++++++------ .../utils/decomposedfs/decomposedfs.go | 27 ++++-------- .../utils/decomposedfs/upload_async_test.go | 7 +++- pkg/upload/coordinator.go | 8 ++++ 5 files changed, 67 insertions(+), 48 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 0a121fc3efe..23233f46cd4 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -47,6 +47,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -81,6 +82,8 @@ type eventconfig struct { EnableTLS bool `mapstructure:"nats_enable_tls" docs:"events tls switch"` AuthUsername string `mapstructure:"nats_username" docs:"event stream username"` AuthPassword string `mapstructure:"nats_password" docs:"event stream password"` + ConsumerGroup string `mapstructure:"consumer_group" docs:"dcfs;Consumer group for the upload coordinator"` + NumConsumers int `mapstructure:"numconsumers" docs:"1;Number of concurrent postprocessing event consumers"` } func (c *config) init() { @@ -101,16 +104,16 @@ func (c *config) init() { if len(c.AvailableXS) == 0 { c.AvailableXS = map[string]uint32{"md5": 100, "unset": 1000} } -} -// uploadCoordinatorProvider is satisfied by storage drivers that can vend a -// ready-to-use upload coordinator (e.g. decomposedfs). Using a local interface -// avoids importing the decomposedfs or pkg/upload packages here, which would -// create an import cycle through decomposedfs/node → storageprovider. -type uploadCoordinatorProvider interface { - UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) + if c.Events.ConsumerGroup == "" { + c.Events.ConsumerGroup = "storageprovider" + } + if c.Events.NumConsumers <= 0 { + c.Events.NumConsumers = 1 + } } + type Service struct { conf *config Storage storage.FS @@ -211,17 +214,21 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } - // Build and start the upload coordinator if the driver supports it. + // Build and start the upload coordinator if the driver exposes a session store. var coordinator storage.FS - if cp, ok := fs.(uploadCoordinatorProvider); ok { + if sp, ok := fs.(pkgupload.UploadSessionStoreProvider); ok { evstream, err := estreamFromConfig(c.Events) if err != nil { return nil, err } - coordinator, err = cp.UploadCoordinator(evstream, log) - if err != nil { - return nil, err + coord := pkgupload.NewCoordinator(fs, sp.UploadSessionStore(), evstream, + c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log) + if evstream != nil { + if err := coord.Start(evstream); err != nil { + return nil, err + } } + coordinator = coord } service := &Service{ @@ -1326,7 +1333,15 @@ func estreamFromConfig(c eventconfig) (events.Stream, error) { return nil, nil } - return stream.NatsFromConfig("storageprovider", false, stream.NatsConfig(c)) + return stream.NatsFromConfig("storageprovider", false, stream.NatsConfig{ + Endpoint: c.Endpoint, + Cluster: c.Cluster, + TLSInsecure: c.TLSInsecure, + TLSRootCACertificate: c.TLSRootCACertificate, + EnableTLS: c.EnableTLS, + AuthUsername: c.AuthUsername, + AuthPassword: c.AuthPassword, + }) } func canLockPublicShare(ctx context.Context) bool { diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 774a1a1e4d1..df8434396ba 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -33,15 +33,9 @@ import ( "github.com/owncloud/reva/v2/pkg/rhttp/router" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/fs/registry" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" ) -// uploadCoordinatorProvider is satisfied by storage drivers that can vend a -// ready-to-use upload coordinator. Using a local interface avoids importing -// decomposedfs, which would create an import cycle. -type uploadCoordinatorProvider interface { - UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) -} - func init() { global.Register("dataprovider", New) } @@ -51,6 +45,7 @@ type config struct { Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go;The configuration for the storage driver"` DataTXs map[string]map[string]interface{} `mapstructure:"data_txs" docs:"url:pkg/rhttp/datatx/manager/simple/simple.go;The configuration for the data tx protocols"` + MountID string `mapstructure:"mount_id"` NatsAddress string `mapstructure:"nats_address"` NatsClusterID string `mapstructure:"nats_clusterID"` NatsTLSInsecure bool `mapstructure:"nats_tls_insecure"` @@ -58,6 +53,8 @@ type config struct { NatsEnableTLS bool `mapstructure:"nats_enable_tls"` NatsUsername string `mapstructure:"nats_username"` NatsPassword string `mapstructure:"nats_password"` + ConsumerGroup string `mapstructure:"consumer_group"` + NumConsumers int `mapstructure:"numconsumers"` } func (c *config) init() { @@ -67,6 +64,12 @@ func (c *config) init() { if c.Driver == "" { c.Driver = "localhome" } + if c.ConsumerGroup == "" { + c.ConsumerGroup = "storageprovider" + } + if c.NumConsumers <= 0 { + c.NumConsumers = 1 + } } type svc struct { @@ -111,15 +114,18 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - // Wrap the FS in the upload coordinator if the driver supports it. + // Wrap the FS in the upload coordinator if the driver exposes a session store. // The coordinator IS-A storage.FS, so it passes unchanged to getDataTXs. txFS := storage.FS(fs) - if cp, ok := fs.(uploadCoordinatorProvider); ok { - coordinator, err := cp.UploadCoordinator(evstream, log) - if err != nil { - return nil, err + if sp, ok := fs.(pkgupload.UploadSessionStoreProvider); ok { + coord := pkgupload.NewCoordinator(fs, sp.UploadSessionStore(), evstream, + conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log) + if evstream != nil { + if err := coord.Start(evstream); err != nil { + return nil, err + } } - txFS = coordinator + txFS = coord } dataTXs, err := getDataTXs(conf, txFS, evstream, log) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 2e39e3f75ed..d8712504c8f 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -267,9 +267,9 @@ func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.R return n.RevertCurrentRevision(ctx) } -// ocisSessionStoreAdapter adapts *upload.OcisStore to pkgupload.SessionStore, -// bridging the concrete *OcisSession return types to the generic Session interface. -type ocisSessionStoreAdapter struct{ s *upload.OcisStore } +// ocisSessionStoreAdapter adapts the decomposedfs SessionStore (which returns +// concrete *OcisSession) to the generic pkgupload.SessionStore interface. +type ocisSessionStoreAdapter struct{ s SessionStore } func (a ocisSessionStoreAdapter) New(ctx context.Context) pkgupload.Session { return a.s.New(ctx) @@ -289,23 +289,10 @@ func (a ocisSessionStoreAdapter) List(ctx context.Context) ([]pkgupload.Session, return result, nil } -// UploadCoordinator constructs, wires, and starts a driver-agnostic upload -// coordinator for this decomposedfs instance. The returned storage.FS embeds -// decomposedfs and overrides the upload-related methods. Callers (storageprovider, -// dataprovider) only interact with storage.FS — they need not import pkg/upload -// or decomposedfs directly. -func (fs *Decomposedfs) UploadCoordinator(stream events.Stream, log *zerolog.Logger) (storage.FS, error) { - var store pkgupload.SessionStore - if ocisStore, ok := fs.sessionStore.(*upload.OcisStore); ok { - store = ocisSessionStoreAdapter{s: ocisStore} - } - c := pkgupload.NewCoordinator(fs, store, fs.stream, fs.o.MountID, fs.o.Events.ConsumerGroup, fs.o.Events.NumConsumers, log) - if stream != nil { - if err := c.Start(stream); err != nil { - return nil, err - } - } - return c, nil +// UploadSessionStore implements pkgupload.UploadSessionStoreProvider so that +// storageprovider and dataprovider can construct the Coordinator themselves. +func (fs *Decomposedfs) UploadSessionStore() pkgupload.SessionStore { + return ocisSessionStoreAdapter{s: fs.sessionStore} } // GetQuota returns the quota available diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 3e4d9eede26..0f397e6b710 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -28,6 +28,7 @@ import ( treemocks "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/tree/mocks" "github.com/owncloud/reva/v2/pkg/storagespace" "github.com/owncloud/reva/v2/pkg/store" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" "github.com/owncloud/reva/v2/tests/helpers" "github.com/rs/zerolog" @@ -201,8 +202,10 @@ var _ = Describe("Async file uploads", Ordered, func() { dfs, err := New(o, aspects, &zerolog.Logger{}) Expect(err).ToNot(HaveOccurred()) // Wire the coordinator so postprocessing events are consumed during tests. - fs, err = dfs.(*Decomposedfs).UploadCoordinator(aspects.EventStream, &zerolog.Logger{}) - Expect(err).ToNot(HaveOccurred()) + d := dfs.(*Decomposedfs) + coord := pkgupload.NewCoordinator(d, d.UploadSessionStore(), aspects.EventStream, "", "dcfs", 1, &zerolog.Logger{}) + Expect(coord.Start(aspects.EventStream)).To(Succeed()) + fs = coord resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 87b16b675f3..20dda0c8316 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -176,6 +176,14 @@ type RevisionReverter interface { RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error } +// UploadSessionStoreProvider is implemented by storage drivers that own an +// upload session store. storageprovider and dataprovider use this to obtain +// the SessionStore when constructing the Coordinator, so the Coordinator is +// wired entirely outside the driver. +type UploadSessionStoreProvider interface { + UploadSessionStore() SessionStore +} + // Coordinator owns the upload state machine: // - TUS HTTP layer (InitiateUpload, UseIn, GetUpload, ListUploadSessions) // - postprocessing event loop (PostprocessingFinished, PostprocessingStepFinished, From fac7359bbe87402bd665c385f3768cd7d1faf1f4 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 1 Jul 2026 16:54:13 +0200 Subject: [PATCH 05/20] feat(OCISDEV-900): replace UploadAborter with Delete; write AV scan results via SetArbitraryMetadata --- .../utils/decomposedfs/decomposedfs.go | 29 --------- pkg/storage/utils/decomposedfs/node/node.go | 17 +++-- .../utils/decomposedfs/upload_async_test.go | 4 +- pkg/upload/coordinator.go | 63 ++++++++++++------- 4 files changed, 54 insertions(+), 59 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index d8712504c8f..94452822416 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -58,7 +58,6 @@ import ( "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/timemanager" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/trashbin" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/tree" - pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/usermapper" "github.com/owncloud/reva/v2/pkg/storage/utils/filelocks" @@ -267,34 +266,6 @@ func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.R return n.RevertCurrentRevision(ctx) } -// ocisSessionStoreAdapter adapts the decomposedfs SessionStore (which returns -// concrete *OcisSession) to the generic pkgupload.SessionStore interface. -type ocisSessionStoreAdapter struct{ s SessionStore } - -func (a ocisSessionStoreAdapter) New(ctx context.Context) pkgupload.Session { - return a.s.New(ctx) -} -func (a ocisSessionStoreAdapter) Get(ctx context.Context, id string) (pkgupload.Session, error) { - return a.s.Get(ctx, id) -} -func (a ocisSessionStoreAdapter) List(ctx context.Context) ([]pkgupload.Session, error) { - sessions, err := a.s.List(ctx) - if err != nil { - return nil, err - } - result := make([]pkgupload.Session, len(sessions)) - for i, s := range sessions { - result[i] = s - } - return result, nil -} - -// UploadSessionStore implements pkgupload.UploadSessionStoreProvider so that -// storageprovider and dataprovider can construct the Coordinator themselves. -func (fs *Decomposedfs) UploadSessionStore() pkgupload.SessionStore { - return ocisSessionStoreAdapter{s: fs.sessionStore} -} - // GetQuota returns the quota available // TODO Document in the cs3 should we return quota or free space? func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, remaining uint64, err error) { diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 3f3dadb7352..25fa308f519 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -1283,9 +1283,16 @@ func (n *Node) SetScanData(ctx context.Context, info string, date time.Time) err return n.SetXattrsWithContext(ctx, attribs, true) } -// ScanData returns scanning information of the node +// ScanData returns scanning information of the node. +// It first checks the new arbitrary-metadata keys (written by the coordinator +// via SetArbitraryMetadata), then falls back to the legacy xattr keys so that +// nodes scanned before this change still return correct results. func (n *Node) ScanData(ctx context.Context) (scanned bool, virus string, scantime time.Time) { - ti, _ := n.XattrString(ctx, prefixes.ScanDatePrefix) + ti, _ := n.XattrString(ctx, prefixes.MetadataPrefix+"scandate") + if ti == "" { + // fall back to legacy xattr key written by the old decomposedfs path + ti, _ = n.XattrString(ctx, prefixes.ScanDatePrefix) + } if ti == "" { return // not scanned yet } @@ -1295,9 +1302,9 @@ func (n *Node) ScanData(ctx context.Context) (scanned bool, virus string, scanti return } - i, err := n.XattrString(ctx, prefixes.ScanStatusPrefix) - if err != nil { - return + i, _ := n.XattrString(ctx, prefixes.MetadataPrefix+"scanstatus") + if i == "" { + i, _ = n.XattrString(ctx, prefixes.ScanStatusPrefix) } return true, i, t diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 0f397e6b710..721c36ac7f7 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -186,6 +186,7 @@ var _ = Describe("Async file uploads", Ordered, func() { InitiateFileUpload: true, ListContainer: true, ListFileVersions: true, + Delete: true, }, nil) // setup fs @@ -203,7 +204,8 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(err).ToNot(HaveOccurred()) // Wire the coordinator so postprocessing events are consumed during tests. d := dfs.(*Decomposedfs) - coord := pkgupload.NewCoordinator(d, d.UploadSessionStore(), aspects.EventStream, "", "dcfs", 1, &zerolog.Logger{}) + fileStore := pkgupload.NewFileStore(o.Root, pkgupload.TokenOptions{}, &zerolog.Logger{}) + coord := pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, "", "dcfs", 1, &zerolog.Logger{}) Expect(coord.Start(aspects.EventStream)).To(Succeed()) fs = coord diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 20dda0c8316..d93953231c5 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -45,7 +45,6 @@ import ( "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -176,14 +175,6 @@ type RevisionReverter interface { RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error } -// UploadSessionStoreProvider is implemented by storage drivers that own an -// upload session store. storageprovider and dataprovider use this to obtain -// the SessionStore when constructing the Coordinator, so the Coordinator is -// wired entirely outside the driver. -type UploadSessionStoreProvider interface { - UploadSessionStore() SessionStore -} - // Coordinator owns the upload state machine: // - TUS HTTP layer (InitiateUpload, UseIn, GetUpload, ListUploadSessions) // - postprocessing event loop (PostprocessingFinished, PostprocessingStepFinished, @@ -300,8 +291,8 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event failed bool revertNodeMetadata bool keepUpload bool + retryCommit bool ) - unmarkPostprocessing := true switch ev.Outcome { default: @@ -319,12 +310,11 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event if fopenErr != nil { log.Error().Err(fopenErr).Msg("could not open staged binary for CommitUpload") failed = true - revertNodeMetadata = false keepUpload = true - unmarkPostprocessing = false + retryCommit = true } else { - ref := session.Reference() - _, commitErr := c.FS.CommitUpload(ctx, &ref, storage.UploadSource{ + commitRef := session.Reference() + _, commitErr := c.FS.CommitUpload(ctx, &commitRef, storage.UploadSource{ Body: f, Length: session.Size(), Metadata: session.Metadata(), @@ -333,9 +323,8 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event if commitErr != nil { log.Error().Err(commitErr).Msg("could not commit upload") failed = true - revertNodeMetadata = false keepUpload = true - unmarkPostprocessing = false + retryCommit = true } else { metrics.UploadSessionsFinalized.Inc() } @@ -350,7 +339,23 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event now := time.Now() - session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) + // Clean up bin and info files. Node reversion (for aborted new-file uploads) + // is handled below via the FS interface, not inside session.Cleanup. + session.Cleanup(false, !keepUpload, !keepUpload, false) + + nodeRef := session.Reference() + if !retryCommit { + if err := c.FS.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing after postprocessing finished") + } + if revertNodeMetadata { + if _, delErr := c.FS.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node on abort") + } + } + } + } var isVersion bool if session.NodeExists() { @@ -430,7 +435,19 @@ func (c *Coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUplo log.Error().Err(err).Msg("Failed to get upload") return } - session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) + ctx = session.Context(ctx) + session.Cleanup(false, !ev.KeepUpload, !ev.KeepUpload, false) + nodeRef := session.Reference() + if err := c.FS.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during CleanUpload") + } + if !session.NodeExists() { + if _, delErr := c.FS.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node during CleanUpload") + } + } + } } func (c *Coordinator) handleRevertRevision(ctx context.Context, ev events.RevertRevision) { @@ -482,17 +499,15 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e log.Error().Err(err).Msg("Failed to persist scan results") } - // TODO(OCISDEV-900): other callers of node.SetScanData may need the same - // refactor — scanning concerns should likely move out of decomposedfs entirely. + ctx = session.Context(ctx) ref := session.Reference() if err := c.FS.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ Metadata: map[string]string{ - prefixes.ScanStatusPrefix: res.Description, - prefixes.ScanDatePrefix: res.Scandate.Format(time.RFC3339Nano), + "scanstatus": res.Description, + "scandate": res.Scandate.Format(time.RFC3339Nano), }, }); err != nil { - log.Error().Err(err).Msg("Failed to set scan results on node") - return + log.Error().Err(err).Msg("Failed to write scan results to node") } metrics.UploadSessionsScanned.Inc() From 2c9fe046623ef952ee29c48f51504e3e82ad4368 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 1 Jul 2026 16:58:28 +0200 Subject: [PATCH 06/20] feat(OCISDEV-900): check quota in InitiateUpload before accepting bytes --- pkg/upload/coordinator.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index d93953231c5..87fc194dfeb 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -542,7 +542,35 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc // or fall back to the owner field (which is the node owner, not space owner). // The session field is used to populate BytesReceived / UploadReady events. spaceOwner = existing.GetOwner() + + // Check quota before accepting the upload. Skip for size-deferred uploads + // (uploadLength == -1) since the final size is unknown at this point. + // For overwrites the existing bytes will be freed on commit, so the net + // required space is uploadLength - existing.Size. + if uploadLength >= 0 { + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + if _, _, remaining, qErr := c.FS.GetQuota(ctx, spaceRef); qErr == nil { + existingSize := existing.GetSize() + netRequired := uint64(uploadLength) + if existingSize < netRequired { + netRequired -= existingSize + } else { + netRequired = 0 + } + if remaining < netRequired { + return nil, errtypes.InsufficientStorage("quota exceeded") + } + } + } } else { + // Check quota before creating the placeholder node. The ref's ResourceId + // points to the space root, which is sufficient for GetQuota. + if uploadLength > 0 { + if _, _, remaining, qErr := c.FS.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { + return nil, errtypes.InsufficientStorage("quota exceeded") + } + } + // New file: create the placeholder node via TouchFile. result, tfErr := c.FS.TouchFile(ctx, ref, false, mtime) if tfErr != nil { From 411f26736e3c6000bc11a25410e19a3607b62b31 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Thu, 2 Jul 2026 09:39:26 +0200 Subject: [PATCH 07/20] filestore & session implementation --- .../storageprovider/storageprovider.go | 9 +- .../services/dataprovider/dataprovider.go | 8 +- pkg/upload/filestore.go | 175 ++++++ pkg/upload/session.go | 514 ++++++++++++++++++ 4 files changed, 699 insertions(+), 7 deletions(-) create mode 100644 pkg/upload/filestore.go create mode 100644 pkg/upload/session.go diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 23233f46cd4..64c89e3e0be 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -214,14 +214,17 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } - // Build and start the upload coordinator if the driver exposes a session store. + // Build and start the upload coordinator from the service's own config. + // The session store is independent of the driver: we construct it directly + // from the driver config's root + tokens block rather than relying on the + // driver to expose a session store via an interface. var coordinator storage.FS - if sp, ok := fs.(pkgupload.UploadSessionStoreProvider); ok { + if store := pkgupload.FileStoreFromDriverConf(c.Drivers[c.Driver], log); store != nil { evstream, err := estreamFromConfig(c.Events) if err != nil { return nil, err } - coord := pkgupload.NewCoordinator(fs, sp.UploadSessionStore(), evstream, + coord := pkgupload.NewCoordinator(fs, store, evstream, c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log) if evstream != nil { if err := coord.Start(evstream); err != nil { diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index df8434396ba..7ee0b0e8feb 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -114,11 +114,11 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - // Wrap the FS in the upload coordinator if the driver exposes a session store. - // The coordinator IS-A storage.FS, so it passes unchanged to getDataTXs. + // Wrap the FS in the upload coordinator. The session store is constructed + // directly from the driver config (root + tokens), independent of the driver. txFS := storage.FS(fs) - if sp, ok := fs.(pkgupload.UploadSessionStoreProvider); ok { - coord := pkgupload.NewCoordinator(fs, sp.UploadSessionStore(), evstream, + if store := pkgupload.FileStoreFromDriverConf(conf.Drivers[conf.Driver], log); store != nil { + coord := pkgupload.NewCoordinator(fs, store, evstream, conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log) if evstream != nil { if err := coord.Start(evstream); err != nil { diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go new file mode 100644 index 00000000000..d32fc70fe2e --- /dev/null +++ b/pkg/upload/filestore.go @@ -0,0 +1,175 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "encoding/json" + iofs "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/google/uuid" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +// TokenOptions carries the JWT-signing configuration needed to produce transfer +// URLs for the postprocessing service. +type TokenOptions struct { + DownloadEndpoint string + DataGatewayEndpoint string + TransferSharedSecret string + TransferExpires int64 +} + +// FileStore is a filesystem-backed SessionStore. Sessions are stored as a pair +// of files under /uploads/: +// +// - .info — JSON-encoded tusd.FileInfo +// - — staged binary bytes +// +// This is the same on-disk format used by OcisStore so existing sessions +// survive a rolling deploy that switches to FileStore. +type FileStore struct { + root string + opts TokenOptions + log *zerolog.Logger +} + +// FileStoreFromDriverConf builds a FileStore from a reva driver config map. +// Returns nil if the config carries no root path (driver does not support +// coordinated uploads). Each service that mounts the same driver calls this +// independently — the same pattern decomposedfs uses for its Aspects. +func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if driverConf == nil { + return nil + } + + type driverRootConf struct { + Root string `mapstructure:"root"` + UploadDirectory string `mapstructure:"upload_directory"` + } + var rc driverRootConf + _ = mapstructure.Decode(driverConf, &rc) + + root := rc.UploadDirectory + if root == "" { + root = rc.Root + } + if root == "" { + return nil + } + + type tokenConf struct { + DownloadEndpoint string `mapstructure:"download_endpoint"` + DataGatewayEndpoint string `mapstructure:"datagateway_endpoint"` + TransferSharedSecret string `mapstructure:"transfer_shared_secret"` + TransferExpires int64 `mapstructure:"transfer_expires"` + } + var tc tokenConf + if tokens, ok := driverConf["tokens"]; ok { + _ = mapstructure.Decode(tokens, &tc) + } + + return NewFileStore(root, TokenOptions{ + DownloadEndpoint: tc.DownloadEndpoint, + DataGatewayEndpoint: tc.DataGatewayEndpoint, + TransferSharedSecret: tc.TransferSharedSecret, + TransferExpires: tc.TransferExpires, + }, log) +} + +// NewFileStore creates a FileStore rooted at root. +// root must be on a shared filesystem when multiple pods handle the same space. +func NewFileStore(root string, opts TokenOptions, log *zerolog.Logger) *FileStore { + return &FileStore{root: root, opts: opts, log: log} +} + +// New allocates a fresh session with a new UUID. +func (fs *FileStore) New(_ context.Context) Session { + return &FileSession{ + store: fs, + info: tusd.FileInfo{ + ID: uuid.New().String(), + Storage: map[string]string{ + "Type": "FileStore", + }, + MetaData: tusd.MetaData{}, + }, + } +} + +// Get loads the session with the given id from disk. +func (fs *FileStore) Get(ctx context.Context, id string) (Session, error) { + infoPath := fileSessionPath(fs.root, id) + + data, err := os.ReadFile(infoPath) + if err != nil { + if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE { + return nil, tusd.ErrNotFound + } + if errors.Is(err, iofs.ErrNotExist) { + return nil, tusd.ErrNotFound + } + return nil, err + } + + var info tusd.FileInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + session := &FileSession{store: fs, info: info} + + stat, err := os.Stat(session.binPath()) + if err != nil { + if os.IsNotExist(err) { + return nil, tusd.ErrNotFound + } + return nil, err + } + session.info.Offset = stat.Size() + + return session, nil +} + +// List returns all sessions found under /uploads/*.info. +func (fs *FileStore) List(ctx context.Context) ([]Session, error) { + infoFiles, err := filepath.Glob(filepath.Join(fs.root, "uploads", "*.info")) + if err != nil { + return nil, err + } + + sessions := make([]Session, 0, len(infoFiles)) + for _, path := range infoFiles { + id := strings.TrimSuffix(filepath.Base(path), ".info") + session, err := fs.Get(ctx, id) + if err != nil { + fs.log.Error().Str("path", path).Err(err).Msg("filestore: could not load session") + continue + } + sessions = append(sessions, session) + } + return sessions, nil +} diff --git a/pkg/upload/session.go b/pkg/upload/session.go new file mode 100644 index 00000000000..32a73233de6 --- /dev/null +++ b/pkg/upload/session.go @@ -0,0 +1,514 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "crypto/md5" //nolint:gosec + "crypto/sha1" //nolint:gosec + "hash/adler32" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/golang-jwt/jwt/v5" + "github.com/google/renameio/v2" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/owncloud/reva/v2/pkg/appctx" + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/utils" +) + +const defaultFilePerm = os.FileMode(0664) + +// FileSession is a driver-agnostic upload session backed by a .info file and a +// .bin staging file on disk. It satisfies the Session interface so the +// Coordinator can drive it without any knowledge of decomposedfs internals. +type FileSession struct { + store *FileStore + info tusd.FileInfo +} + +// --- tusd.Upload --- + +// GetInfo returns the TUS FileInfo for this session. +func (s *FileSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { + return s.info, nil +} + +// GetReader opens the staged binary file for reading. +func (s *FileSession) GetReader(_ context.Context) (io.ReadCloser, error) { + return os.Open(s.binPath()) +} + +// WriteChunk appends src to the staged binary file at the given offset. +func (s *FileSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + if err != nil && err != io.ErrUnexpectedEOF { + return n, err + } + s.info.Offset += n + return n, nil +} + +// FinishUpload is the tusd hook called after all bytes arrive. +// For the coordinator path this is handled by coordinatedUpload.FinishUpload; +// this method exists to satisfy the tusd.Upload interface. +func (s *FileSession) FinishUpload(ctx context.Context) error { + return s.FinishBytesOnly(ctx) +} + +// FinishBytesOnly computes and validates checksums and stores them on the session. +// It does not commit anything to the storage backend. +func (s *FileSession) FinishBytesOnly(ctx context.Context) error { + sha1h, md5h, adler32h, err := calculateChecksums(ctx, s.binPath()) + if err != nil { + return err + } + + if s.info.MetaData["checksum"] != "" { + parts := strings.SplitN(s.info.MetaData["checksum"], " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + var checkErr error + switch parts[0] { + case "sha1": + checkErr = checkHash(parts[1], sha1h) + case "md5": + checkErr = checkHash(parts[1], md5h) + case "adler32": + checkErr = checkHash(parts[1], adler32h) + default: + checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if checkErr != nil { + s.Cleanup(false, true, true, false) + return checkErr + } + } + + s.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + +// Terminate removes all on-disk state for this session. +func (s *FileSession) Terminate(_ context.Context) error { + s.Cleanup(true, true, true, true) + return nil +} + +// DeclareLength updates the upload length on the session and persists it. +func (s *FileSession) DeclareLength(ctx context.Context, length int64) error { + s.info.Size = length + s.info.SizeIsDeferred = false + return s.Persist(ctx) +} + +// ConcatUploads concatenates the staged binaries of partialUploads into this session's bin. +func (s *FileSession) ConcatUploads(_ context.Context, partialUploads []tusd.Upload) error { + file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partial := range partialUploads { + fs, ok := partial.(*FileSession) + if !ok { + return fmt.Errorf("filestore: ConcatUploads: unexpected upload type %T", partial) + } + src, err := os.Open(fs.binPath()) + if err != nil { + return err + } + _, copyErr := io.Copy(file, src) + src.Close() + if copyErr != nil { + return copyErr + } + } + return nil +} + +// --- storage.UploadSession --- + +// Purge removes all on-disk state for this session. +func (s *FileSession) Purge(ctx context.Context) { + s.Cleanup(true, true, true, true) +} + +// ScanData returns the AV scan result and scan date stored on the session. +func (s *FileSession) ScanData() (string, time.Time) { + date := s.info.MetaData["scanDate"] + if date == "" { + return "", time.Time{} + } + d, _ := time.Parse(time.RFC3339, date) + return s.info.MetaData["scanResult"], d +} + +// --- Session (coordinator-specific accessors) --- + +// ID returns the upload session ID. +func (s *FileSession) ID() string { + return s.info.ID +} + +// Filename returns the filename stored in the session. +func (s *FileSession) Filename() string { + return s.info.Storage["NodeName"] +} + +// Size returns the declared upload size. +func (s *FileSession) Size() int64 { + return s.info.Size +} + +// SizeDiff returns the size difference computed after upload. +func (s *FileSession) SizeDiff() int64 { + v, _ := strconv.ParseInt(s.info.MetaData["sizeDiff"], 10, 64) + return v +} + +// Offset returns the current upload offset. +func (s *FileSession) Offset() int64 { + return s.info.Offset +} + +// BinPath returns the path to the staged binary file. +func (s *FileSession) BinPath() string { + return s.binPath() +} + +// SpaceGid returns the numeric GID of the space owner, or "" if not set. +func (s *FileSession) SpaceGid() string { + return s.info.Storage["SpaceGid"] +} + +// ProviderID returns the storage provider ID stored in the session. +func (s *FileSession) ProviderID() string { + return s.info.MetaData["providerID"] +} + +// SpaceID returns the space (root) ID. +func (s *FileSession) SpaceID() string { + return s.info.Storage["SpaceRoot"] +} + +// NodeID returns the node ID for this upload. +func (s *FileSession) NodeID() string { + return s.info.Storage["NodeId"] +} + +// NodeExists returns whether the target node existed when the upload was initiated. +func (s *FileSession) NodeExists() bool { + return s.info.Storage["NodeExists"] == "true" +} + +// Dir returns the directory portion of the upload path. +func (s *FileSession) Dir() string { + return s.info.Storage["Dir"] +} + +// IsProcessing returns true if all bytes are received but postprocessing has not finished. +func (s *FileSession) IsProcessing() bool { + return s.info.Size == s.info.Offset && s.info.MetaData["scanResult"] == "" +} + +// SpaceOwner returns the space owner user ID. +func (s *FileSession) SpaceOwner() *userpb.UserId { + return &userpb.UserId{ + OpaqueId: s.info.Storage["SpaceOwnerOrManager"], + } +} + +// Executant returns the user ID of the user who initiated this upload. +func (s *FileSession) Executant() userpb.UserId { + return userpb.UserId{ + Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + } +} + +// Expires returns the upload expiry time. +func (s *FileSession) Expires() time.Time { + var t time.Time + if value, ok := s.info.MetaData["expires"]; ok { + t, _ = utils.MTimeToTime(value) + } + return t +} + +// Reference returns a CS3 reference for the resource being uploaded. +func (s *FileSession) Reference() provider.Reference { + return provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: s.info.MetaData["providerID"], + SpaceId: s.info.Storage["SpaceRoot"], + OpaqueId: s.info.Storage["NodeId"], + }, + } +} + +// Checksums returns the pre-computed checksums stored on the session. +func (s *FileSession) Checksums() storage.UploadChecksums { + decode := func(key string) []byte { + b, _ := hex.DecodeString(s.info.MetaData[key]) + return b + } + return storage.UploadChecksums{ + SHA1: decode("checksumSHA1"), + MD5: decode("checksumMD5"), + Adler32: decode("checksumAdler32"), + } +} + +// Metadata returns the upload metadata map passed to CommitUpload. +func (s *FileSession) Metadata() map[string]string { + return map[string]string{ + "providerID": s.info.MetaData["providerID"], + "mtime": s.info.MetaData["mtime"], + "nodeExists": s.info.Storage["NodeExists"], + "versionsPath": s.info.MetaData["versionsPath"], + "sessionID": s.info.ID, + } +} + +// SetScanData stores AV scan results on the session. +func (s *FileSession) SetScanData(result string, date time.Time) { + s.info.MetaData["scanResult"] = result + s.info.MetaData["scanDate"] = date.Format(time.RFC3339) +} + +// SetChecksums stores pre-computed checksums so CommitUpload can use them without +// re-reading the binary file. +func (s *FileSession) SetChecksums(sha1Sum, md5Sum, adler32Sum []byte) { + s.info.MetaData["checksumSHA1"] = hex.EncodeToString(sha1Sum) + s.info.MetaData["checksumMD5"] = hex.EncodeToString(md5Sum) + s.info.MetaData["checksumAdler32"] = hex.EncodeToString(adler32Sum) +} + +// SetMetadata sets a user-visible upload metadata field. +func (s *FileSession) SetMetadata(key, value string) { + s.info.MetaData[key] = value +} + +// SetStorageValue sets an internal storage field on the session. +func (s *FileSession) SetStorageValue(key, value string) { + s.info.Storage[key] = value +} + +// SetSize updates the declared upload size. +func (s *FileSession) SetSize(size int64) { + s.info.Size = size +} + +// SetSizeIsDeferred marks the upload size as not yet known. +func (s *FileSession) SetSizeIsDeferred(value bool) { + s.info.SizeIsDeferred = value +} + +// SetExecutant stores the identity of the user who initiated the upload. +func (s *FileSession) SetExecutant(u *userpb.User) { + s.info.Storage["Idp"] = u.GetId().GetIdp() + s.info.Storage["UserId"] = u.GetId().GetOpaqueId() + s.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type) + s.info.Storage["UserName"] = u.GetUsername() + s.info.Storage["UserDisplayName"] = u.GetDisplayName() + b, _ := json.Marshal(u.GetOpaque()) + s.info.Storage["UserOpaque"] = string(b) +} + +// TouchBin creates the empty staging file. +func (s *FileSession) TouchBin() error { + f, err := os.OpenFile(s.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return err + } + return f.Close() +} + +// Persist writes the session metadata atomically to disk. +func (s *FileSession) Persist(ctx context.Context) error { + infoPath := s.infoPath() + if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil { + return err + } + d, err := json.Marshal(s.info) + if err != nil { + return err + } + return renameio.WriteFile(infoPath, d, 0600) +} + +// Cleanup removes the staged binary and/or info file. +// Unlike OcisSession.Cleanup, this implementation has no knowledge of storage +// nodes — node reverting and unmark-processing are the coordinator's responsibility +// via storage.FS.MarkProcessing / Delete calls. +func (s *FileSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) { + log := s.store.log + if cleanBin { + if err := os.Remove(s.binPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Str("path", s.binPath()).Err(err).Msg("filestore: removing staged binary failed") + } + } + if cleanInfo { + if err := os.Remove(s.infoPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Str("path", s.infoPath()).Err(err).Msg("filestore: removing session info failed") + } + } +} + +// Context reconstructs a context carrying the user, logger, lock ID, and +// initiator ID that were recorded when the upload was initiated. +func (s *FileSession) Context(ctx context.Context) context.Context { + sub := s.store.log.With().Int("pid", os.Getpid()).Logger() + ctx = appctx.WithLogger(ctx, &sub) + ctx = ctxpkg.ContextSetLockID(ctx, s.info.MetaData["lockid"]) + ctx = ctxpkg.ContextSetUser(ctx, s.executantUser()) + return ctxpkg.ContextSetInitiator(ctx, s.info.MetaData["initiatorid"]) +} + +// URL returns a signed JWT URL that the postprocessing service can use to +// download the staged binary via the data gateway. +func (s *FileSession) URL(_ context.Context) (string, error) { + type transferClaims struct { + jwt.RegisteredClaims + Target string `json:"target"` + } + + target := joinURLParts(s.store.opts.DownloadEndpoint, "tus/", s.info.ID) + ttl := time.Duration(s.store.opts.TransferExpires) * time.Second + claims := transferClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), + Audience: jwt.ClaimStrings{"reva"}, + IssuedAt: jwt.NewNumericDate(time.Now()), + }, + Target: target, + } + t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) + tkn, err := t.SignedString([]byte(s.store.opts.TransferSharedSecret)) + if err != nil { + return "", errors.Wrapf(err, "filestore: error signing transfer token with claims %+v", claims) + } + return joinURLParts(s.store.opts.DataGatewayEndpoint, tkn), nil +} + +// ToFileInfo returns the underlying tusd.FileInfo. +func (s *FileSession) ToFileInfo() tusd.FileInfo { + return s.info +} + +// InitiatorID returns the initiator ID stored in the session metadata. +func (s *FileSession) InitiatorID() string { + return s.info.MetaData["initiatorid"] +} + +// --- internal helpers --- + +func (s *FileSession) binPath() string { + return filepath.Join(s.store.root, "uploads", s.info.ID) +} + +func (s *FileSession) infoPath() string { + return fileSessionPath(s.store.root, s.info.ID) +} + +func (s *FileSession) executantUser() *userpb.User { + var o *typespb.Opaque + _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) + return &userpb.User{ + Id: &userpb.UserId{ + Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + }, + Username: s.info.Storage["UserName"], + DisplayName: s.info.Storage["UserDisplayName"], + Opaque: o, + } +} + +// fileSessionPath returns the path to the .info file for the given session ID. +func fileSessionPath(root, id string) string { + return filepath.Join(root, "uploads", id+".info") +} + +// calculateChecksums computes sha1, md5, and adler32 in a single pass over path. +func calculateChecksums(_ context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { + sha1h := sha1.New() //nolint:gosec + md5h := md5.New() //nolint:gosec + adler32h := adler32.New() + + f, err := os.Open(path) + if err != nil { + return nil, nil, nil, err + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + if _, err = io.Copy(adler32h, r2); err != nil { + return nil, nil, nil, err + } + return sha1h, md5h, adler32h, nil +} + +func checkHash(expected string, h hash.Hash) error { + got := hex.EncodeToString(h.Sum(nil)) + if expected != got { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %s", expected, got)) + } + return nil +} + +// joinURLParts concatenates URL path segments, inserting "/" between them if needed. +func joinURLParts(parts ...string) string { + var b strings.Builder + for i, p := range parts { + b.WriteString(p) + if i < len(parts)-1 && !strings.HasSuffix(p, "/") { + b.WriteByte('/') + } + } + return b.String() +} From a1002a1de603c56a51671247edb7a7ed0d8156ee Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Tue, 7 Jul 2026 16:54:39 +0200 Subject: [PATCH 08/20] refactor(upload): decouple Coordinator from storage.FS --- .../storageprovider/storageprovider.go | 46 +-- .../services/dataprovider/dataprovider.go | 36 ++- pkg/rhttp/datatx/datatx.go | 3 +- pkg/rhttp/datatx/manager/simple/simple.go | 7 +- pkg/rhttp/datatx/manager/spaces/spaces.go | 7 +- pkg/rhttp/datatx/manager/tus/tus.go | 75 ++--- pkg/storage/fs/posix/posix.go | 6 +- pkg/storage/storage.go | 7 + pkg/storage/uploads.go | 3 + .../utils/decomposedfs/upload_async_test.go | 34 +-- pkg/storage/utils/middleware/middleware.go | 6 +- pkg/upload/coordinator.go | 264 +++++++++++++----- pkg/upload/filestore.go | 23 +- 13 files changed, 336 insertions(+), 181 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 64c89e3e0be..57722135dc5 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -71,6 +71,7 @@ type config struct { CustomMimeTypesJSON string `mapstructure:"custom_mimetypes_json" docs:"nil;An optional mapping file with the list of supported custom file extensions and corresponding mime types."` MountID string `mapstructure:"mount_id"` UploadExpiration int64 `mapstructure:"upload_expiration" docs:"0;Duration for how long uploads will be valid."` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` Events eventconfig `mapstructure:"events" docs:"0;Event stream configuration"` } @@ -106,7 +107,7 @@ func (c *config) init() { } if c.Events.ConsumerGroup == "" { - c.Events.ConsumerGroup = "storageprovider" + c.Events.ConsumerGroup = "dcfs" } if c.Events.NumConsumers <= 0 { c.Events.NumConsumers = 1 @@ -117,7 +118,7 @@ func (c *config) init() { type Service struct { conf *config Storage storage.FS - Coordinator storage.FS // nil when driver does not support coordinated uploads + Coordinator pkgupload.Coordinator dataServerURL *url.URL availableXS []*provider.ResourceChecksumPriority } @@ -214,30 +215,35 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } - // Build and start the upload coordinator from the service's own config. // The session store is independent of the driver: we construct it directly // from the driver config's root + tokens block rather than relying on the // driver to expose a session store via an interface. - var coordinator storage.FS - if store := pkgupload.FileStoreFromDriverConf(c.Drivers[c.Driver], log); store != nil { - evstream, err := estreamFromConfig(c.Events) - if err != nil { + store := pkgupload.NewFileStoreFromConfig(c.UploadDirectory, c.Drivers[c.Driver], log) + if store == nil { + return nil, fmt.Errorf("storageprovider: cannot determine upload directory — set upload_directory in config or driver root") + } + if err := store.Setup(); err != nil { + return nil, fmt.Errorf("storageprovider: upload directory setup failed: %w", err) + } + evstream, err := estreamFromConfig(c.Events) + if err != nil { + return nil, err + } + async := evstream != nil + coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, + c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log) + if err != nil { + return nil, err + } + if async { + if err := coord.Start(evstream); err != nil { return nil, err } - coord := pkgupload.NewCoordinator(fs, store, evstream, - c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log) - if evstream != nil { - if err := coord.Start(evstream); err != nil { - return nil, err - } - } - coordinator = coord } - service := &Service{ conf: c, Storage: fs, - Coordinator: coordinator, + Coordinator: coord, dataServerURL: u, availableXS: xsTypes, } @@ -460,11 +466,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate metadata["expires"] = strconv.Itoa(int(expirationTimestamp.Seconds)) } - var initiator storage.FS = s.Storage - if s.Coordinator != nil { - initiator = s.Coordinator - } - uploadIDs, err := initiator.InitiateUpload(ctx, req.Ref, uploadLength, metadata) + uploadIDs, err := s.Coordinator.InitiateUpload(ctx, req.Ref, uploadLength, metadata) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 7ee0b0e8feb..1160e15ab0a 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -55,6 +55,7 @@ type config struct { NatsPassword string `mapstructure:"nats_password"` ConsumerGroup string `mapstructure:"consumer_group"` NumConsumers int `mapstructure:"numconsumers"` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` } func (c *config) init() { @@ -65,7 +66,7 @@ func (c *config) init() { c.Driver = "localhome" } if c.ConsumerGroup == "" { - c.ConsumerGroup = "storageprovider" + c.ConsumerGroup = "dcfs" } if c.NumConsumers <= 0 { c.NumConsumers = 1 @@ -114,21 +115,26 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - // Wrap the FS in the upload coordinator. The session store is constructed - // directly from the driver config (root + tokens), independent of the driver. - txFS := storage.FS(fs) - if store := pkgupload.FileStoreFromDriverConf(conf.Drivers[conf.Driver], log); store != nil { - coord := pkgupload.NewCoordinator(fs, store, evstream, - conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log) - if evstream != nil { - if err := coord.Start(evstream); err != nil { - return nil, err - } + store := pkgupload.NewFileStoreFromConfig(conf.UploadDirectory, conf.Drivers[conf.Driver], log) + if store == nil { + return nil, fmt.Errorf("dataprovider: cannot determine upload directory — set upload_directory in config or driver root") + } + if err := store.Setup(); err != nil { + return nil, fmt.Errorf("dataprovider: upload directory setup failed: %w", err) + } + async := evstream != nil + coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, + conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log) + if err != nil { + return nil, err + } + if async { + if err := coord.Start(evstream); err != nil { + return nil, err } - txFS = coord } - dataTXs, err := getDataTXs(conf, txFS, evstream, log) + dataTXs, err := getDataTXs(conf, coord, fs, evstream, log) if err != nil { return nil, err } @@ -150,7 +156,7 @@ func getFS(c *config, stream events.Stream, log *zerolog.Logger) (storage.FS, er return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { +func getDataTXs(c *config, coord pkgupload.Coordinator, driver storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { if c.DataTXs == nil { c.DataTXs = make(map[string]map[string]interface{}) } @@ -170,7 +176,7 @@ func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerol for t := range c.DataTXs { if f, ok := datatxregistry.NewFuncs[t]; ok { if tx, err := f(c.DataTXs[t], publisher, log); err == nil { - if handler, err := tx.Handler(fs); err == nil { + if handler, err := tx.Handler(coord, driver); err == nil { txs[t] = handler } } diff --git a/pkg/rhttp/datatx/datatx.go b/pkg/rhttp/datatx/datatx.go index b770f73b890..80af29ba8df 100644 --- a/pkg/rhttp/datatx/datatx.go +++ b/pkg/rhttp/datatx/datatx.go @@ -28,12 +28,13 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/storage" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) // DataTX provides an abstraction around various data transfer protocols. type DataTX interface { - Handler(fs storage.FS) (http.Handler, error) + Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) } // EmitFileUploadedEvent is a helper function which publishes a FileUploaded event diff --git a/pkg/rhttp/datatx/manager/simple/simple.go b/pkg/rhttp/datatx/manager/simple/simple.go index 39e60e15662..dfad2743a21 100644 --- a/pkg/rhttp/datatx/manager/simple/simple.go +++ b/pkg/rhttp/datatx/manager/simple/simple.go @@ -40,6 +40,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/cache" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -78,7 +79,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("path", r.URL.Path).Logger() r = r.WithContext(appctx.WithLogger(r.Context(), &sublog)) @@ -92,7 +93,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, "") + download.GetOrHeadFile(w, r, driver, "") case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -114,7 +115,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { ctx = ctxpkg.ContextSetLockID(ctx, lockID) } - info, err := fs.Upload(ctx, storage.UploadRequest{ + info, err := coord.Upload(ctx, storage.UploadRequest{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/pkg/rhttp/datatx/manager/spaces/spaces.go b/pkg/rhttp/datatx/manager/spaces/spaces.go index b0b2f3b6ad8..6250f3d9a5b 100644 --- a/pkg/rhttp/datatx/manager/spaces/spaces.go +++ b/pkg/rhttp/datatx/manager/spaces/spaces.go @@ -42,6 +42,7 @@ import ( "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/cache" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -80,7 +81,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord pkgupload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var spaceID string spaceID, r.URL.Path = router.ShiftPath(r.URL.Path) @@ -97,7 +98,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, spaceID) + download.GetOrHeadFile(w, r, driver, spaceID) case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -117,7 +118,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { Path: fn, } var info *provider.ResourceInfo - info, err = fs.Upload(ctx, storage.UploadRequest{ + info, err = coord.Upload(ctx, storage.UploadRequest{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/pkg/rhttp/datatx/manager/tus/tus.go b/pkg/rhttp/datatx/manager/tus/tus.go index ac0e037846e..59778934b2e 100644 --- a/pkg/rhttp/datatx/manager/tus/tus.go +++ b/pkg/rhttp/datatx/manager/tus/tus.go @@ -33,13 +33,13 @@ import ( "github.com/owncloud/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/owncloud/reva/v2/pkg/appctx" - "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/manager/registry" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storagespace" + pkgupload "github.com/owncloud/reva/v2/pkg/upload" ) func init() { @@ -87,20 +87,9 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { - composable, ok := fs.(storage.ComposableFS) - if !ok { - return nil, errtypes.NotSupported("file system does not support the tus protocol") - } - - // A storage backend for tusd may consist of multiple different parts which - // handle upload creation, locking, termination and so on. The composer is a - // place where all those separated pieces are joined together. In this example - // we only use the file store but you may plug in multiple. +func (m *manager) Handler(coord pkgupload.Coordinator, _ storage.FS) (http.Handler, error) { composer := tusd.NewStoreComposer() - - // let the composable storage tell tus which extensions it supports - composable.UseIn(composer) + coord.UseIn(composer) config := tusd.Config{ StoreComposer: composer, @@ -130,33 +119,28 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return nil, err } - if usl, ok := fs.(storage.UploadSessionLister); ok { - // We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info - go func() { - for { - ev := <-handler.CompleteUploads - // We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files - // so we create a Progress instance here that is used to read the correct properties - ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) - if err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") - } else { - if len(ups) < 1 { - appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") - continue - } - up := ups[0] - executant := up.Executant() - ref := up.Reference() - if m.publisher != nil { - if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") - } - } + go func() { + for { + ev := <-handler.CompleteUploads + ups, err := coord.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) + if err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") + continue + } + if len(ups) < 1 { + appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") + continue + } + up := ups[0] + executant := up.Executant() + ref := up.Reference() + if m.publisher != nil { + if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") } } - }() - } + } + }() h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("uploadid", r.URL.Path).Logger() @@ -174,7 +158,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, w, r) handler.PostFile(w, r) case "HEAD": handler.HeadFile(w, r) @@ -184,7 +168,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, w, r) handler.PatchFile(w, r) case "DELETE": handler.DelFile(w, r) @@ -205,15 +189,10 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return h, nil } -func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { +func setHeaders(coord pkgupload.Coordinator, w http.ResponseWriter, r *http.Request) { ctx := r.Context() id := path.Base(r.URL.Path) - datastore, ok := fs.(tusd.DataStore) - if !ok { - appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore") - return - } - upload, err := datastore.GetUpload(ctx, id) + upload, err := coord.GetUpload(ctx, id) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage") return diff --git a/pkg/storage/fs/posix/posix.go b/pkg/storage/fs/posix/posix.go index 03cf89ce7ee..1c7d776f5a9 100644 --- a/pkg/storage/fs/posix/posix.go +++ b/pkg/storage/fs/posix/posix.go @@ -180,21 +180,23 @@ func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (s } // ListUploadSessions returns the upload sessions matching the given filter +// TODO(OCISDEV-901): remove once UploadSessionLister is removed from storage.FS (coordinator responsibility). func (fs *posixFS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { return fs.FS.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) } // UseIn tells the tus upload middleware which extensions it supports. +// TODO(OCISDEV-901): coordinator now handles TUS registration globally — is per-driver UseIn still needed for +// capability variation, or can this be removed along with storage.ComposableFS? func (fs *posixFS) UseIn(composer *tusd.StoreComposer) { fs.FS.(storage.ComposableFS).UseIn(composer) } -// NewUpload returns a new tus Upload instance +// TODO(OCISDEV-901): remove NewUpload and GetUpload — TUS DataStore is the coordinator's responsibility, not the driver's. func (fs *posixFS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { return fs.FS.(tusd.DataStore).NewUpload(ctx, info) } -// NewUpload returns a new tus Upload instance func (fs *posixFS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { return fs.FS.(tusd.DataStore).GetUpload(ctx, id) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 6c8dc675850..9f7c216bffc 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -236,6 +236,13 @@ type ComposableFS interface { UseIn(composer *tusd.StoreComposer) } +// TODO(OCISDEV-901): ComposableFS and all driver-side UseIn implementations +// (middleware.FS, posix.FS) are dead after the coordinator decoupling. The +// upload coordinator is the universal tusd DataStore for every driver, so TUS +// capabilities are global — no per-driver UseIn dispatch needed. Delete this +// interface and its implementations once OCISDEV-901 wires storage-system and +// removes the legacy decomposedfs upload path. + // Registry is the interface that storage registries implement // for discovering storage providers type Registry interface { diff --git a/pkg/storage/uploads.go b/pkg/storage/uploads.go index 59ff5088550..dd7e89fd5d7 100644 --- a/pkg/storage/uploads.go +++ b/pkg/storage/uploads.go @@ -46,6 +46,9 @@ type UploadsManager interface { } // UploadSessionLister defines the interface for FS implementations that allow listing and purging upload sessions +// TODO(OCISDEV-901): ListUploadSessions belongs on the coordinator, not the driver. Remove this interface and all +// driver-side implementations (decomposedfs, posix, middleware, ocm). The CLI (storage-users uploads list) currently +// instantiates the driver directly to call this — it needs to be rerouted through the coordinator before removal. type UploadSessionLister interface { // ListUploadSessions returns the upload sessions matching the given filter ListUploadSessions(ctx context.Context, filter UploadSessionFilter) ([]UploadSession, error) diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 721c36ac7f7..ef6b00501b8 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -10,7 +10,6 @@ import ( cs3permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - tusd "github.com/tus/tusd/v2/pkg/handler" ruser "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/events/stream" @@ -74,8 +73,9 @@ var _ = Describe("Async file uploads", Ordered, func() { con chan interface{} uploadID string - fs storage.FS - o *options.Options + fs storage.FS + coord pkgupload.Coordinator + o *options.Options lu *lookup.Lookup pmock *mocks.PermissionsChecker cs3permissionsclient *mocks.CS3PermissionsClient @@ -125,9 +125,7 @@ var _ = Describe("Async file uploads", Ordered, func() { // the node already exists; FinishUploadDecomposed (legacy path) would // try to create it again and fail with EEXIST. tusUpload = func(id string, content []byte) { - ds, ok := fs.(tusd.DataStore) - Expect(ok).To(BeTrue(), "fs must implement tusd.DataStore") - up, err := ds.GetUpload(ctx, id) + up, err := coord.GetUpload(ctx, id) Expect(err).ToNot(HaveOccurred()) _, err = up.WriteChunk(ctx, 0, bytes.NewReader(content)) Expect(err).ToNot(HaveOccurred()) @@ -205,9 +203,11 @@ var _ = Describe("Async file uploads", Ordered, func() { // Wire the coordinator so postprocessing events are consumed during tests. d := dfs.(*Decomposedfs) fileStore := pkgupload.NewFileStore(o.Root, pkgupload.TokenOptions{}, &zerolog.Logger{}) - coord := pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, "", "dcfs", 1, &zerolog.Logger{}) + var coordErr error + coord, coordErr = pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, true, "", "dcfs", 1, &zerolog.Logger{}) + Expect(coordErr).ToNot(HaveOccurred()) Expect(coord.Start(aspects.EventStream)).To(Succeed()) - fs = coord + fs = d resp, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{Owner: user, Type: "personal"}) Expect(err).ToNot(HaveOccurred()) @@ -220,7 +220,7 @@ var _ = Describe("Async file uploads", Ordered, func() { Return(nil) // start upload of a file - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 10, map[string]string{}) Expect(err).ToNot(HaveOccurred()) Expect(len(uploadIds)).To(Equal(2)) Expect(uploadIds["simple"]).ToNot(BeEmpty()) @@ -341,7 +341,7 @@ var _ = Describe("Async file uploads", Ordered, func() { Expect(len(revs)).To(Equal(0)) // upload again - uploadIds, err := fs.InitiateUpload(ctx, ref, 10, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 10, map[string]string{}) Expect(err).ToNot(HaveOccurred()) Expect(len(uploadIds)).To(Equal(2)) Expect(uploadIds["simple"]).ToNot(BeEmpty()) @@ -412,14 +412,12 @@ var _ = Describe("Async file uploads", Ordered, func() { It("rejects the second FinishUpload with ResourceProcessing", func() { // First upload is in postprocessing (BytesReceived consumed in BeforeEach). // Initiate a second upload. - uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).ToNot(HaveOccurred()) secondUploadID := uploadIds["simple"] // Write bytes for the second upload. - ds, ok := fs.(tusd.DataStore) - Expect(ok).To(BeTrue()) - up, err := ds.GetUpload(ctx, secondUploadID) + up, err := coord.GetUpload(ctx, secondUploadID) Expect(err).ToNot(HaveOccurred()) _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) Expect(err).ToNot(HaveOccurred()) @@ -433,13 +431,11 @@ var _ = Describe("Async file uploads", Ordered, func() { }) It("cleans up the rejected session's bin and info files", func() { - uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).ToNot(HaveOccurred()) secondUploadID := uploadIds["simple"] - ds, ok := fs.(tusd.DataStore) - Expect(ok).To(BeTrue()) - up, err := ds.GetUpload(ctx, secondUploadID) + up, err := coord.GetUpload(ctx, secondUploadID) Expect(err).ToNot(HaveOccurred()) _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) Expect(err).ToNot(HaveOccurred()) @@ -463,7 +459,7 @@ var _ = Describe("Async file uploads", Ordered, func() { succeedPostprocessing(uploadID) // Second upload. - uploadIds, err := fs.InitiateUpload(ctx, ref, 20, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).ToNot(HaveOccurred()) secondUploadID = uploadIds["simple"] tusUpload(secondUploadID, secondContent) diff --git a/pkg/storage/utils/middleware/middleware.go b/pkg/storage/utils/middleware/middleware.go index e589f645e8a..da9278c363a 100644 --- a/pkg/storage/utils/middleware/middleware.go +++ b/pkg/storage/utils/middleware/middleware.go @@ -51,21 +51,23 @@ func NewFS(next storage.FS, hooks ...Hook) *FS { } // ListUploadSessions returns the upload sessions matching the given filter +// TODO(OCISDEV-901): remove once UploadSessionLister is removed from storage.FS (coordinator responsibility). func (f *FS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { return f.next.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) } // UseIn tells the tus upload middleware which extensions it supports. +// TODO(OCISDEV-901): coordinator now handles TUS registration globally — is per-driver UseIn still needed for +// capability variation, or can this be removed along with storage.ComposableFS? func (f *FS) UseIn(composer *tusd.StoreComposer) { f.next.(storage.ComposableFS).UseIn(composer) } -// NewUpload returns a new tus Upload instance +// TODO(OCISDEV-901): remove NewUpload and GetUpload — TUS DataStore is the coordinator's responsibility, not the driver's. func (f *FS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { return f.next.(tusd.DataStore).NewUpload(ctx, info) } -// NewUpload returns a new tus Upload instance func (f *FS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { return f.next.(tusd.DataStore).GetUpload(ctx, id) } diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 87fc194dfeb..93b9d5c64ff 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -18,8 +18,8 @@ // Package upload provides the driver-agnostic upload coordinator: // TUS session management, postprocessing event loop, lifecycle event -// publishing. Any storage driver can embed the Coordinator to inherit -// the full oCIS upload pipeline. +// publishing. The Coordinator interface is independent of storage.FS — +// it uses a driver but is not a driver. package upload import ( @@ -95,34 +95,26 @@ type Session interface { } -// bytesFinisher is implemented by OcisSession to compute checksums without -// touching the node. The coordinator uses it from coordinatedUpload.FinishUpload. -type bytesFinisher interface { - FinishBytesOnly(ctx context.Context) error -} - // coordinatedUpload wraps a Session so the TUS FinishUpload hook runs the // coordinator path (checksums + MarkProcessing + BytesReceived) instead of // the legacy FinishUploadDecomposed path. type coordinatedUpload struct { tusd.Upload session Session - coord *Coordinator + coord *coordinator } func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { - if bf, ok := u.session.(bytesFinisher); ok { - if err := bf.FinishBytesOnly(ctx); err != nil { - return err - } - // Persist checksums to disk so the postprocessing handler can read them - // from the session store after the BytesReceived event is processed. - if err := u.session.Persist(ctx); err != nil { - return err - } + if err := u.session.FinishBytesOnly(ctx); err != nil { + return err + } + // Persist checksums to disk so the postprocessing handler can read them + // from the session store after the BytesReceived event is processed. + if err := u.session.Persist(ctx); err != nil { + return err } ref := u.session.Reference() - if err := u.coord.FS.MarkProcessing(ctx, &ref, true, u.session.ID()); err != nil { + if err := u.coord.fs.MarkProcessing(ctx, &ref, true, u.session.ID()); err != nil { // Slot already owned by another session. Clean up this session's bin and // info files — they will never reach postprocessing. u.session.Cleanup(false, true, true, false) @@ -132,7 +124,12 @@ func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { metrics.UploadProcessing.Inc() metrics.UploadSessionsBytesReceived.Inc() - if u.coord.pub != nil && u.session.Size() > 0 { + if !u.coord.async { + // Sync mode (e.g. storage-system): commit inline, no NATS required. + return u.coord.commitSync(ctx, u.session) + } + + if u.session.Size() > 0 { s, err := u.session.URL(ctx) if err != nil { return err @@ -162,6 +159,34 @@ func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { return nil } +// commitSync runs CommitUpload inline and cleans up the session. +// Used by the sync path (async=false) — called from both coordinatedUpload.FinishUpload +// (TUS) and Coordinator.Upload (simple PUT). +func (c *coordinator)commitSync(ctx context.Context, session Session) error { + ref := session.Reference() + f, err := os.Open(session.BinPath()) + if err != nil { + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(true, true, true, false) + return err + } + defer f.Close() + if _, err := c.fs.CommitUpload(ctx, &ref, storage.UploadSource{ + Body: f, + Length: session.Size(), + Metadata: session.Metadata(), + Checksums: session.Checksums(), + }); err != nil { + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(true, true, true, false) + return err + } + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(false, true, true, false) + metrics.UploadSessionsFinalized.Inc() + return nil +} + // SessionStore abstracts upload-session persistence for the Coordinator. type SessionStore interface { New(ctx context.Context) Session @@ -175,18 +200,23 @@ type RevisionReverter interface { RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error } -// Coordinator owns the upload state machine: -// - TUS HTTP layer (InitiateUpload, UseIn, GetUpload, ListUploadSessions) -// - postprocessing event loop (PostprocessingFinished, PostprocessingStepFinished, -// RestartPostprocessing, CleanUpload, RevertRevision) -// -// It embeds storage.FS so it IS-A storage.FS and can be passed wherever the -// underlying driver is expected. All methods not overridden here delegate to -// the embedded FS. -type Coordinator struct { - storage.FS +// Coordinator is the upload orchestrator interface. It is not a storage.FS — +// driver operations are handled separately. +type Coordinator interface { + InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) + Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) + GetUpload(ctx context.Context, id string) (tusd.Upload, error) + UseIn(composer *tusd.StoreComposer) + ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) + Start(stream events.Consumer) error +} + +// coordinator is the concrete implementation of Coordinator. +type coordinator struct { + fs storage.FS store SessionStore pub events.Publisher + async bool mountID string numConc int conGroup string @@ -194,32 +224,38 @@ type Coordinator struct { } // NewCoordinator constructs a Coordinator. Call Start to begin consuming events. +// async=true requires a non-nil pub; fail-fast mirrors decomposedfs.New(). func NewCoordinator( fs storage.FS, store SessionStore, pub events.Publisher, + async bool, mountID string, consumerGroup string, numConsumers int, log *zerolog.Logger, -) *Coordinator { +) (Coordinator, error) { + if async && pub == nil { + return nil, fmt.Errorf("need event stream for async upload processing") + } if numConsumers <= 0 { numConsumers = 1 } - return &Coordinator{ - FS: fs, + return &coordinator{ + fs: fs, store: store, pub: pub, + async: async, mountID: mountID, numConc: numConsumers, conGroup: consumerGroup, log: log, - } + }, nil } // Start subscribes to the event stream and launches numConsumers goroutines // that process postprocessing events. -func (c *Coordinator) Start(stream events.Consumer) error { +func (c *coordinator)Start(stream events.Consumer) error { ch, err := events.Consume( stream, c.conGroup, @@ -238,13 +274,13 @@ func (c *Coordinator) Start(stream events.Consumer) error { return nil } -func (c *Coordinator) postprocessingLoop(ch <-chan events.Event) { +func (c *coordinator)postprocessingLoop(ch <-chan events.Event) { for event := range ch { c.processEvent(context.Background(), event) } } -func (c *Coordinator) processEvent(evCtx context.Context, event events.Event) { +func (c *coordinator)processEvent(evCtx context.Context, event events.Event) { ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) defer span.End() @@ -265,7 +301,7 @@ func (c *Coordinator) processEvent(evCtx context.Context, event events.Event) { } } -func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { +func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { log := c.log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { log.Debug().Msg("ignoring event for different storage") @@ -281,9 +317,12 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() ref := session.Reference() - if _, err := c.FS.GetMD(ctx, &ref, []string{}, []string{}); err != nil { + if _, err := c.fs.GetMD(ctx, &ref, []string{}, []string{}); err != nil { log.Debug().Err(err).Msg("node no longer exists or not accessible; cleaning up") session.Cleanup(false, true, true, false) + if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during cleanup of inaccessible node") + } return } @@ -313,8 +352,9 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event keepUpload = true retryCommit = true } else { + defer f.Close() commitRef := session.Reference() - _, commitErr := c.FS.CommitUpload(ctx, &commitRef, storage.UploadSource{ + _, commitErr := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ Body: f, Length: session.Size(), Metadata: session.Metadata(), @@ -345,11 +385,11 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event nodeRef := session.Reference() if !retryCommit { - if err := c.FS.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { log.Error().Err(err).Msg("could not unmark processing after postprocessing finished") } if revertNodeMetadata { - if _, delErr := c.FS.Delete(ctx, &nodeRef); delErr != nil { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { if _, ok := delErr.(errtypes.NotFound); !ok { log.Error().Err(delErr).Msg("could not delete placeholder node on abort") } @@ -396,7 +436,7 @@ func (c *Coordinator) handlePostprocessingFinished(ctx context.Context, ev event } } -func (c *Coordinator) handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { +func (c *coordinator)handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { log := c.log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() session, err := c.store.Get(ctx, ev.UploadID) if err != nil { @@ -428,7 +468,7 @@ func (c *Coordinator) handleRestartPostprocessing(ctx context.Context, ev events } } -func (c *Coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUpload) { +func (c *coordinator)handleCleanUpload(ctx context.Context, ev events.CleanUpload) { log := c.log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() session, err := c.store.Get(ctx, ev.UploadID) if err != nil { @@ -438,11 +478,11 @@ func (c *Coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUplo ctx = session.Context(ctx) session.Cleanup(false, !ev.KeepUpload, !ev.KeepUpload, false) nodeRef := session.Reference() - if err := c.FS.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { log.Error().Err(err).Msg("could not unmark processing during CleanUpload") } if !session.NodeExists() { - if _, delErr := c.FS.Delete(ctx, &nodeRef); delErr != nil { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { if _, ok := delErr.(errtypes.NotFound); !ok { log.Error().Err(delErr).Msg("could not delete placeholder node during CleanUpload") } @@ -450,13 +490,13 @@ func (c *Coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUplo } } -func (c *Coordinator) handleRevertRevision(ctx context.Context, ev events.RevertRevision) { +func (c *coordinator)handleRevertRevision(ctx context.Context, ev events.RevertRevision) { log := c.log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { log.Debug().Msg("ignoring event for different storage") return } - rr, ok := c.FS.(RevisionReverter) + rr, ok := c.fs.(RevisionReverter) if !ok { log.Error().Msg("storage driver does not implement RevisionReverter") return @@ -466,7 +506,7 @@ func (c *Coordinator) handleRevertRevision(ctx context.Context, ev events.Revert } } -func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { +func (c *coordinator)handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { log.Debug().Msg("ignoring event for different storage") @@ -476,7 +516,11 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e return } - res := ev.Result.(events.VirusscanResult) + res, ok := ev.Result.(events.VirusscanResult) + if !ok { + log.Error().Msgf("coordinator: unexpected antivirus result type %T", ev.Result) + return + } if res.ErrorMsg != "" { return } @@ -501,7 +545,7 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e ctx = session.Context(ctx) ref := session.Reference() - if err := c.FS.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ + if err := c.fs.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ Metadata: map[string]string{ "scanstatus": res.Description, "scandate": res.Scandate.Format(time.RFC3339Nano), @@ -517,10 +561,10 @@ func (c *Coordinator) handlePostprocessingStepFinished(ctx context.Context, ev e // session. For new files TouchFile creates the node; for overwrites it already // exists and we skip it. All session fields are populated via typed setters so // the coordinator has no knowledge of internal storage key names. -func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { +func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { // Resolve node metadata: determine whether the target exists, get its ID, // parent ID, space ID, space owner, and the path for Dir. - existing, err := c.FS.GetMD(ctx, ref, []string{}, []string{}) + existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) nodeExists := err == nil mtime := "" @@ -549,7 +593,7 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc // required space is uploadLength - existing.Size. if uploadLength >= 0 { spaceRef := &provider.Reference{ResourceId: existing.GetId()} - if _, _, remaining, qErr := c.FS.GetQuota(ctx, spaceRef); qErr == nil { + if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil { existingSize := existing.GetSize() netRequired := uint64(uploadLength) if existingSize < netRequired { @@ -566,13 +610,13 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc // Check quota before creating the placeholder node. The ref's ResourceId // points to the space root, which is sufficient for GetQuota. if uploadLength > 0 { - if _, _, remaining, qErr := c.FS.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { + if _, _, remaining, qErr := c.fs.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { return nil, errtypes.InsufficientStorage("quota exceeded") } } // New file: create the placeholder node via TouchFile. - result, tfErr := c.FS.TouchFile(ctx, ref, false, mtime) + result, tfErr := c.fs.TouchFile(ctx, ref, false, mtime) if tfErr != nil { return nil, tfErr } @@ -587,7 +631,7 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc ResourceId: ref.ResourceId, Path: filepath.Dir(ref.GetPath()), } - if parentInfo, pErr := c.FS.GetMD(ctx, parentRef, []string{}, []string{}); pErr == nil { + if parentInfo, pErr := c.fs.GetMD(ctx, parentRef, []string{}, []string{}); pErr == nil { parentID = parentInfo.GetId().GetOpaqueId() } } @@ -682,7 +726,7 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc return nil, fmt.Errorf("coordinator: zero-length FinishBytesOnly: %w", err) } commitRef := session.Reference() - if _, err := c.FS.CommitUpload(ctx, &commitRef, storage.UploadSource{ + if _, err := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ Body: io.NopCloser(bytes.NewReader(nil)), Length: 0, Metadata: session.Metadata(), @@ -705,8 +749,98 @@ func (c *Coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc }, nil } +// Upload handles the simple (single-PUT) upload path so the coordinator owns +// the complete upload lifecycle regardless of the datatx protocol used. +// simple.go calls fs.Upload(); when fs is a *Coordinator this method intercepts. +func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { + id := strings.TrimPrefix(req.Ref.GetPath(), "/") + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + ctx = session.Context(ctx) + + size, err := session.WriteChunk(ctx, 0, req.Body) + if err != nil { + return nil, err + } + if size != req.Length { + return nil, errtypes.PartialContent(req.Ref.String()) + } + + if err := session.FinishBytesOnly(ctx); err != nil { + return nil, err + } + if err := session.Persist(ctx); err != nil { + return nil, err + } + + ref := session.Reference() + if err := c.fs.MarkProcessing(ctx, &ref, true, session.ID()); err != nil { + session.Cleanup(false, true, true, false) + return nil, err + } + + metrics.UploadProcessing.Inc() + metrics.UploadSessionsBytesReceived.Inc() + + if !c.async { + if err := c.commitSync(ctx, session); err != nil { + return nil, err + } + } else { + s, err := session.URL(ctx) + if err != nil { + return nil, err + } + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: &user.User{ + Id: &user.UserId{ + Type: session.Executant().Type, + Idp: session.Executant().Idp, + OpaqueId: session.Executant().OpaqueId, + }, + }, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + }); err != nil { + return nil, err + } + } + + executant := session.Executant() + uploadRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + } + if uff != nil { + uff(session.SpaceOwner(), &executant, uploadRef) + } + + return &provider.ResourceInfo{ + Id: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Name: session.Filename(), + }, nil +} + // UseIn registers the coordinator as the TUS data store in the composer. -func (c *Coordinator) UseIn(composer *tusd.StoreComposer) { +func (c *coordinator)UseIn(composer *tusd.StoreComposer) { composer.UseCore(c) composer.UseTerminater(c) composer.UseConcater(c) @@ -714,13 +848,13 @@ func (c *Coordinator) UseIn(composer *tusd.StoreComposer) { } // NewUpload is not supported; uploads are initiated via the CS3 API. -func (c *Coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { +func (c *coordinator)NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { return nil, errNotImplemented } // GetUpload returns the upload session wrapped in a coordinatedUpload so the // TUS FinishUpload hook runs the coordinator path rather than the legacy one. -func (c *Coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { +func (c *coordinator)GetUpload(ctx context.Context, id string) (tusd.Upload, error) { session, err := c.store.Get(ctx, id) if err != nil { return nil, err @@ -729,7 +863,7 @@ func (c *Coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, er } // ListUploadSessions returns upload sessions matching the given filter. -func (c *Coordinator) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { +func (c *coordinator)ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { sessions, err := c.store.List(ctx) if err != nil { return nil, err @@ -767,16 +901,16 @@ func (c *Coordinator) ListUploadSessions(ctx context.Context, filter storage.Upl } // AsTerminatableUpload returns the upload as a TerminatableUpload. -func (c *Coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { +func (c *coordinator)AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { return up.(tusd.TerminatableUpload) } // AsLengthDeclarableUpload returns the upload as a LengthDeclarableUpload. -func (c *Coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { +func (c *coordinator)AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { return up.(tusd.LengthDeclarableUpload) } // AsConcatableUpload returns the upload as a ConcatableUpload. -func (c *Coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { +func (c *coordinator)AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { return up.(tusd.ConcatableUpload) } diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go index d32fc70fe2e..d4e46fdc1dd 100644 --- a/pkg/upload/filestore.go +++ b/pkg/upload/filestore.go @@ -81,6 +81,22 @@ func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Log return nil } + return newFileStoreWithTokens(root, driverConf, log) +} + +// NewFileStoreFromConfig builds a FileStore using uploadDir when set, falling +// back to root/upload_directory from the active driver config. This allows +// drivers that have no local root (e.g. KW) to still get a coordinator by +// setting upload_directory at the service level rather than inside the driver. +// Returns nil only when neither source resolves to a non-empty path. +func NewFileStoreFromConfig(uploadDir string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if uploadDir != "" { + return NewFileStore(uploadDir, TokenOptions{}, log) + } + return FileStoreFromDriverConf(driverConf, log) +} + +func newFileStoreWithTokens(root string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { type tokenConf struct { DownloadEndpoint string `mapstructure:"download_endpoint"` DataGatewayEndpoint string `mapstructure:"datagateway_endpoint"` @@ -91,7 +107,6 @@ func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Log if tokens, ok := driverConf["tokens"]; ok { _ = mapstructure.Decode(tokens, &tc) } - return NewFileStore(root, TokenOptions{ DownloadEndpoint: tc.DownloadEndpoint, DataGatewayEndpoint: tc.DataGatewayEndpoint, @@ -106,6 +121,12 @@ func NewFileStore(root string, opts TokenOptions, log *zerolog.Logger) *FileStor return &FileStore{root: root, opts: opts, log: log} } +// Setup creates the uploads directory eagerly so permission problems are caught +// at startup rather than on the first upload. Mirrors decomposedfs tree.Setup(). +func (fs *FileStore) Setup() error { + return os.MkdirAll(filepath.Join(fs.root, "uploads"), 0700) +} + // New allocates a fresh session with a new UUID. func (fs *FileStore) New(_ context.Context) Session { return &FileSession{ From 4fbd21f049a52f4cf8f37b5d041f4390a7de858e Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 8 Jul 2026 14:42:59 +0200 Subject: [PATCH 09/20] cleanup --- .../storageprovider/storageprovider.go | 5 +- .../services/dataprovider/dataprovider.go | 2 +- .../utils/decomposedfs/decomposedfs.go | 10 - .../utils/decomposedfs/upload/session.go | 10 +- .../utils/decomposedfs/upload_async_test.go | 50 +-- pkg/storage/utils/middleware/middleware.go | 11 +- pkg/upload/coordinated_upload.go | 227 +++++++++++++ pkg/upload/coordinator.go | 303 +++++++----------- pkg/upload/filestore.go | 4 +- pkg/upload/session.go | 112 +------ 10 files changed, 378 insertions(+), 356 deletions(-) create mode 100644 pkg/upload/coordinated_upload.go diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 57722135dc5..061bb88acb3 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -215,12 +215,9 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } - // The session store is independent of the driver: we construct it directly - // from the driver config's root + tokens block rather than relying on the - // driver to expose a session store via an interface. store := pkgupload.NewFileStoreFromConfig(c.UploadDirectory, c.Drivers[c.Driver], log) if store == nil { - return nil, fmt.Errorf("storageprovider: cannot determine upload directory — set upload_directory in config or driver root") + return nil, fmt.Errorf("storageprovider: cannot determine upload directory, set upload_directory in config or driver root") } if err := store.Setup(); err != nil { return nil, fmt.Errorf("storageprovider: upload directory setup failed: %w", err) diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 1160e15ab0a..c1ff6ae22fd 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -117,7 +117,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) store := pkgupload.NewFileStoreFromConfig(conf.UploadDirectory, conf.Drivers[conf.Driver], log) if store == nil { - return nil, fmt.Errorf("dataprovider: cannot determine upload directory — set upload_directory in config or driver root") + return nil, fmt.Errorf("dataprovider: cannot determine upload directory, set upload_directory in config or driver root") } if err := store.Setup(); err != nil { return nil, fmt.Errorf("dataprovider: upload directory setup failed: %w", err) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 94452822416..415b7c463bf 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -256,16 +256,6 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error { return nil } -// RevertUploadRevision reverts an in-flight upload by calling RevertCurrentRevision on the node. -// It implements upload.RevisionReverter so the coordinator can delegate RevertRevision events. -func (fs *Decomposedfs) RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error { - n, err := fs.lu.NodeFromID(ctx, id) - if err != nil { - return err - } - return n.RevertCurrentRevision(ctx) -} - // GetQuota returns the quota available // TODO Document in the cs3 should we return quota or free space? func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, remaining uint64, err error) { diff --git a/pkg/storage/utils/decomposedfs/upload/session.go b/pkg/storage/utils/decomposedfs/upload/session.go index 4f8fa854f80..bc69e59da5d 100644 --- a/pkg/storage/utils/decomposedfs/upload/session.go +++ b/pkg/storage/utils/decomposedfs/upload/session.go @@ -360,13 +360,9 @@ func (s *OcisSession) Checksums() storage.UploadChecksums { } } -// Metadata returns a map of upload metadata needed to call CommitUpload. -// "nodeExists" carries whether the target node existed at InitiateUpload time. -// "versionsPath" carries the version archive path already created by -// CreateNodeForUpload so CommitUpload reuses it rather than creating a duplicate. -// "sessionID" is included so CommitUpload can tell whether this session still -// owns the processing slot; if a newer session took over, node size metadata -// is left intact so the in-progress upload's expected size is preserved. +// Metadata returns metadata passed to CommitUpload. +// "versionsPath" is pre-created by CreateNodeForUpload so CommitUpload reuses it. +// "sessionID" lets CommitUpload detect whether this session still owns the processing slot. func (s *OcisSession) Metadata() map[string]string { return map[string]string{ "providerID": s.info.MetaData["providerID"], diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index ef6b00501b8..39e739958d0 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -405,49 +405,27 @@ var _ = Describe("Async file uploads", Ordered, func() { }) When("a second upload is attempted while the first is still in postprocessing", func() { - // The coordinator serializes uploads via MarkProcessing: a second FinishUpload - // call while the first session holds the processing slot returns ResourceProcessing - // and the second session is cleaned up immediately. + // The coordinator serializes uploads via MarkProcessing: a second InitiateUpload + // while the first session holds the processing slot returns ResourceProcessing + // and cleans up immediately. - It("rejects the second FinishUpload with ResourceProcessing", func() { + It("rejects the second InitiateUpload with ResourceProcessing", func() { // First upload is in postprocessing (BytesReceived consumed in BeforeEach). - // Initiate a second upload. - uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) - Expect(err).ToNot(HaveOccurred()) - secondUploadID := uploadIds["simple"] - - // Write bytes for the second upload. - up, err := coord.GetUpload(ctx, secondUploadID) - Expect(err).ToNot(HaveOccurred()) - _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) - Expect(err).ToNot(HaveOccurred()) - - // FinishUpload must fail because the node is already processing. - err = up.FinishUpload(ctx) + _, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) Expect(err).To(HaveOccurred()) - _, isProcessing := err.(interface{ IsResourceProcessing() bool }) - _ = isProcessing Expect(err.Error()).To(ContainSubstring("resource is processing")) }) - It("cleans up the rejected session's bin and info files", func() { - uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) - Expect(err).ToNot(HaveOccurred()) - secondUploadID := uploadIds["simple"] - - up, err := coord.GetUpload(ctx, secondUploadID) - Expect(err).ToNot(HaveOccurred()) - _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) - Expect(err).ToNot(HaveOccurred()) - _ = up.FinishUpload(ctx) // expect failure; error checked in other test - - // Bin file must be removed. - _, statErr := os.Stat(filepath.Join(o.Root, "uploads", secondUploadID)) - Expect(statErr).ToNot(BeNil(), "bin file should be cleaned up after rejection") + It("leaves no orphan session files after rejection", func() { + // InitiateUpload fails before TouchBin/Persist, so no .bin or .info are created. + _, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).To(HaveOccurred()) - // Info file must be removed. - _, statErr = os.Stat(filepath.Join(o.Root, "uploads", secondUploadID+".info")) - Expect(statErr).ToNot(BeNil(), "info file should be cleaned up after rejection") + // No uploads directory entries should exist beyond the first session. + entries, readErr := os.ReadDir(filepath.Join(o.Root, "uploads")) + Expect(readErr).ToNot(HaveOccurred()) + // Only the first upload's .bin and .info should be present. + Expect(len(entries)).To(Equal(2)) }) }) diff --git a/pkg/storage/utils/middleware/middleware.go b/pkg/storage/utils/middleware/middleware.go index da9278c363a..1f858b9005d 100644 --- a/pkg/storage/utils/middleware/middleware.go +++ b/pkg/storage/utils/middleware/middleware.go @@ -27,7 +27,6 @@ import ( tusd "github.com/tus/tusd/v2/pkg/handler" "github.com/owncloud/reva/v2/pkg/storage" - "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/owncloud/reva/v2/pkg/storagespace" ) @@ -57,13 +56,13 @@ func (f *FS) ListUploadSessions(ctx context.Context, filter storage.UploadSessio } // UseIn tells the tus upload middleware which extensions it supports. -// TODO(OCISDEV-901): coordinator now handles TUS registration globally — is per-driver UseIn still needed for +// TODO(OCISDEV-901): coordinator now handles TUS registration globally. Is per-driver UseIn still needed for // capability variation, or can this be removed along with storage.ComposableFS? func (f *FS) UseIn(composer *tusd.StoreComposer) { f.next.(storage.ComposableFS).UseIn(composer) } -// TODO(OCISDEV-901): remove NewUpload and GetUpload — TUS DataStore is the coordinator's responsibility, not the driver's. +// TODO(OCISDEV-901): remove NewUpload and GetUpload. TUS DataStore is the coordinator's responsibility, not the driver's. func (f *FS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { return f.next.(tusd.DataStore).NewUpload(ctx, info) } @@ -76,21 +75,21 @@ func (f *FS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err // To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination // the storage needs to implement AsTerminatableUpload func (f *FS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.OcisSession) + return up.(tusd.TerminatableUpload) } // AsLengthDeclarableUpload returns a LengthDeclarableUpload // To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation // the storage needs to implement AsLengthDeclarableUpload func (f *FS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.OcisSession) + return up.(tusd.LengthDeclarableUpload) } // AsConcatableUpload returns a ConcatableUpload // To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation // the storage needs to implement AsConcatableUpload func (f *FS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.OcisSession) + return up.(tusd.ConcatableUpload) } func (f *FS) GetHome(ctx context.Context) (string, error) { diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go new file mode 100644 index 00000000000..755cd6d55c9 --- /dev/null +++ b/pkg/upload/coordinated_upload.go @@ -0,0 +1,227 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" +) + +// coordinatedUpload is the object the TUS handler talks to during an upload. +// TUS is a resumable upload protocol: clients upload files in chunks via HTTP PATCH, +// can resume after interruption, and optionally delete (Terminate), declare size +// upfront (DeclareLength), or assemble partial uploads (ConcatUploads). +// +// To plug into the TUS handler, a type must implement: +// - tusd.Upload: GetInfo, GetReader, WriteChunk, FinishUpload (core read/write) +// - tusd.TerminatableUpload: Terminate (DELETE support) +// - tusd.LengthDeclarableUpload: DeclareLength (deferred-length uploads) +// - tusd.ConcatableUpload: ConcatUploads (parallel chunk concatenation) +// +// coordinatedUpload owns all upload lifecycle logic (checksums, MarkProcessing, +// event publishing). It delegates raw data access to the Session. +type coordinatedUpload struct { + session Session + coord *coordinator +} + +func (u *coordinatedUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return u.session.GetInfo(ctx) +} + +func (u *coordinatedUpload) GetReader(ctx context.Context) (io.ReadCloser, error) { + return u.session.GetReader(ctx) +} + +func (u *coordinatedUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + return u.session.WriteChunk(ctx, offset, src) +} + +func (u *coordinatedUpload) Terminate(ctx context.Context) error { + ref := u.session.Reference() + u.session.Cleanup(true, true) + _ = u.coord.fs.MarkProcessing(ctx, &ref, false, u.session.ID()) + if !u.session.NodeExists() { + _, _ = u.coord.fs.Delete(ctx, &ref) + } + return nil +} + +func (u *coordinatedUpload) DeclareLength(ctx context.Context, length int64) error { + u.session.SetSize(length) + u.session.SetSizeIsDeferred(false) + return u.session.Persist(ctx) +} + +func (u *coordinatedUpload) ConcatUploads(ctx context.Context, partials []tusd.Upload) error { + file, err := os.OpenFile(u.session.BinPath(), os.O_WRONLY|os.O_APPEND, 0664) + if err != nil { + return err + } + defer file.Close() + for _, partial := range partials { + cu, ok := partial.(*coordinatedUpload) + if !ok { + return fmt.Errorf("coordinator: unexpected partial type %T", partial) + } + src, err := cu.session.GetReader(ctx) + if err != nil { + return err + } + _, copyErr := io.Copy(file, src) + src.Close() + if copyErr != nil { + return copyErr + } + } + return nil +} + +func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { + if err := checksumAndFinish(ctx, u.session); err != nil { + u.coord.rollback(ctx, u.session) + return err + } + // Persist checksums so the postprocessing handler can read them after BytesReceived. + if err := u.session.Persist(ctx); err != nil { + u.coord.rollback(ctx, u.session) + return err + } + + metrics.UploadProcessing.Inc() + metrics.UploadSessionsBytesReceived.Inc() + + if !u.coord.async { + return u.coord.commitSync(ctx, u.session) + } + + if u.session.Size() > 0 { + s, err := u.session.URL(ctx) + if err != nil { + u.coord.rollback(ctx, u.session) + return err + } + if err := events.Publish(ctx, u.coord.pub, events.BytesReceived{ + UploadID: u.session.ID(), + URL: s, + SpaceOwner: u.session.SpaceOwner(), + ExecutingUser: &user.User{ + Id: &user.UserId{ + Type: u.session.Executant().Type, + Idp: u.session.Executant().Idp, + OpaqueId: u.session.Executant().OpaqueId, + }, + }, + ResourceID: &provider.ResourceId{ + StorageId: u.session.ProviderID(), + SpaceId: u.session.SpaceID(), + OpaqueId: u.session.NodeID(), + }, + Filename: u.session.Filename(), + Filesize: uint64(u.session.Size()), + ImpersonatingUser: impersonatingUser(ctx), + }); err != nil { + u.coord.rollback(ctx, u.session) + return err + } + } + return nil +} + +// checksumAndFinish computes and validates checksums, then stores them on the session. +// Used by both the TUS and simple PUT paths. +func checksumAndFinish(ctx context.Context, session Session) error { + sha1h, md5h, adler32h, err := calculateChecksums(ctx, session.BinPath()) + if err != nil { + return err + } + info, err := session.GetInfo(ctx) + if err != nil { + return err + } + if checksum := info.MetaData["checksum"]; checksum != "" { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + var checkErr error + switch parts[0] { + case "sha1": + checkErr = checkHash(parts[1], sha1h) + case "md5": + checkErr = checkHash(parts[1], md5h) + case "adler32": + checkErr = checkHash(parts[1], adler32h) + default: + checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if checkErr != nil { + session.Cleanup(true, true) + return checkErr + } + } + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + +// UseIn registers the coordinator as the TUS data store in the composer. +func (c *coordinator) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(c) + composer.UseTerminater(c) + composer.UseConcater(c) + composer.UseLengthDeferrer(c) +} + +// NewUpload is not supported; uploads are initiated via the CS3 API. +func (c *coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { + return nil, errNotImplemented +} + +// GetUpload returns the upload session wrapped in a coordinatedUpload so the +// TUS FinishUpload hook runs the coordinator path rather than the legacy one. +func (c *coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + return &coordinatedUpload{session: session, coord: c}, nil +} + +func (c *coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*coordinatedUpload) +} + +func (c *coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*coordinatedUpload) +} + +func (c *coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*coordinatedUpload) +} diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 93b9d5c64ff..ef7383d8e50 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -18,8 +18,7 @@ // Package upload provides the driver-agnostic upload coordinator: // TUS session management, postprocessing event loop, lifecycle event -// publishing. The Coordinator interface is independent of storage.FS — -// it uses a driver but is not a driver. +// publishing. package upload import ( @@ -54,36 +53,59 @@ func init() { tracer = otel.Tracer("github.com/owncloud/reva/pkg/upload") } +// impersonatingUser extracts the impersonating user from the context user's opaque field. +// Returns nil when no impersonation is in effect. +func impersonatingUser(ctx context.Context) *user.User { + u, ok := ctxpkg.ContextGetUser(ctx) + if !ok || u == nil { + return nil + } + if !utils.ExistsInOpaque(u.Opaque, "impersonating-user") { + return nil + } + iu := &user.User{} + if err := utils.ReadJSONFromOpaque(u.Opaque, "impersonating-user", iu); err != nil { + return nil + } + return iu +} + var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) // Session is the driver-agnostic view of an upload session the Coordinator -// needs. OcisSession satisfies this interface. +// needs. Implementations must be pure state (CRUD): protocol orchestration +// belongs to coordinatedUpload or the coordinator itself. type Session interface { - tusd.Upload storage.UploadSession + + // Data access — delegated to by coordinatedUpload for TUS reads/writes. + GetInfo(ctx context.Context) (tusd.FileInfo, error) + GetReader(ctx context.Context) (io.ReadCloser, error) + WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) + + // Internal coordinator plumbing. ID() string Filename() string Size() int64 SizeDiff() int64 BinPath() string - SpaceGid() string ProviderID() string SpaceID() string NodeID() string NodeExists() bool Dir() string - IsProcessing() bool SpaceOwner() *user.UserId Executant() user.UserId Reference() provider.Reference URL(ctx context.Context) (string, error) SetScanData(result string, date time.Time) Checksums() storage.UploadChecksums + SetChecksums(sha1, md5, adler32 []byte) Metadata() map[string]string Persist(ctx context.Context) error - FinishBytesOnly(ctx context.Context) error - Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) + Cleanup(cleanBin, cleanInfo bool) Context(ctx context.Context) context.Context + // Typed setters used by Coordinator.InitiateUpload to populate a new session // without knowing internal storage key names. SetStorageValue(key, value string) @@ -94,95 +116,37 @@ type Session interface { TouchBin() error } - -// coordinatedUpload wraps a Session so the TUS FinishUpload hook runs the -// coordinator path (checksums + MarkProcessing + BytesReceived) instead of -// the legacy FinishUploadDecomposed path. -type coordinatedUpload struct { - tusd.Upload - session Session - coord *coordinator -} - -func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { - if err := u.session.FinishBytesOnly(ctx); err != nil { - return err - } - // Persist checksums to disk so the postprocessing handler can read them - // from the session store after the BytesReceived event is processed. - if err := u.session.Persist(ctx); err != nil { - return err - } - ref := u.session.Reference() - if err := u.coord.fs.MarkProcessing(ctx, &ref, true, u.session.ID()); err != nil { - // Slot already owned by another session. Clean up this session's bin and - // info files — they will never reach postprocessing. - u.session.Cleanup(false, true, true, false) - return err - } - - metrics.UploadProcessing.Inc() - metrics.UploadSessionsBytesReceived.Inc() - - if !u.coord.async { - // Sync mode (e.g. storage-system): commit inline, no NATS required. - return u.coord.commitSync(ctx, u.session) - } - - if u.session.Size() > 0 { - s, err := u.session.URL(ctx) - if err != nil { - return err - } - if err := events.Publish(ctx, u.coord.pub, events.BytesReceived{ - UploadID: u.session.ID(), - URL: s, - SpaceOwner: u.session.SpaceOwner(), - ExecutingUser: &user.User{ - Id: &user.UserId{ - Type: u.session.Executant().Type, - Idp: u.session.Executant().Idp, - OpaqueId: u.session.Executant().OpaqueId, - }, - }, - ResourceID: &provider.ResourceId{ - StorageId: u.session.ProviderID(), - SpaceId: u.session.SpaceID(), - OpaqueId: u.session.NodeID(), - }, - Filename: u.session.Filename(), - Filesize: uint64(u.session.Size()), - }); err != nil { - return err - } +// rollback unmarks processing, cleans up session files, and deletes the placeholder +// node if it was created by this upload (NodeExists=false at initiation). +func (c *coordinator) rollback(ctx context.Context, session Session) { + ref := session.Reference() + _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) + session.Cleanup(true, true) + if !session.NodeExists() { + _, _ = c.fs.Delete(ctx, &ref) } - return nil } // commitSync runs CommitUpload inline and cleans up the session. -// Used by the sync path (async=false) — called from both coordinatedUpload.FinishUpload -// (TUS) and Coordinator.Upload (simple PUT). -func (c *coordinator)commitSync(ctx context.Context, session Session) error { +// Used by the sync path (async=false) from FinishUpload (TUS) and Upload (simple PUT). +func (c *coordinator) commitSync(ctx context.Context, session Session) error { ref := session.Reference() f, err := os.Open(session.BinPath()) if err != nil { - _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) - session.Cleanup(true, true, true, false) + c.rollback(ctx, session) return err } - defer f.Close() if _, err := c.fs.CommitUpload(ctx, &ref, storage.UploadSource{ Body: f, Length: session.Size(), Metadata: session.Metadata(), Checksums: session.Checksums(), }); err != nil { - _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) - session.Cleanup(true, true, true, false) + c.rollback(ctx, session) return err } _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) - session.Cleanup(false, true, true, false) + session.Cleanup(true, false) metrics.UploadSessionsFinalized.Inc() return nil } @@ -194,14 +158,8 @@ type SessionStore interface { List(ctx context.Context) ([]Session, error) } -// RevisionReverter is an optional interface a storage driver may implement -// to handle RevertRevision postprocessing events. Decomposedfs implements it. -type RevisionReverter interface { - RevertUploadRevision(ctx context.Context, id *provider.ResourceId) error -} - -// Coordinator is the upload orchestrator interface. It is not a storage.FS — -// driver operations are handled separately. +// Coordinator owns the full upload lifecycle: session initiation, TUS data transfer, +// postprocessing event loop, and UploadReady publishing. type Coordinator interface { InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) @@ -224,7 +182,7 @@ type coordinator struct { } // NewCoordinator constructs a Coordinator. Call Start to begin consuming events. -// async=true requires a non-nil pub; fail-fast mirrors decomposedfs.New(). +// async=true requires a non-nil pub. func NewCoordinator( fs storage.FS, store SessionStore, @@ -263,7 +221,6 @@ func (c *coordinator)Start(stream events.Consumer) error { events.PostprocessingStepFinished{}, events.RestartPostprocessing{}, events.CleanUpload{}, - events.RevertRevision{}, ) if err != nil { return err @@ -294,8 +251,6 @@ func (c *coordinator)processEvent(evCtx context.Context, event events.Event) { c.handleRestartPostprocessing(ctx, ev) case events.CleanUpload: c.handleCleanUpload(ctx, ev) - case events.RevertRevision: - c.handleRevertRevision(ctx, ev) default: c.log.Error().Interface("event", ev).Msg("coordinator: unknown event") } @@ -310,6 +265,15 @@ func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events session, err := c.store.Get(ctx, ev.UploadID) if err != nil { log.Error().Err(err).Msg("Failed to get upload") + // Session file gone (e.g. coordinator restarted mid-postprocessing). + // Clear the processing flag directly using the node ID from the event so + // the node does not stay stuck returning 429 Too Early forever. + if ev.ResourceID != nil && ev.ResourceID.GetOpaqueId() != "" { + ref := provider.Reference{ResourceId: ev.ResourceID} + if mpErr := c.fs.MarkProcessing(ctx, &ref, false, ev.UploadID); mpErr != nil { + log.Error().Err(mpErr).Msg("could not unmark processing after lost session") + } + } return } @@ -319,7 +283,7 @@ func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events ref := session.Reference() if _, err := c.fs.GetMD(ctx, &ref, []string{}, []string{}); err != nil { log.Debug().Err(err).Msg("node no longer exists or not accessible; cleaning up") - session.Cleanup(false, true, true, false) + session.Cleanup(true, true) if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { log.Error().Err(err).Msg("could not unmark processing during cleanup of inaccessible node") } @@ -339,8 +303,8 @@ func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events fallthrough case events.PPOutcomeAbort: failed = true - // Only revert node metadata for new files: for overwrites, CommitUpload - // never ran so the node still holds the previous content — nothing to undo. + // Only revert node metadata for new files. For overwrites the node still + // holds the previous content. revertNodeMetadata = !session.NodeExists() keepUpload = true metrics.UploadSessionsAborted.Inc() @@ -371,17 +335,15 @@ func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events } case events.PPOutcomeDelete: failed = true - // Only revert node metadata for new files: for overwrites, CommitUpload - // never ran so the node still holds the previous content — nothing to undo. + // Only revert node metadata for new files. For overwrites the node still + // holds the previous content. revertNodeMetadata = !session.NodeExists() metrics.UploadSessionsDeleted.Inc() } now := time.Now() - // Clean up bin and info files. Node reversion (for aborted new-file uploads) - // is handled below via the FS interface, not inside session.Cleanup. - session.Cleanup(false, !keepUpload, !keepUpload, false) + session.Cleanup(!keepUpload, !keepUpload) nodeRef := session.Reference() if !retryCommit { @@ -443,6 +405,7 @@ func (c *coordinator)handleRestartPostprocessing(ctx context.Context, ev events. log.Error().Err(err).Msg("Failed to get upload") return } + ctx = session.Context(ctx) log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() s, err := session.URL(ctx) if err != nil { @@ -476,7 +439,7 @@ func (c *coordinator)handleCleanUpload(ctx context.Context, ev events.CleanUploa return } ctx = session.Context(ctx) - session.Cleanup(false, !ev.KeepUpload, !ev.KeepUpload, false) + session.Cleanup(!ev.KeepUpload, !ev.KeepUpload) nodeRef := session.Reference() if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { log.Error().Err(err).Msg("could not unmark processing during CleanUpload") @@ -490,22 +453,6 @@ func (c *coordinator)handleCleanUpload(ctx context.Context, ev events.CleanUploa } } -func (c *coordinator)handleRevertRevision(ctx context.Context, ev events.RevertRevision) { - log := c.log.With().Str("event", "RevertRevision").Interface("nodeid", ev.ResourceID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { - log.Debug().Msg("ignoring event for different storage") - return - } - rr, ok := c.fs.(RevisionReverter) - if !ok { - log.Error().Msg("storage driver does not implement RevisionReverter") - return - } - if err := rr.RevertUploadRevision(ctx, ev.ResourceID); err != nil { - log.Error().Err(err).Msg("Failed to revert revision") - } -} - func (c *coordinator)handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { @@ -557,15 +504,18 @@ func (c *coordinator)handlePostprocessingStepFinished(ctx context.Context, ev ev metrics.UploadSessionsScanned.Inc() } -// InitiateUpload creates a node placeholder via TouchFile and builds an upload -// session. For new files TouchFile creates the node; for overwrites it already -// exists and we skip it. All session fields are populated via typed setters so -// the coordinator has no knowledge of internal storage key names. +// InitiateUpload creates a node placeholder via TouchFile and builds an upload session. func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - // Resolve node metadata: determine whether the target exists, get its ID, - // parent ID, space ID, space owner, and the path for Dir. existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) - nodeExists := err == nil + var nodeExists bool + switch err.(type) { + case nil: + nodeExists = true + case errtypes.IsNotFound: + nodeExists = false + default: + return nil, err + } mtime := "" if m, ok := metadata["mtime"]; ok && m != "null" { @@ -576,21 +526,15 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference var spaceOwner *user.UserId if nodeExists { - // Overwrite: node is already there; TouchFile is a no-op for existing nodes. nodeID = existing.GetId().GetOpaqueId() spaceID = existing.GetId().GetSpaceId() parentID = existing.GetParentId().GetOpaqueId() dir = filepath.Dir(existing.GetPath()) nodeName = existing.GetName() - // SpaceOwner is not on ResourceInfo directly; read from a space listing - // or fall back to the owner field (which is the node owner, not space owner). - // The session field is used to populate BytesReceived / UploadReady events. spaceOwner = existing.GetOwner() - // Check quota before accepting the upload. Skip for size-deferred uploads - // (uploadLength == -1) since the final size is unknown at this point. - // For overwrites the existing bytes will be freed on commit, so the net - // required space is uploadLength - existing.Size. + // For overwrites the existing bytes will be freed on commit, so net required + // space is uploadLength - existing.Size. Skip for size-deferred uploads. if uploadLength >= 0 { spaceRef := &provider.Reference{ResourceId: existing.GetId()} if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil { @@ -607,15 +551,12 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference } } } else { - // Check quota before creating the placeholder node. The ref's ResourceId - // points to the space root, which is sufficient for GetQuota. if uploadLength > 0 { if _, _, remaining, qErr := c.fs.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { return nil, errtypes.InsufficientStorage("quota exceeded") } } - // New file: create the placeholder node via TouchFile. result, tfErr := c.fs.TouchFile(ctx, ref, false, mtime) if tfErr != nil { return nil, tfErr @@ -623,10 +564,9 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference nodeID = result.ResourceID.GetOpaqueId() spaceID = result.SpaceID spaceOwner = result.SpaceOwner - // Derive dir and name from the ref path — ref must carry a path for new files. + // Derive dir and name from the ref path (ref must carry a path for new files). dir = filepath.Dir(ref.GetPath()) nodeName = filepath.Base(ref.GetPath()) - // Parent ResourceId is not returned by TouchFile; derive from GetMD on the parent. parentRef := &provider.Reference{ ResourceId: ref.ResourceId, Path: filepath.Dir(ref.GetPath()), @@ -656,6 +596,8 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference session.SetStorageValue("NodeParentId", parentID) if spaceOwner != nil { session.SetStorageValue("SpaceOwnerOrManager", spaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", spaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(spaceOwner.GetType())) } usr := ctxpkg.ContextMustGetUser(ctx) @@ -710,21 +652,32 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference } if err := session.TouchBin(); err != nil { + if !nodeExists { + _, _ = c.fs.Delete(ctx, ref) + } return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) } if err := session.Persist(ctx); err != nil { + session.Cleanup(true, false) + if !nodeExists { + _, _ = c.fs.Delete(ctx, ref) + } return nil, fmt.Errorf("coordinator: could not persist session: %w", err) } + sessionRef := session.Reference() + if err := c.fs.MarkProcessing(ctx, &sessionRef, true, session.ID()); err != nil { + session.Cleanup(true, true) + if !nodeExists { + _, _ = c.fs.Delete(ctx, ref) + } + return nil, fmt.Errorf("coordinator: could not mark processing: %w", err) + } + metrics.UploadSessionsInitiated.Inc() if uploadLength == 0 { - // Zero-length uploads complete immediately: compute checksums on the empty - // bin, commit, and clean up — no postprocessing needed. - if err := session.FinishBytesOnly(ctx); err != nil { - session.Cleanup(false, true, true, false) - return nil, fmt.Errorf("coordinator: zero-length FinishBytesOnly: %w", err) - } + // Zero-length uploads complete immediately without postprocessing. commitRef := session.Reference() if _, err := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ Body: io.NopCloser(bytes.NewReader(nil)), @@ -732,10 +685,11 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference Metadata: session.Metadata(), Checksums: session.Checksums(), }); err != nil { - session.Cleanup(false, true, true, false) + c.rollback(ctx, session) return nil, fmt.Errorf("coordinator: zero-length CommitUpload: %w", err) } - session.Cleanup(false, true, true, false) + _ = c.fs.MarkProcessing(ctx, &commitRef, false, session.ID()) + session.Cleanup(true, true) metrics.UploadSessionsFinalized.Inc() return map[string]string{ "simple": session.ID(), @@ -768,16 +722,12 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff return nil, errtypes.PartialContent(req.Ref.String()) } - if err := session.FinishBytesOnly(ctx); err != nil { + if err := checksumAndFinish(ctx, session); err != nil { + c.rollback(ctx, session) return nil, err } if err := session.Persist(ctx); err != nil { - return nil, err - } - - ref := session.Reference() - if err := c.fs.MarkProcessing(ctx, &ref, true, session.ID()); err != nil { - session.Cleanup(false, true, true, false) + c.rollback(ctx, session) return nil, err } @@ -791,6 +741,7 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff } else { s, err := session.URL(ctx) if err != nil { + c.rollback(ctx, session) return nil, err } if err := events.Publish(ctx, c.pub, events.BytesReceived{ @@ -809,9 +760,11 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff SpaceId: session.SpaceID(), OpaqueId: session.NodeID(), }, - Filename: session.Filename(), - Filesize: uint64(session.Size()), + Filename: session.Filename(), + Filesize: uint64(session.Size()), + ImpersonatingUser: impersonatingUser(ctx), }); err != nil { + c.rollback(ctx, session) return nil, err } } @@ -839,31 +792,15 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff }, nil } -// UseIn registers the coordinator as the TUS data store in the composer. -func (c *coordinator)UseIn(composer *tusd.StoreComposer) { - composer.UseCore(c) - composer.UseTerminater(c) - composer.UseConcater(c) - composer.UseLengthDeferrer(c) -} - -// NewUpload is not supported; uploads are initiated via the CS3 API. -func (c *coordinator)NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { - return nil, errNotImplemented -} - -// GetUpload returns the upload session wrapped in a coordinatedUpload so the -// TUS FinishUpload hook runs the coordinator path rather than the legacy one. -func (c *coordinator)GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - session, err := c.store.Get(ctx, id) - if err != nil { - return nil, err - } - return &coordinatedUpload{Upload: session, session: session, coord: c}, nil -} - // ListUploadSessions returns upload sessions matching the given filter. func (c *coordinator)ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + if filter.ID != nil && *filter.ID != "" { + session, err := c.store.Get(ctx, *filter.ID) + if err != nil { + return nil, err + } + return []storage.UploadSession{session}, nil + } sessions, err := c.store.List(ctx) if err != nil { return nil, err @@ -900,17 +837,3 @@ func (c *coordinator)ListUploadSessions(ctx context.Context, filter storage.Uplo return result, nil } -// AsTerminatableUpload returns the upload as a TerminatableUpload. -func (c *coordinator)AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(tusd.TerminatableUpload) -} - -// AsLengthDeclarableUpload returns the upload as a LengthDeclarableUpload. -func (c *coordinator)AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(tusd.LengthDeclarableUpload) -} - -// AsConcatableUpload returns the upload as a ConcatableUpload. -func (c *coordinator)AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(tusd.ConcatableUpload) -} diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go index d4e46fdc1dd..ed1f4766502 100644 --- a/pkg/upload/filestore.go +++ b/pkg/upload/filestore.go @@ -60,7 +60,7 @@ type FileStore struct { // FileStoreFromDriverConf builds a FileStore from a reva driver config map. // Returns nil if the config carries no root path (driver does not support // coordinated uploads). Each service that mounts the same driver calls this -// independently — the same pattern decomposedfs uses for its Aspects. +// independently. func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { if driverConf == nil { return nil @@ -122,7 +122,7 @@ func NewFileStore(root string, opts TokenOptions, log *zerolog.Logger) *FileStor } // Setup creates the uploads directory eagerly so permission problems are caught -// at startup rather than on the first upload. Mirrors decomposedfs tree.Setup(). +// at startup rather than on the first upload. func (fs *FileStore) Setup() error { return os.MkdirAll(filepath.Join(fs.root, "uploads"), 0700) } diff --git a/pkg/upload/session.go b/pkg/upload/session.go index 32a73233de6..ce307400758 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -51,27 +51,26 @@ import ( const defaultFilePerm = os.FileMode(0664) -// FileSession is a driver-agnostic upload session backed by a .info file and a -// .bin staging file on disk. It satisfies the Session interface so the -// Coordinator can drive it without any knowledge of decomposedfs internals. +// FileSession is the Session implementation for disk-backed uploads. While an +// upload is in progress, incoming bytes are staged in a .bin file and upload +// metadata (size, owner, checksums, etc.) is persisted in a .info file. Both +// survive process restarts, allowing TUS resumption. +// +// In scope: read/write the staged .bin file, persist/load upload metadata in the .info file. +// Out of scope: TUS protocol, checksums, event publishing, postprocessing — those live in coordinatedUpload and coordinator. type FileSession struct { store *FileStore info tusd.FileInfo } -// --- tusd.Upload --- - -// GetInfo returns the TUS FileInfo for this session. func (s *FileSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { return s.info, nil } -// GetReader opens the staged binary file for reading. func (s *FileSession) GetReader(_ context.Context) (io.ReadCloser, error) { return os.Open(s.binPath()) } -// WriteChunk appends src to the staged binary file at the given offset. func (s *FileSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) if err != nil { @@ -87,91 +86,10 @@ func (s *FileSession) WriteChunk(ctx context.Context, offset int64, src io.Reade return n, nil } -// FinishUpload is the tusd hook called after all bytes arrive. -// For the coordinator path this is handled by coordinatedUpload.FinishUpload; -// this method exists to satisfy the tusd.Upload interface. -func (s *FileSession) FinishUpload(ctx context.Context) error { - return s.FinishBytesOnly(ctx) -} - -// FinishBytesOnly computes and validates checksums and stores them on the session. -// It does not commit anything to the storage backend. -func (s *FileSession) FinishBytesOnly(ctx context.Context) error { - sha1h, md5h, adler32h, err := calculateChecksums(ctx, s.binPath()) - if err != nil { - return err - } - - if s.info.MetaData["checksum"] != "" { - parts := strings.SplitN(s.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - var checkErr error - switch parts[0] { - case "sha1": - checkErr = checkHash(parts[1], sha1h) - case "md5": - checkErr = checkHash(parts[1], md5h) - case "adler32": - checkErr = checkHash(parts[1], adler32h) - default: - checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if checkErr != nil { - s.Cleanup(false, true, true, false) - return checkErr - } - } - - s.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) - return nil -} - -// Terminate removes all on-disk state for this session. -func (s *FileSession) Terminate(_ context.Context) error { - s.Cleanup(true, true, true, true) - return nil -} - -// DeclareLength updates the upload length on the session and persists it. -func (s *FileSession) DeclareLength(ctx context.Context, length int64) error { - s.info.Size = length - s.info.SizeIsDeferred = false - return s.Persist(ctx) -} - -// ConcatUploads concatenates the staged binaries of partialUploads into this session's bin. -func (s *FileSession) ConcatUploads(_ context.Context, partialUploads []tusd.Upload) error { - file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partial := range partialUploads { - fs, ok := partial.(*FileSession) - if !ok { - return fmt.Errorf("filestore: ConcatUploads: unexpected upload type %T", partial) - } - src, err := os.Open(fs.binPath()) - if err != nil { - return err - } - _, copyErr := io.Copy(file, src) - src.Close() - if copyErr != nil { - return copyErr - } - } - return nil -} - -// --- storage.UploadSession --- // Purge removes all on-disk state for this session. func (s *FileSession) Purge(ctx context.Context) { - s.Cleanup(true, true, true, true) + s.Cleanup(true, true) } // ScanData returns the AV scan result and scan date stored on the session. @@ -184,8 +102,6 @@ func (s *FileSession) ScanData() (string, time.Time) { return s.info.MetaData["scanResult"], d } -// --- Session (coordinator-specific accessors) --- - // ID returns the upload session ID. func (s *FileSession) ID() string { return s.info.ID @@ -256,6 +172,8 @@ func (s *FileSession) IsProcessing() bool { func (s *FileSession) SpaceOwner() *userpb.UserId { return &userpb.UserId{ OpaqueId: s.info.Storage["SpaceOwnerOrManager"], + Idp: s.info.Storage["SpaceOwnerIdp"], + Type: userpb.UserType(userpb.UserType_value[s.info.Storage["SpaceOwnerType"]]), } } @@ -380,10 +298,8 @@ func (s *FileSession) Persist(ctx context.Context) error { } // Cleanup removes the staged binary and/or info file. -// Unlike OcisSession.Cleanup, this implementation has no knowledge of storage -// nodes — node reverting and unmark-processing are the coordinator's responsibility -// via storage.FS.MarkProcessing / Delete calls. -func (s *FileSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) { +// Node deletion and processing flag changes are the coordinator's responsibility. +func (s *FileSession) Cleanup(cleanBin, cleanInfo bool) { log := s.store.log if cleanBin { if err := os.Remove(s.binPath()); err != nil && !errors.Is(err, os.ErrNotExist) { @@ -433,18 +349,14 @@ func (s *FileSession) URL(_ context.Context) (string, error) { return joinURLParts(s.store.opts.DataGatewayEndpoint, tkn), nil } -// ToFileInfo returns the underlying tusd.FileInfo. func (s *FileSession) ToFileInfo() tusd.FileInfo { return s.info } -// InitiatorID returns the initiator ID stored in the session metadata. func (s *FileSession) InitiatorID() string { return s.info.MetaData["initiatorid"] } -// --- internal helpers --- - func (s *FileSession) binPath() string { return filepath.Join(s.store.root, "uploads", s.info.ID) } From 0292c47629b7c1123a8043bbbccd89e8237aef1c Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 8 Jul 2026 15:17:26 +0200 Subject: [PATCH 10/20] tests --- pkg/upload/coordinator_events_test.go | 498 ++++++++++++++++++++++++ pkg/upload/coordinator_initiate_test.go | 397 +++++++++++++++++++ pkg/upload/coordinator_test.go | 262 +++++++++++++ pkg/upload/coordinator_upload_test.go | 381 ++++++++++++++++++ pkg/upload/filestore_test.go | 312 +++++++++++++++ pkg/upload/mock_fs_test.go | 276 +++++++++++++ pkg/upload/session.go | 4 +- pkg/upload/session_test.go | 456 ++++++++++++++++++++++ 8 files changed, 2584 insertions(+), 2 deletions(-) create mode 100644 pkg/upload/coordinator_events_test.go create mode 100644 pkg/upload/coordinator_initiate_test.go create mode 100644 pkg/upload/coordinator_test.go create mode 100644 pkg/upload/coordinator_upload_test.go create mode 100644 pkg/upload/filestore_test.go create mode 100644 pkg/upload/mock_fs_test.go create mode 100644 pkg/upload/session_test.go diff --git a/pkg/upload/coordinator_events_test.go b/pkg/upload/coordinator_events_test.go new file mode 100644 index 00000000000..9ec65da815e --- /dev/null +++ b/pkg/upload/coordinator_events_test.go @@ -0,0 +1,498 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "os" + "testing" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// TestHandlePostprocessingFinished_Continue covers PPOutcomeContinue. +func TestHandlePostprocessingFinished_Continue(t *testing.T) { + t.Run("happy path: CommitUpload, MarkProcessing(false), UploadReady{Failed:false}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.False(t, ev.Failed) + assert.Equal(t, session.ID(), ev.UploadID) + }) + + t.Run("CommitUpload fails: retryCommit=true, no MarkProcessing, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("disk full")) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + // MarkProcessing must NOT be called (retryCommit=true). + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + }) +} + +// TestHandlePostprocessingFinished_Abort covers PPOutcomeAbort. +func TestHandlePostprocessingFinished_Abort(t *testing.T) { + t.Run("new file: MarkProcessing(false), Delete, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + // Abort + new file: revertNodeMetadata=true → Delete called + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeAbort, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + }) + + t.Run("overwrite: MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeAbort, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + }) +} + +// TestHandlePostprocessingFinished_Delete covers PPOutcomeDelete. +func TestHandlePostprocessingFinished_Delete(t *testing.T) { + t.Run("new file: session cleaned, MarkProcessing(false), Delete, UploadReady{Failed:true}", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeDelete, + }) + + mockFs.AssertExpectations(t) + require.Len(t, pub.events, 1) + ev := pub.events[0].(events.UploadReady) + assert.True(t, ev.Failed) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("overwrite: session cleaned, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return(&provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: "n1", SpaceId: "sp1"}, + }, nil) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeDelete, + }) + + mockFs.AssertExpectations(t) + }) +} + +// TestHandlePostprocessingFinished_EdgeCases covers edge cases. +func TestHandlePostprocessingFinished_EdgeCases(t *testing.T) { + t.Run("different storageId ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingFinished(context.Background(), events.PostprocessingFinished{ + UploadID: "any", + ResourceID: &provider.ResourceId{ + StorageId: "different-mount", + OpaqueId: "node1", + }, + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertNotCalled(t, "GetMD", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + assert.Empty(t, pub.events) + }) + + t.Run("session not found: MarkProcessing via ResourceID ref", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + resID := &provider.ResourceId{ + StorageId: "test-mount", + OpaqueId: "node-orphan", + } + mockFs.On("MarkProcessing", mock.Anything, &provider.Reference{ResourceId: resID}, false, "orphan-id").Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(context.Background(), events.PostprocessingFinished{ + UploadID: "orphan-id", + ResourceID: resID, + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + assert.Empty(t, pub.events) + }) + + t.Run("GetMD fails for node: session cleaned, MarkProcessing, no UploadReady", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ctx := context.Background() + ref := session.Reference() + mockFs.On("GetMD", mock.Anything, &ref, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("n1")) + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handlePostprocessingFinished(ctx, events.PostprocessingFinished{ + UploadID: session.ID(), + Outcome: events.PPOutcomeContinue, + }) + + mockFs.AssertExpectations(t) + assert.Empty(t, pub.events) + assert.NoFileExists(t, session.BinPath()) + }) +} + +// TestHandleRestartPostprocessing covers handleRestartPostprocessing. +func TestHandleRestartPostprocessing(t *testing.T) { + t.Run("happy path: BytesReceived with postprocessing-restart executant", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, _, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handleRestartPostprocessing(context.Background(), events.RestartPostprocessing{ + UploadID: session.ID(), + }) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, session.ID(), ev.UploadID) + assert.Equal(t, "postprocessing-restart", ev.ExecutingUser.GetId().GetOpaqueId()) + }) + + t.Run("session not found: no event published", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, _, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handleRestartPostprocessing(context.Background(), events.RestartPostprocessing{ + UploadID: "nonexistent", + }) + + assert.Empty(t, pub.events) + }) +} + +// TestHandleCleanUpload covers handleCleanUpload. +func TestHandleCleanUpload(t *testing.T) { + t.Run("KeepUpload=false, new file: Cleanup(true,true), MarkProcessing(false), Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + + mockFs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("KeepUpload=true, new file: Cleanup(false,false), MarkProcessing(false), Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: true, + }) + + mockFs.AssertExpectations(t) + // KeepUpload=true means Cleanup(false,false): bin and info stay. + assert.FileExists(t, session.BinPath()) + }) + + t.Run("KeepUpload=false, overwrite: Cleanup, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", true) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + + mockFs.AssertExpectations(t) + }) + + t.Run("Delete returns NotFound: silently ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), errtypes.NotFound("n1")) + + // Should not panic or return error to caller. + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: session.ID(), + KeepUpload: false, + }) + mockFs.AssertExpectations(t) + }) + + t.Run("session not found: no FS calls", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handleCleanUpload(context.Background(), events.CleanUpload{ + UploadID: "nonexistent", + KeepUpload: false, + }) + + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) +} + +// TestHandlePostprocessingStepFinished covers handlePostprocessingStepFinished. +func TestHandlePostprocessingStepFinished(t *testing.T) { + t.Run("antivirus clean scan: SetArbitraryMetadata called, session scan data persisted", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("SetArbitraryMetadata", mock.Anything, &ref, mock.MatchedBy(func(md *provider.ArbitraryMetadata) bool { + return md.Metadata["scanstatus"] == "clean" && md.Metadata["scandate"] != "" + })).Return(nil) + + scanDate := time.Now() + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + Infected: false, + Description: "clean", + Scandate: scanDate, + }, + }) + + mockFs.AssertExpectations(t) + + // Reload session from disk and check scan data was persisted. + reloaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + desc, _ := reloaded.ScanData() + assert.Equal(t, "clean", desc) + }) + + t.Run("antivirus result with ErrorMsg: SetArbitraryMetadata not called", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + ErrorMsg: "scanner unavailable", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("non-antivirus step: no side effects", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepDelay, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + mockFs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("result wrong type: no panic", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + session := newPopulatedSession(t, store, "/dir", "f.txt", "n1", "sp1", false) + + // Should not panic; wrong result type is logged. + require.NotPanics(t, func() { + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: session.ID(), + FinishedStep: events.PPStepAntivirus, + Result: "not-a-virusscan-result", + }) + }) + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("empty UploadID (on-demand scan): return early, no session lookup", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: "", + FinishedStep: events.PPStepAntivirus, + Result: events.VirusscanResult{ + Description: "clean", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("different storageId: ignored", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + + coord.(*coordinator).handlePostprocessingStepFinished(context.Background(), events.PostprocessingStepFinished{ + UploadID: "any", + FinishedStep: events.PPStepAntivirus, + ResourceID: &provider.ResourceId{StorageId: "other-mount"}, + Result: events.VirusscanResult{ + Description: "clean", + }, + }) + + mockFs.AssertNotCalled(t, "SetArbitraryMetadata", mock.Anything, mock.Anything, mock.Anything) + }) +} + +// Ensure userpb is referenced to satisfy the import check. +var _ = userpb.UserType_USER_TYPE_PRIMARY diff --git a/pkg/upload/coordinator_initiate_test.go b/pkg/upload/coordinator_initiate_test.go new file mode 100644 index 00000000000..7e1e33999e7 --- /dev/null +++ b/pkg/upload/coordinator_initiate_test.go @@ -0,0 +1,397 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + tusd "github.com/tus/tusd/v2/pkg/handler" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// authedCtx returns a context carrying a user, as required by InitiateUpload. +func authedCtx() context.Context { + return ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) +} + +// touchFileResult is a convenience helper for a successful TouchFile result. +func touchFileResult(nodeID, spaceID string) *storage.TouchFileResult { + return &storage.TouchFileResult{ + ResourceID: &provider.ResourceId{OpaqueId: nodeID}, + SpaceID: spaceID, + SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + +// ref builds a simple path-only reference. +func ref(path string) *provider.Reference { + return &provider.Reference{Path: path} +} + +// existingNodeInfo builds a ResourceInfo as GetMD would return for an existing node. +func existingNodeInfo(nodeID, spaceID string, size uint64) *provider.ResourceInfo { + return &provider.ResourceInfo{ + Id: &provider.ResourceId{OpaqueId: nodeID, SpaceId: spaceID}, + Name: filepath.Base("/dir/file.txt"), + Path: "/dir/file.txt", + Size: size, + Owner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + +// TestInitiateUpload_NewFile covers new-file scenarios. +func TestInitiateUpload_NewFile(t *testing.T) { + t.Run("happy path creates session files and returns IDs", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + fs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + fs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + ids, err := coord.InitiateUpload(ctx, r, 10, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + assert.Equal(t, ids["simple"], ids["tus"]) + + sessionID := ids["simple"] + binPath := filepath.Join(store.root, "uploads", sessionID) + infoPath := filepath.Join(store.root, "uploads", sessionID+".info") + assert.FileExists(t, binPath) + assert.FileExists(t, infoPath) + }) + + t.Run("quota exceeded returns InsufficientStorage, no TouchFile", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(90), uint64(5), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + _, isInsufficient := err.(errtypes.InsufficientStorage) + assert.True(t, isInsufficient) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("GetQuota failure skips quota check and proceeds", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + fs.On("GetQuota", mock.Anything, r).Return(uint64(0), uint64(0), uint64(0), errors.New("quota unavailable")) + fs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + fs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.NoError(t, err) + fs.AssertCalled(t, "TouchFile", mock.Anything, r, false, "") + }) +} + +// TestInitiateUpload_Overwrite covers overwrite (existing node) scenarios. +func TestInitiateUpload_Overwrite(t *testing.T) { + t.Run("happy path overwrite proceeds with MarkProcessing(true)", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 20) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + // quota ref will be based on existing node id + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(50), uint64(50), nil) + fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + ids, err := coord.InitiateUpload(ctx, r, 30, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("quota exceeded for overwrite", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 5) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + // remaining=90, net_required=100-5=95 > 90 + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(200), uint64(110), uint64(90), nil) + + _, err := coord.InitiateUpload(ctx, r, 100, nil) + require.Error(t, err) + _, isInsufficient := err.(errtypes.InsufficientStorage) + assert.True(t, isInsufficient) + }) + + t.Run("upload smaller than existing: net_required=0, proceeds", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 50) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + spaceRef := &provider.Reference{ResourceId: existing.GetId()} + // remaining=0 but net_required=0, should still succeed + fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(100), uint64(0), nil) + fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, 30, nil) + require.NoError(t, err) + }) + + t.Run("size-deferred (uploadLength=-1): GetQuota not called", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + existing := existingNodeInfo("node1", "space1", 20) + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, -1, map[string]string{"sizedeferred": "true"}) + require.NoError(t, err) + fs.AssertNotCalled(t, "GetQuota", mock.Anything, mock.Anything) + }) +} + +// TestInitiateUpload_ZeroLength covers the zero-length inline commit path. +func TestInitiateUpload_ZeroLength(t *testing.T) { + t.Run("zero-length: CommitUpload called inline, files cleaned", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + + ids, err := coord.InitiateUpload(ctx, r, 0, nil) + require.NoError(t, err) + require.NotEmpty(t, ids["simple"]) + + sessionID := ids["simple"] + binPath := filepath.Join(store.root, "uploads", sessionID) + infoPath := filepath.Join(store.root, "uploads", sessionID+".info") + assert.NoFileExists(t, binPath) + assert.NoFileExists(t, infoPath) + }) + + t.Run("zero-length CommitUpload failure triggers rollback", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("commit failed")) + // rollback: MarkProcessing(false), Delete (ref is session.Reference(), ResourceId-based) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + mockFs.On("Delete", mock.Anything, mock.Anything).Return((*storage.DeleteResult)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, 0, nil) + require.Error(t, err) + }) +} + +// TestInitiateUpload_ErrorPaths covers failure cases during session setup. +func TestInitiateUpload_ErrorPaths(t *testing.T) { + t.Run("GetMD unexpected error returns error without TouchFile", func(t *testing.T) { + root := t.TempDir() + coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errors.New("unexpected")) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("TouchFile failure returns error, no session on disk", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return((*storage.TouchFileResult)(nil), errors.New("touch failed")) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + + // No session files should exist. + infos, globErr := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) + require.NoError(t, globErr) + assert.Empty(t, infos) + }) + + t.Run("MarkProcessing(true) failure: session files cleaned, node deleted", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(errors.New("mark failed")) + mockFs.On("Delete", mock.Anything, r).Return((*storage.DeleteResult)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + + infos, globErr := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) + require.NoError(t, globErr) + assert.Empty(t, infos) + }) + + t.Run("TouchBin failure when uploads dir is missing: node deleted", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + // Remove the uploads directory so TouchBin fails. + require.NoError(t, os.RemoveAll(filepath.Join(store.root, "uploads"))) + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("Delete", mock.Anything, r).Return((*storage.DeleteResult)(nil), nil) + + _, err := coord.InitiateUpload(ctx, r, 10, nil) + require.Error(t, err) + mockFs.AssertCalled(t, "Delete", mock.Anything, r) + }) +} + +// TestInitiateUpload_Metadata covers metadata/checksum validation. +func TestInitiateUpload_Metadata(t *testing.T) { + t.Run("unsupported checksum algorithm returns BadRequest", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "crc32 abc123", + }) + require.Error(t, err) + _, isBad := err.(errtypes.BadRequest) + assert.True(t, isBad) + }) + + t.Run("malformed checksum (no space) returns BadRequest", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "nospace", + }) + require.Error(t, err) + _, isBad := err.(errtypes.BadRequest) + assert.True(t, isBad) + }) + + t.Run("valid sha1 checksum accepted", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + ctx := authedCtx() + r := ref("/dir/file.txt") + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ + "checksum": "sha1 aabbccdd", + }) + require.NoError(t, err) + }) +} + +// noSuchPath returns true when the given path does not exist on disk. +func noSuchPath(path string) bool { + _, err := os.Stat(path) + return errors.Is(err, fs.ErrNotExist) +} + +// Ensure tusd is imported so it stays in go.mod. +var _ = tusd.ErrNotFound diff --git a/pkg/upload/coordinator_test.go b/pkg/upload/coordinator_test.go new file mode 100644 index 00000000000..511c97c0a45 --- /dev/null +++ b/pkg/upload/coordinator_test.go @@ -0,0 +1,262 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/owncloud/reva/v2/pkg/storage" +) + +// TestNewCoordinator covers the constructor's validation logic. +func TestNewCoordinator(t *testing.T) { + root := t.TempDir() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + + t.Run("async without publisher returns error", func(t *testing.T) { + _, err := NewCoordinator(fs, store, nil, true, "m", "g", 1, &log) + require.Error(t, err) + }) + + t.Run("sync without publisher succeeds", func(t *testing.T) { + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 1, &log) + require.NoError(t, err) + require.NotNil(t, coord) + }) + + t.Run("async with publisher succeeds", func(t *testing.T) { + pub := &mockPublisher{} + coord, err := NewCoordinator(fs, store, pub, true, "m", "g", 1, &log) + require.NoError(t, err) + require.NotNil(t, coord) + }) + + t.Run("numConsumers zero defaults to 1", func(t *testing.T) { + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 0, &log) + require.NoError(t, err) + require.NotNil(t, coord) + c := coord.(*coordinator) + assert.Equal(t, 1, c.numConc) + }) +} + +// TestRollback verifies rollback behaviour: unmarks processing, removes session +// files, and (for new files) deletes the placeholder node. +func TestRollback(t *testing.T) { + t.Run("new file: unmarks processing, cleans files, deletes node", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + coord.(*coordinator).rollback(context.Background(), session) + + fs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + infoPath := fileSessionPath(store.root, session.ID()) + assert.NoFileExists(t, infoPath) + }) + + t.Run("overwrite: unmarks processing, cleans files, no delete", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", true) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + coord.(*coordinator).rollback(context.Background(), session) + + fs.AssertExpectations(t) + // Delete must NOT have been called — if it were, mock would record it as unexpected. + }) +} + +// TestCommitSync covers commitSync: happy path, missing .bin, and CommitUpload failure. +func TestCommitSync(t *testing.T) { + t.Run("happy path: commits, unmarks, removes bin, keeps info", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + // Write content to the bin file. + require.NoError(t, os.WriteFile(session.BinPath(), []byte("hello"), 0600)) + // Reload session so offset is correct. + loaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + + ref := loaded.Reference() + fs.On("CommitUpload", mock.Anything, &ref, mock.AnythingOfType("storage.UploadSource")).Return((*provider.ResourceInfo)(nil), nil) + fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) + + err = coord.(*coordinator).commitSync(context.Background(), loaded) + require.NoError(t, err) + + fs.AssertExpectations(t) + assert.NoFileExists(t, loaded.BinPath()) + infoPath := fileSessionPath(store.root, loaded.ID()) + assert.FileExists(t, infoPath) + }) + + t.Run("missing bin triggers rollback and returns error", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + // Remove the bin file to simulate it being missing. + require.NoError(t, os.Remove(session.BinPath())) + + ref := session.Reference() + fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + err := coord.(*coordinator).commitSync(context.Background(), session) + require.Error(t, err) + fs.AssertExpectations(t) + }) + + t.Run("CommitUpload failure triggers rollback and returns error", func(t *testing.T) { + root := t.TempDir() + coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + require.NoError(t, os.WriteFile(session.BinPath(), []byte("data"), 0600)) + loaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + + ref := loaded.Reference() + fs.On("CommitUpload", mock.Anything, &ref, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("commit error")) + // rollback calls MarkProcessing(false) and Delete (new file) + fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) + fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + err = coord.(*coordinator).commitSync(context.Background(), loaded) + require.Error(t, err) + fs.AssertExpectations(t) + }) +} + +// TestListUploadSessions covers filtering logic. +func TestListUploadSessions(t *testing.T) { + root := t.TempDir() + coord, _, store := newTestCoordinatorWithStore(t, root, false, nil) + ctx := context.Background() + + // Create three sessions with different characteristics. + s1 := newPopulatedSession(t, store, "/", "a.txt", "n1", "sp1", false) + s2 := newPopulatedSession(t, store, "/", "b.txt", "n2", "sp2", false) + s3 := newPopulatedSession(t, store, "/", "c.txt", "n3", "sp3", false) + + // Mark s1 as fully received (processing) by aligning offset with size. + // FileSession.IsProcessing() returns true when size==offset && scanResult=="". + // Since size defaults to 0, all sessions with empty bins are "processing" by that + // logic. Set s2 size to 10 so it is NOT processing (offset=0 != size=10). + { + s := s2 + s.SetSize(10) + require.NoError(t, s.Persist(ctx)) + // Touch the bin to keep the store happy (offset will be 0, != size 10). + } + + // Mark s3 as expired by setting expires in the past. + { + s := s3 + past := time.Now().Add(-time.Hour) + s.SetMetadata("expires", fmt.Sprintf("%d", past.Unix())) + require.NoError(t, s.Persist(ctx)) + } + + t.Run("by ID found", func(t *testing.T) { + id := s1.ID() + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{ID: &id}) + require.NoError(t, err) + require.Len(t, sessions, 1) + assert.Equal(t, s1.ID(), sessions[0].ID()) + }) + + t.Run("by ID not found returns error", func(t *testing.T) { + id := "nonexistent-id" + _, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{ID: &id}) + require.Error(t, err) + }) + + t.Run("filter Processing=true", func(t *testing.T) { + tr := true + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Processing: &tr}) + require.NoError(t, err) + // s1 and s3 have size==0==offset; s2 has size=10 != offset=0 + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s1.ID()) + assert.NotContains(t, ids, s2.ID()) + }) + + t.Run("filter Processing=false", func(t *testing.T) { + fl := false + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Processing: &fl}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s2.ID()) + assert.NotContains(t, ids, s1.ID()) + }) + + t.Run("filter Expired=true", func(t *testing.T) { + tr := true + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{Expired: &tr}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s3.ID()) + // s1 and s2 have zero Expires() which is before now, so they may or may not be + // included depending on the zero-time semantics — document that s3 is included. + _ = s1 + }) + + t.Run("no filter returns all", func(t *testing.T) { + sessions, err := coord.ListUploadSessions(ctx, storage.UploadSessionFilter{}) + require.NoError(t, err) + ids := make([]string, len(sessions)) + for i, s := range sessions { + ids[i] = s.ID() + } + assert.Contains(t, ids, s1.ID()) + assert.Contains(t, ids, s2.ID()) + assert.Contains(t, ids, s3.ID()) + }) +} diff --git a/pkg/upload/coordinator_upload_test.go b/pkg/upload/coordinator_upload_test.go new file mode 100644 index 00000000000..613a0762925 --- /dev/null +++ b/pkg/upload/coordinator_upload_test.go @@ -0,0 +1,381 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "bytes" + "context" + "crypto/sha1" //nolint:gosec + "encoding/hex" + "io" + "strings" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// initiateAndGetID calls InitiateUpload on coord and returns the session ID. +// It sets up the expected FS mock calls for a new-file happy path. +func initiateAndGetID(t *testing.T, coord Coordinator, mockFs *mockFS, content string) string { + t.Helper() + ctx := ctxpkg.ContextSetUser(context.Background(), &userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) + r := &provider.Reference{Path: "/dir/file.txt"} + + mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) + mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) + mockFs.On("TouchFile", mock.Anything, r, false, "").Return(&storage.TouchFileResult{ + ResourceID: &provider.ResourceId{OpaqueId: "node1"}, + SpaceID: "space1", + SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, + }, nil) + mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + + ids, err := coord.InitiateUpload(ctx, r, int64(len(content)), nil) + require.NoError(t, err) + return ids["simple"] +} + +// newUploadStore creates a new FileStore at the same root as used by the coordinator. +func newUploadStore(t *testing.T, root string) *FileStore { + t.Helper() + log := zerolog.Nop() + return NewFileStore(root, TokenOptions{ + DownloadEndpoint: "http://dl", + DataGatewayEndpoint: "http://gw", + TransferSharedSecret: "secret", + TransferExpires: 3600, + }, &log) +} + +// TestChecksumAndFinish tests the checksumAndFinish package-level function. +func TestChecksumAndFinish(t *testing.T) { + t.Run("no checksum in metadata: sets checksums on session", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello world" + n, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + require.Equal(t, int64(len(content)), n) + + err = checksumAndFinish(context.Background(), session) + require.NoError(t, err) + + cs := session.Checksums() + assert.NotEmpty(t, cs.SHA1) + assert.NotEmpty(t, cs.MD5) + assert.NotEmpty(t, cs.Adler32) + }) + + t.Run("correct sha1 checksum passes", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello" + _, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + + h := sha1.New() //nolint:gosec + h.Write([]byte(content)) + expected := hex.EncodeToString(h.Sum(nil)) + session.SetMetadata("checksum", "sha1 "+expected) + require.NoError(t, session.Persist(context.Background())) + + require.NoError(t, checksumAndFinish(context.Background(), session)) + }) + + t.Run("wrong sha1 checksum returns ChecksumMismatch and cleans bin", func(t *testing.T) { + root := t.TempDir() + _, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) + + content := "hello" + _, err := session.WriteChunk(context.Background(), 0, strings.NewReader(content)) + require.NoError(t, err) + + session.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") + require.NoError(t, session.Persist(context.Background())) + + err = checksumAndFinish(context.Background(), session) + require.Error(t, err) + _, isMismatch := err.(errtypes.ChecksumMismatch) + assert.True(t, isMismatch) + assert.NoFileExists(t, session.BinPath()) + }) +} + +// TestUpload_Sync tests Upload in sync mode. +func TestUpload_Sync(t *testing.T) { + t.Run("happy path: CommitUpload called, uff invoked, ResourceInfo returned", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "hello" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) + + ctx := context.Background() + var uffSpaceOwner, uffExecutant *userpb.UserId + var uffRef *provider.Reference + uff := func(so, ex *userpb.UserId, r *provider.Reference) { + uffSpaceOwner = so + uffExecutant = ex + uffRef = r + } + + ri, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, uff) + require.NoError(t, err) + require.NotNil(t, ri) + assert.Equal(t, "node1", ri.GetId().GetOpaqueId()) + assert.Equal(t, "space1", ri.GetId().GetSpaceId()) + assert.NotNil(t, uffSpaceOwner) + assert.NotNil(t, uffExecutant) + assert.NotNil(t, uffRef) + }) + + t.Run("size mismatch returns PartialContent", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "hello" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + _, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString("hi")), + Length: int64(len(content)), + }, nil) + require.Error(t, err) + _, isPartial := err.(errtypes.PartialContent) + assert.True(t, isPartial) + }) + + t.Run("session not found returns error", func(t *testing.T) { + root := t.TempDir() + coord, _, _ := newTestCoordinatorWithStore(t, root, false, nil) + + ctx := context.Background() + _, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/nonexistent-id"}, + Body: io.NopCloser(bytes.NewBufferString("data")), + Length: 4, + }, nil) + require.Error(t, err) + }) + + t.Run("checksumAndFinish failure triggers rollback", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "test" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + // Inject a bad checksum into the persisted session. + secondStore := newUploadStore(t, root) + sess, err := secondStore.Get(context.Background(), sessionID) + require.NoError(t, err) + sess.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") + require.NoError(t, sess.Persist(context.Background())) + + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) + mockFs.On("Delete", mock.Anything, mock.Anything).Return((*storage.DeleteResult)(nil), nil) + + ctx := context.Background() + _, err = coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, nil) + require.Error(t, err) + }) +} + +// TestUpload_Async tests Upload in async mode. +func TestUpload_Async(t *testing.T) { + t.Run("happy path: BytesReceived published, uff invoked", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + content := "asyncdata" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + var uffCalled bool + uff := func(_, _ *userpb.UserId, _ *provider.Reference) { uffCalled = true } + + ri, err := coord.Upload(ctx, storage.UploadRequest{ + Ref: &provider.Reference{Path: "/" + sessionID}, + Body: io.NopCloser(bytes.NewBufferString(content)), + Length: int64(len(content)), + }, uff) + require.NoError(t, err) + require.NotNil(t, ri) + assert.True(t, uffCalled) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, sessionID, ev.UploadID) + assert.Equal(t, "file.txt", ev.Filename) + assert.Equal(t, uint64(len(content)), ev.Filesize) + }) +} + +// TestCoordinatedUpload_FinishUpload tests the TUS FinishUpload path. +func TestCoordinatedUpload_FinishUpload(t *testing.T) { + t.Run("sync: CommitUpload called, bin removed", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) + content := "finishme" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, sessionID) + require.NoError(t, err) + _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) + require.NoError(t, err) + + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + mockFs.AssertExpectations(t) + }) + + t.Run("async: BytesReceived published for non-zero size", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, _ := newTestCoordinatorWithStore(t, root, true, pub) + content := "asyncfinish" + sessionID := initiateAndGetID(t, coord, mockFs, content) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, sessionID) + require.NoError(t, err) + _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) + require.NoError(t, err) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + + require.Len(t, pub.events, 1) + ev, ok := pub.events[0].(events.BytesReceived) + require.True(t, ok) + assert.Equal(t, sessionID, ev.UploadID) + }) + + t.Run("async: size=0 no BytesReceived published", func(t *testing.T) { + root := t.TempDir() + pub := &mockPublisher{} + coord, mockFs, store := newTestCoordinatorWithStore(t, root, true, pub) + // Create a session with NodeExists=true and size=0 (overwrite). + session := newPopulatedSession(t, store, "/dir", "zero.txt", "n2", "sp2", true) + session.SetSize(0) + require.NoError(t, session.Persist(context.Background())) + + ctx := context.Background() + up, err := coord.GetUpload(ctx, session.ID()) + require.NoError(t, err) + + mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, session.ID()).Return(nil) + + err = up.FinishUpload(ctx) + require.NoError(t, err) + assert.Empty(t, pub.events) + }) +} + +// TestCoordinatedUpload_Terminate tests the TUS Terminate path. +func TestCoordinatedUpload_Terminate(t *testing.T) { + t.Run("new file: Cleanup, MarkProcessing(false), Delete called", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "term.txt", "n1", "sp1", false) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + mockFs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.Terminate(context.Background())) + + mockFs.AssertExpectations(t) + assert.NoFileExists(t, session.BinPath()) + }) + + t.Run("overwrite: Cleanup, MarkProcessing(false), no Delete", func(t *testing.T) { + root := t.TempDir() + coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "term.txt", "n1", "sp1", true) + + ref := session.Reference() + mockFs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.Terminate(context.Background())) + + mockFs.AssertExpectations(t) + }) +} + +// TestCoordinatedUpload_DeclareLength tests deferred-length declaration. +func TestCoordinatedUpload_DeclareLength(t *testing.T) { + root := t.TempDir() + coord, _, store := newTestCoordinatorWithStore(t, root, false, nil) + session := newPopulatedSession(t, store, "/dir", "dl.txt", "n1", "sp1", false) + session.SetSizeIsDeferred(true) + require.NoError(t, session.Persist(context.Background())) + + up, err := coord.GetUpload(context.Background(), session.ID()) + require.NoError(t, err) + cu := up.(*coordinatedUpload) + require.NoError(t, cu.DeclareLength(context.Background(), 42)) + + reloaded, err := store.Get(context.Background(), session.ID()) + require.NoError(t, err) + assert.Equal(t, int64(42), reloaded.Size()) + info, err := reloaded.GetInfo(context.Background()) + require.NoError(t, err) + assert.False(t, info.SizeIsDeferred) +} diff --git a/pkg/upload/filestore_test.go b/pkg/upload/filestore_test.go new file mode 100644 index 00000000000..8ad5c5b9141 --- /dev/null +++ b/pkg/upload/filestore_test.go @@ -0,0 +1,312 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tusd "github.com/tus/tusd/v2/pkg/handler" +) + +func nopLog() *zerolog.Logger { + l := zerolog.Nop() + return &l +} + +// FileStore.Setup + +func TestFileStoreSetup_CreatesUploadsDir(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + + err := fs.Setup() + + require.NoError(t, err) + info, err := os.Stat(filepath.Join(root, "uploads")) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func TestFileStoreSetup_Idempotent(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + + require.NoError(t, fs.Setup()) + assert.NoError(t, fs.Setup()) +} + +// FileStore.New + +func TestFileStoreNew_NonEmptyID(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + assert.NotEmpty(t, s.ID()) +} + +func TestFileStoreNew_UniqueIDs(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s1 := fs.New(context.Background()) + s2 := fs.New(context.Background()) + + assert.NotEqual(t, s1.ID(), s2.ID()) +} + +func TestFileStoreNew_StorageTypeIsFileStore(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + info, err := s.GetInfo(context.Background()) + require.NoError(t, err) + assert.Equal(t, "FileStore", info.Storage["Type"]) +} + +func TestFileStoreNew_MetaDataIsNotNil(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(context.Background()) + + info, err := s.GetInfo(context.Background()) + require.NoError(t, err) + assert.NotNil(t, info.MetaData) +} + +// FileStore.Get + +func TestFileStoreGet_HappyPath(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + got, err := fs.Get(ctx, s.ID()) + + require.NoError(t, err) + assert.Equal(t, s.ID(), got.ID()) + info, err := got.GetInfo(ctx) + require.NoError(t, err) + assert.Equal(t, "FileStore", info.Storage["Type"]) +} + +func TestFileStoreGet_OffsetFromBinSize(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + payload := []byte("hello world") + require.NoError(t, os.WriteFile(s.binPath(), payload, 0600)) + + got, err := fs.Get(ctx, s.ID()) + + require.NoError(t, err) + assert.Equal(t, int64(len(payload)), got.Offset()) +} + +func TestFileStoreGet_MissingInfoReturnsErrNotFound(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + _, err := fs.Get(context.Background(), "no-such-id") + + assert.ErrorIs(t, err, tusd.ErrNotFound) +} + +func TestFileStoreGet_CorruptInfoReturnsError(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + + // Overwrite the .info with garbage JSON. + require.NoError(t, os.WriteFile(s.infoPath(), []byte("{not valid json"), 0600)) + + _, err := fs.Get(ctx, s.ID()) + + require.Error(t, err) + assert.NotErrorIs(t, err, tusd.ErrNotFound) +} + +func TestFileStoreGet_MissingBinReturnsErrNotFound(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s := fs.New(ctx).(*FileSession) + // Write .info but do NOT create the .bin file. + require.NoError(t, s.Persist(ctx)) + + _, err := fs.Get(ctx, s.ID()) + + assert.ErrorIs(t, err, tusd.ErrNotFound) +} + +// FileStore.List + +func TestFileStoreList_EmptyDir(t *testing.T) { + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + sessions, err := fs.List(context.Background()) + + require.NoError(t, err) + assert.Empty(t, sessions) +} + +func TestFileStoreList_ReturnsBothSessions(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + s1 := fs.New(ctx).(*FileSession) + require.NoError(t, s1.TouchBin()) + require.NoError(t, s1.Persist(ctx)) + + s2 := fs.New(ctx).(*FileSession) + require.NoError(t, s2.TouchBin()) + require.NoError(t, s2.Persist(ctx)) + + sessions, err := fs.List(ctx) + + require.NoError(t, err) + ids := make([]string, 0, len(sessions)) + for _, s := range sessions { + ids = append(ids, s.ID()) + } + assert.ElementsMatch(t, []string{s1.ID(), s2.ID()}, ids) +} + +func TestFileStoreList_SkipsSessionWithMissingBin(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, nopLog()) + require.NoError(t, fs.Setup()) + + good := fs.New(ctx).(*FileSession) + require.NoError(t, good.TouchBin()) + require.NoError(t, good.Persist(ctx)) + + bad := fs.New(ctx).(*FileSession) + require.NoError(t, bad.TouchBin()) + require.NoError(t, bad.Persist(ctx)) + require.NoError(t, os.Remove(bad.binPath())) + + sessions, err := fs.List(ctx) + + require.NoError(t, err) + require.Len(t, sessions, 1) + assert.Equal(t, good.ID(), sessions[0].ID()) +} + +// FileStoreFromDriverConf + +func TestFileStoreFromDriverConf_NilConfigReturnsNil(t *testing.T) { + assert.Nil(t, FileStoreFromDriverConf(nil, nopLog())) +} + +func TestFileStoreFromDriverConf_RootKey(t *testing.T) { + root := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{"root": root}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, root, fs.root) +} + +func TestFileStoreFromDriverConf_UploadDirectoryKey(t *testing.T) { + dir := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{"upload_directory": dir}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, dir, fs.root) +} + +func TestFileStoreFromDriverConf_UploadDirectoryWinsOverRoot(t *testing.T) { + root := t.TempDir() + uploadDir := t.TempDir() + fs := FileStoreFromDriverConf(map[string]interface{}{ + "root": root, + "upload_directory": uploadDir, + }, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, uploadDir, fs.root) +} + +func TestFileStoreFromDriverConf_NeitherKeyReturnsNil(t *testing.T) { + fs := FileStoreFromDriverConf(map[string]interface{}{"some_other_key": "value"}, nopLog()) + + assert.Nil(t, fs) +} + +// NewFileStoreFromConfig + +func TestNewFileStoreFromConfig_UploadDirUsed(t *testing.T) { + uploadDir := t.TempDir() + fs := NewFileStoreFromConfig(uploadDir, map[string]interface{}{"root": "/ignored"}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, uploadDir, fs.root) +} + +func TestNewFileStoreFromConfig_FallsBackToDriverConf(t *testing.T) { + root := t.TempDir() + fs := NewFileStoreFromConfig("", map[string]interface{}{"root": root}, nopLog()) + + require.NotNil(t, fs) + assert.Equal(t, root, fs.root) +} + +func TestNewFileStoreFromConfig_BothEmptyReturnsNil(t *testing.T) { + fs := NewFileStoreFromConfig("", nil, nopLog()) + + assert.Nil(t, fs) +} diff --git a/pkg/upload/mock_fs_test.go b/pkg/upload/mock_fs_test.go new file mode 100644 index 00000000000..f15122a9df5 --- /dev/null +++ b/pkg/upload/mock_fs_test.go @@ -0,0 +1,276 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "io" + "net/url" + "testing" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + goevents "go-micro.dev/v4/events" + + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/storage" +) + +// mockFS implements storage.FS for the coordinator tests. +// Only the methods actually exercised by the coordinator are wired to testify/mock; +// all others panic so accidental calls are caught immediately. +type mockFS struct { + mock.Mock +} + +func (m *mockFS) GetMD(ctx context.Context, ref *provider.Reference, mdKeys, fieldMask []string) (*provider.ResourceInfo, error) { + args := m.Called(ctx, ref, mdKeys, fieldMask) + ri, _ := args.Get(0).(*provider.ResourceInfo) + return ri, args.Error(1) +} + +func (m *mockFS) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, uint64, error) { + args := m.Called(ctx, ref) + return args.Get(0).(uint64), args.Get(1).(uint64), args.Get(2).(uint64), args.Error(3) +} + +func (m *mockFS) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) (*storage.TouchFileResult, error) { + args := m.Called(ctx, ref, markprocessing, mtime) + r, _ := args.Get(0).(*storage.TouchFileResult) + return r, args.Error(1) +} + +func (m *mockFS) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { + args := m.Called(ctx, ref, processing, sessionID) + return args.Error(0) +} + +func (m *mockFS) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { + args := m.Called(ctx, ref, source) + ri, _ := args.Get(0).(*provider.ResourceInfo) + return ri, args.Error(1) +} + +func (m *mockFS) Delete(ctx context.Context, ref *provider.Reference) (*storage.DeleteResult, error) { + args := m.Called(ctx, ref) + dr, _ := args.Get(0).(*storage.DeleteResult) + return dr, args.Error(1) +} + +func (m *mockFS) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { + args := m.Called(ctx, ref, md) + return args.Error(0) +} + +// unimplemented methods panic to catch accidental calls + +func (m *mockFS) Shutdown(_ context.Context) error { panic("not implemented: Shutdown") } + +func (m *mockFS) ListStorageSpaces(_ context.Context, _ []*provider.ListStorageSpacesRequest_Filter, _ bool) ([]*provider.StorageSpace, error) { + panic("not implemented: ListStorageSpaces") +} + +func (m *mockFS) ListFolder(_ context.Context, _ *provider.Reference, _, _ []string) ([]*provider.ResourceInfo, error) { + panic("not implemented: ListFolder") +} + +func (m *mockFS) Download(_ context.Context, _ *provider.Reference, _ func(*provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { + panic("not implemented: Download") +} + +func (m *mockFS) GetPathByID(_ context.Context, _ *provider.ResourceId) (string, error) { + panic("not implemented: GetPathByID") +} + +func (m *mockFS) CreateReference(_ context.Context, _ string, _ *url.URL) error { + panic("not implemented: CreateReference") +} + +func (m *mockFS) CreateDir(_ context.Context, _ *provider.Reference) (*storage.CreateDirResult, error) { + panic("not implemented: CreateDir") +} + +func (m *mockFS) Move(_ context.Context, _, _ *provider.Reference) (*storage.MoveResult, error) { + panic("not implemented: Move") +} + +func (m *mockFS) InitiateUpload(_ context.Context, _ *provider.Reference, _ int64, _ map[string]string) (map[string]string, error) { + panic("not implemented: InitiateUpload") +} + +func (m *mockFS) Upload(_ context.Context, _ storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { + panic("not implemented: Upload") +} + +func (m *mockFS) ListRevisions(_ context.Context, _ *provider.Reference) ([]*provider.FileVersion, error) { + panic("not implemented: ListRevisions") +} + +func (m *mockFS) DownloadRevision(_ context.Context, _ *provider.Reference, _ string, _ func(*provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { + panic("not implemented: DownloadRevision") +} + +func (m *mockFS) RestoreRevision(_ context.Context, _ *provider.Reference, _ string) (*storage.RestoreRevisionResult, error) { + panic("not implemented: RestoreRevision") +} + +func (m *mockFS) ListRecycle(_ context.Context, _ *provider.Reference, _, _ string) ([]*provider.RecycleItem, error) { + panic("not implemented: ListRecycle") +} + +func (m *mockFS) RestoreRecycleItem(_ context.Context, _ *provider.Reference, _, _ string, _ *provider.Reference) (*storage.RestoreRecycleItemResult, error) { + panic("not implemented: RestoreRecycleItem") +} + +func (m *mockFS) PurgeRecycleItem(_ context.Context, _ *provider.Reference, _, _ string) error { + panic("not implemented: PurgeRecycleItem") +} + +func (m *mockFS) EmptyRecycle(_ context.Context, _ *provider.Reference) error { + panic("not implemented: EmptyRecycle") +} + +func (m *mockFS) AddGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: AddGrant") +} + +func (m *mockFS) DenyGrant(_ context.Context, _ *provider.Reference, _ *provider.Grantee) error { + panic("not implemented: DenyGrant") +} + +func (m *mockFS) RemoveGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: RemoveGrant") +} + +func (m *mockFS) UpdateGrant(_ context.Context, _ *provider.Reference, _ *provider.Grant) error { + panic("not implemented: UpdateGrant") +} + +func (m *mockFS) ListGrants(_ context.Context, _ *provider.Reference) ([]*provider.Grant, error) { + panic("not implemented: ListGrants") +} + +func (m *mockFS) UnsetArbitraryMetadata(_ context.Context, _ *provider.Reference, _ []string) error { + panic("not implemented: UnsetArbitraryMetadata") +} + +func (m *mockFS) GetLock(_ context.Context, _ *provider.Reference) (*provider.Lock, error) { + panic("not implemented: GetLock") +} + +func (m *mockFS) SetLock(_ context.Context, _ *provider.Reference, _ *provider.Lock) (*storage.SetLockResult, error) { + panic("not implemented: SetLock") +} + +func (m *mockFS) RefreshLock(_ context.Context, _ *provider.Reference, _ *provider.Lock, _ string) error { + panic("not implemented: RefreshLock") +} + +func (m *mockFS) Unlock(_ context.Context, _ *provider.Reference, _ *provider.Lock) (*storage.UnlockResult, error) { + panic("not implemented: Unlock") +} + +func (m *mockFS) CreateStorageSpace(_ context.Context, _ *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { + panic("not implemented: CreateStorageSpace") +} + +func (m *mockFS) UpdateStorageSpace(_ context.Context, _ *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { + panic("not implemented: UpdateStorageSpace") +} + +func (m *mockFS) DeleteStorageSpace(_ context.Context, _ *provider.DeleteStorageSpaceRequest) (*storage.DeleteStorageSpaceResult, error) { + panic("not implemented: DeleteStorageSpace") +} + +func (m *mockFS) CreateHome(_ context.Context) error { panic("not implemented: CreateHome") } +func (m *mockFS) GetHome(_ context.Context) (string, error) { panic("not implemented: GetHome") } + +// mockPublisher captures published events for inspection in tests. +type mockPublisher struct { + events []interface{} +} + +func (p *mockPublisher) Publish(_ string, msg interface{}, _ ...goevents.PublishOption) error { + p.events = append(p.events, msg) + return nil +} + +// newTestStore creates a FileStore with JWT transfer options rooted at root. +func newTestStore(t *testing.T, root string) *FileStore { + t.Helper() + log := zerolog.Nop() + store := NewFileStore(root, TokenOptions{ + DownloadEndpoint: "http://dl", + DataGatewayEndpoint: "http://gw", + TransferSharedSecret: "secret", + TransferExpires: 3600, + }, &log) + require.NoError(t, store.Setup()) + return store +} + +// newTestCoordinator builds a coordinator backed by a real FileStore at root. +// Returns the coordinator and the mockFS so callers can set up expectations. +func newTestCoordinator(t *testing.T, root string, async bool, pub events.Publisher) (Coordinator, *mockFS) { + t.Helper() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log) + require.NoError(t, err) + return coord, fs +} + +// newTestCoordinatorWithStore is like newTestCoordinator but also returns the FileStore. +func newTestCoordinatorWithStore(t *testing.T, root string, async bool, pub events.Publisher) (Coordinator, *mockFS, *FileStore) { + t.Helper() + log := zerolog.Nop() + store := newTestStore(t, root) + fs := &mockFS{} + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log) + require.NoError(t, err) + return coord, fs, store +} + +// newPopulatedSession creates a persisted session with .bin and .info files on disk. +func newPopulatedSession(t *testing.T, store *FileStore, dir, filename, nodeID, spaceID string, nodeExists bool) Session { + t.Helper() + ctx := context.Background() + s := store.New(ctx) + s.SetMetadata("filename", filename) + s.SetStorageValue("NodeName", filename) + s.SetMetadata("dir", dir) + s.SetStorageValue("Dir", dir) + s.SetStorageValue("NodeId", nodeID) + s.SetStorageValue("SpaceRoot", spaceID) + s.SetMetadata("providerID", "test-provider") + s.SetStorageValue("SpaceOwnerOrManager", "owner1") + if nodeExists { + s.SetStorageValue("NodeExists", "true") + } + s.SetExecutant(&userpb.User{ + Id: &userpb.UserId{OpaqueId: "user1", Idp: "idp"}, + }) + require.NoError(t, s.TouchBin()) + require.NoError(t, s.Persist(ctx)) + return s +} diff --git a/pkg/upload/session.go b/pkg/upload/session.go index ce307400758..5f3b88e9cde 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -180,7 +180,7 @@ func (s *FileSession) SpaceOwner() *userpb.UserId { // Executant returns the user ID of the user who initiated this upload. func (s *FileSession) Executant() userpb.UserId { return userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), + Type: utils.UserTypeMap(s.info.Storage["UserType"]), Idp: s.info.Storage["Idp"], OpaqueId: s.info.Storage["UserId"], } @@ -370,7 +370,7 @@ func (s *FileSession) executantUser() *userpb.User { _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) return &userpb.User{ Id: &userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), + Type: utils.UserTypeMap(s.info.Storage["UserType"]), Idp: s.info.Storage["Idp"], OpaqueId: s.info.Storage["UserId"], }, diff --git a/pkg/upload/session_test.go b/pkg/upload/session_test.go new file mode 100644 index 00000000000..8bcf1167a98 --- /dev/null +++ b/pkg/upload/session_test.go @@ -0,0 +1,456 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "crypto/sha1" //nolint:gosec + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/utils" +) + +func newTestSession(t *testing.T) (*FileSession, *FileStore) { + t.Helper() + log := zerolog.Nop() + fs := NewFileStore(t.TempDir(), TokenOptions{}, &log) + sess := fs.New(context.Background()).(*FileSession) + return sess, fs +} + +// SetMetadata / SetStorageValue / SetSize / SetSizeIsDeferred + +func TestSetMetadata(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "storage-1") + assert.Equal(t, "storage-1", sess.ProviderID()) +} + +func TestSetStorageValue(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetStorageValue("SpaceRoot", "space-abc") + assert.Equal(t, "space-abc", sess.SpaceID()) +} + +func TestSetSize(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSize(1234) + assert.Equal(t, int64(1234), sess.Size()) +} + +func TestSetSizeIsDeferred(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSizeIsDeferred(true) + assert.True(t, sess.info.SizeIsDeferred) + sess.SetSizeIsDeferred(false) + assert.False(t, sess.info.SizeIsDeferred) +} + +// SetExecutant / Executant + +func TestSetExecutant_Executant(t *testing.T) { + sess, _ := newTestSession(t) + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp.example.com", + OpaqueId: "user-42", + Type: userpb.UserType_USER_TYPE_PRIMARY, + }, + Username: "alice", + DisplayName: "Alice", + } + sess.SetExecutant(u) + got := sess.Executant() + assert.Equal(t, "idp.example.com", got.Idp) + assert.Equal(t, "user-42", got.OpaqueId) + assert.Equal(t, userpb.UserType_USER_TYPE_PRIMARY, got.Type) +} + +// NodeExists + +func TestNodeExists(t *testing.T) { + sess, _ := newTestSession(t) + + assert.False(t, sess.NodeExists(), "absent key should return false") + + sess.SetStorageValue("NodeExists", "true") + assert.True(t, sess.NodeExists()) + + sess.SetStorageValue("NodeExists", "false") + assert.False(t, sess.NodeExists()) + + sess.SetStorageValue("NodeExists", "yes") + assert.False(t, sess.NodeExists(), "non-'true' value should return false") +} + +// Checksums / SetChecksums + +func TestChecksums_SetChecksums(t *testing.T) { + sess, _ := newTestSession(t) + sha1bytes := []byte{0x01, 0x02, 0x03, 0x04} + md5bytes := []byte{0x05, 0x06, 0x07, 0x08} + adlerBytes := []byte{0x09, 0x0a, 0x0b, 0x0c} + + sess.SetChecksums(sha1bytes, md5bytes, adlerBytes) + got := sess.Checksums() + assert.Equal(t, sha1bytes, got.SHA1) + assert.Equal(t, md5bytes, got.MD5) + assert.Equal(t, adlerBytes, got.Adler32) +} + +// ScanData / SetScanData + +func TestScanData_SetScanData(t *testing.T) { + sess, _ := newTestSession(t) + + result, date := sess.ScanData() + assert.Empty(t, result, "fresh session should have no scan result") + assert.True(t, date.IsZero(), "fresh session should have zero scan date") + + now := time.Now().Truncate(time.Second) + sess.SetScanData("clean", now) + result, date = sess.ScanData() + assert.Equal(t, "clean", result) + assert.WithinDuration(t, now, date, time.Second) +} + +// Expires + +func TestExpires(t *testing.T) { + sess, _ := newTestSession(t) + assert.True(t, sess.Expires().IsZero(), "absent expires should be zero time") + + want := time.Now().Add(time.Hour).Truncate(time.Second) + sess.SetMetadata("expires", utils.TimeToOCMtime(want)) + got := sess.Expires() + assert.WithinDuration(t, want, got, time.Second) +} + +// IsProcessing + +func TestIsProcessing(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetSize(100) + + assert.False(t, sess.IsProcessing(), "size != offset should not be processing") + + sess.info.Offset = 100 + assert.True(t, sess.IsProcessing(), "size == offset with no scan result should be processing") + + sess.SetScanData("clean", time.Now()) + assert.False(t, sess.IsProcessing(), "scan result set means processing finished") +} + +// Reference + +func TestReference(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "prov-1") + sess.SetStorageValue("SpaceRoot", "space-2") + sess.SetStorageValue("NodeId", "node-3") + + ref := sess.Reference() + require.NotNil(t, ref.ResourceId) + assert.Equal(t, "prov-1", ref.ResourceId.StorageId) + assert.Equal(t, "space-2", ref.ResourceId.SpaceId) + assert.Equal(t, "node-3", ref.ResourceId.OpaqueId) +} + +// Metadata + +func TestMetadata(t *testing.T) { + sess, _ := newTestSession(t) + sess.SetMetadata("providerID", "p1") + sess.SetMetadata("mtime", "12345.0") + sess.SetStorageValue("NodeExists", "true") + sess.SetMetadata("versionsPath", "/v/path") + + m := sess.Metadata() + assert.Equal(t, "p1", m["providerID"]) + assert.Equal(t, "12345.0", m["mtime"]) + assert.Equal(t, "true", m["nodeExists"]) + assert.Equal(t, "/v/path", m["versionsPath"]) + assert.Equal(t, sess.ID(), m["sessionID"]) +} + +// Persist + Get round-trip + +func TestPersist_GetRoundtrip(t *testing.T) { + log := zerolog.Nop() + root := t.TempDir() + fs := NewFileStore(root, TokenOptions{}, &log) + require.NoError(t, fs.Setup()) + + sess := fs.New(context.Background()).(*FileSession) + sess.SetSize(512) + sess.SetMetadata("providerID", "prov-rt") + sess.SetStorageValue("NodeId", "nd-rt") + sess.SetStorageValue("NodeExists", "true") + + require.NoError(t, sess.TouchBin()) + require.NoError(t, sess.Persist(context.Background())) + + loaded, err := fs.Get(context.Background(), sess.ID()) + require.NoError(t, err) + ls := loaded.(*FileSession) + + assert.Equal(t, int64(512), ls.Size()) + assert.Equal(t, "prov-rt", ls.ProviderID()) + assert.Equal(t, "nd-rt", ls.NodeID()) + assert.True(t, ls.NodeExists()) +} + +func TestPersist_CreatesIntermediateDirs(t *testing.T) { + log := zerolog.Nop() + root := filepath.Join(t.TempDir(), "sub", "nested") + fs := NewFileStore(root, TokenOptions{}, &log) + + sess := fs.New(context.Background()).(*FileSession) + sess.SetSize(1) + + err := sess.Persist(context.Background()) + assert.NoError(t, err) + _, statErr := os.Stat(sess.infoPath()) + assert.NoError(t, statErr) +} + +// WriteChunk + +func TestWriteChunk(t *testing.T) { + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + require.NoError(t, sess.TouchBin()) + + n, err := sess.WriteChunk(context.Background(), 0, strings.NewReader("hello")) + require.NoError(t, err) + assert.Equal(t, int64(5), n) + assert.Equal(t, int64(5), sess.Offset()) + + n2, err := sess.WriteChunk(context.Background(), 5, strings.NewReader(" world")) + require.NoError(t, err) + assert.Equal(t, int64(6), n2) + assert.Equal(t, int64(11), sess.Offset()) + + data, err := os.ReadFile(sess.binPath()) + require.NoError(t, err) + assert.Equal(t, "hello world", string(data)) +} + +func TestWriteChunk_NoBinFile(t *testing.T) { + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + + _, err := sess.WriteChunk(context.Background(), 0, strings.NewReader("data")) + assert.Error(t, err) +} + +// Cleanup + +func setupCleanupSession(t *testing.T) *FileSession { + t.Helper() + sess, _ := newTestSession(t) + require.NoError(t, os.MkdirAll(filepath.Dir(sess.binPath()), 0700)) + require.NoError(t, sess.TouchBin()) + require.NoError(t, sess.Persist(context.Background())) + return sess +} + +func TestCleanup_BinOnly(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(true, false) + _, err := os.Stat(sess.binPath()) + assert.True(t, os.IsNotExist(err), "bin should be removed") + _, err = os.Stat(sess.infoPath()) + assert.NoError(t, err, "info should survive") +} + +func TestCleanup_InfoOnly(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(false, true) + _, err := os.Stat(sess.binPath()) + assert.NoError(t, err, "bin should survive") + _, err = os.Stat(sess.infoPath()) + assert.True(t, os.IsNotExist(err), "info should be removed") +} + +func TestCleanup_Both(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(true, true) + _, err := os.Stat(sess.binPath()) + assert.True(t, os.IsNotExist(err), "bin should be removed") + _, err = os.Stat(sess.infoPath()) + assert.True(t, os.IsNotExist(err), "info should be removed") +} + +func TestCleanup_Neither(t *testing.T) { + sess := setupCleanupSession(t) + sess.Cleanup(false, false) + _, err := os.Stat(sess.binPath()) + assert.NoError(t, err, "bin should survive") + _, err = os.Stat(sess.infoPath()) + assert.NoError(t, err, "info should survive") +} + +func TestCleanup_MissingFiles_NoError(t *testing.T) { + sess, _ := newTestSession(t) + assert.NotPanics(t, func() { + sess.Cleanup(true, true) + }) +} + +// calculateChecksums + +func TestCalculateChecksums(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "testfile") + require.NoError(t, os.WriteFile(path, []byte("hello"), 0600)) + + sha1h, md5h, adler32h, err := calculateChecksums(context.Background(), path) + require.NoError(t, err) + + assert.Equal(t, "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", hex.EncodeToString(sha1h.Sum(nil))) + assert.Equal(t, "5d41402abc4b2a76b9719d911017c592", hex.EncodeToString(md5h.Sum(nil))) + assert.Equal(t, "062c0215", hex.EncodeToString(adler32h.Sum(nil))) +} + +func TestCalculateChecksums_MissingFile(t *testing.T) { + _, _, _, err := calculateChecksums(context.Background(), "/nonexistent/path/file.bin") + assert.Error(t, err) +} + +// checkHash + +func TestCheckHash_Correct(t *testing.T) { + h := sha1.New() //nolint:gosec + h.Write([]byte("hello")) + err := checkHash("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", h) + assert.NoError(t, err) +} + +func TestCheckHash_Mismatch(t *testing.T) { + h := sha1.New() //nolint:gosec + h.Write([]byte("hello")) + err := checkHash("0000000000000000000000000000000000000000", h) + require.Error(t, err) + assert.ErrorAs(t, err, new(errtypes.ChecksumMismatch)) +} + +// joinURLParts + +func TestJoinURLParts(t *testing.T) { + tests := []struct { + parts []string + want string + }{ + {[]string{"http://host/", "path"}, "http://host/path"}, + {[]string{"http://host", "path"}, "http://host/path"}, + {[]string{"http://host"}, "http://host"}, + {[]string{"http://host", "a", "b"}, "http://host/a/b"}, + {[]string{"http://host/", "a/", "b"}, "http://host/a/b"}, + } + for _, tc := range tests { + got := joinURLParts(tc.parts...) + assert.Equal(t, tc.want, got, "joinURLParts(%v)", tc.parts) + } +} + +// Context + +func TestContext(t *testing.T) { + log := zerolog.Nop() + fs := NewFileStore(t.TempDir(), TokenOptions{}, &log) + sess := fs.New(context.Background()).(*FileSession) + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: "idp.test", + OpaqueId: "ctx-user", + Type: userpb.UserType_USER_TYPE_PRIMARY, + }, + Username: "ctxuser", + } + sess.SetExecutant(u) + sess.SetMetadata("lockid", "lock-xyz") + sess.SetMetadata("initiatorid", "initiator-abc") + + ctx := sess.Context(context.Background()) + + gotUser, ok := ctxpkg.ContextGetUser(ctx) + require.True(t, ok) + assert.Equal(t, "ctx-user", gotUser.GetId().GetOpaqueId()) + assert.Equal(t, "idp.test", gotUser.GetId().GetIdp()) + + lockID, ok := ctxpkg.ContextGetLockID(ctx) + require.True(t, ok) + assert.Equal(t, "lock-xyz", lockID) + + initiator, ok := ctxpkg.ContextGetInitiator(ctx) + require.True(t, ok) + assert.Equal(t, "initiator-abc", initiator) +} + +// URL + +func TestURL(t *testing.T) { + log := zerolog.Nop() + opts := TokenOptions{ + DownloadEndpoint: "http://download.example.com", + DataGatewayEndpoint: "http://gateway.example.com", + TransferSharedSecret: "s3cr3t", + TransferExpires: 3600, + } + fs := NewFileStore(t.TempDir(), opts, &log) + sess := fs.New(context.Background()).(*FileSession) + + url, err := sess.URL(context.Background()) + require.NoError(t, err) + assert.NotEmpty(t, url) + assert.True(t, strings.HasPrefix(url, "http://gateway.example.com"), "URL should start with DataGatewayEndpoint") +} + +func TestURL_NonEmpty(t *testing.T) { + log := zerolog.Nop() + opts := TokenOptions{ + DataGatewayEndpoint: "http://gw.example.com", + TransferSharedSecret: "secret", + TransferExpires: 60, + } + fs := NewFileStore(t.TempDir(), opts, &log) + sess := fs.New(context.Background()).(*FileSession) + + url1, err := sess.URL(context.Background()) + require.NoError(t, err) + url2, err := sess.URL(context.Background()) + require.NoError(t, err) + + assert.NotEmpty(t, url1) + assert.NotEmpty(t, url2) +} From b51ba604820ff1235f429400a92805332fe5e60d Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Wed, 8 Jul 2026 16:25:00 +0200 Subject: [PATCH 11/20] revert unused changes --- .../utils/decomposedfs/decomposedfs.go | 1 - pkg/storage/utils/decomposedfs/upload.go | 178 +++++------------- .../utils/decomposedfs/upload/session.go | 46 ----- .../utils/decomposedfs/upload/upload.go | 40 ---- pkg/upload/coordinator.go | 51 ----- pkg/upload/filestore.go | 7 + pkg/upload/session.go | 45 ++++- 7 files changed, 96 insertions(+), 272 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 415b7c463bf..f81cdaa9ffa 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -90,7 +90,6 @@ func init() { type Session interface { tusd.Upload storage.UploadSession - upload.Session LockID() string } diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 3ef8e694356..6c41810d2ff 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -468,12 +468,7 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc if err != nil { return nil, errors.Wrap(err, "Decomposedfs: failed to read existing node") } - // nodeExists reports whether the target file existed *before* this upload - // was initiated. When false, the current BlobID on the node is a - // placeholder written by CreateNodeForUpload, not real prior content, so - // no version should be archived and the quota check must treat it as new. - nodeExisted := source.Metadata["nodeExists"] == "true" - if _, err := node.CheckQuota(ctx, n.SpaceRoot, nodeExisted, uint64(old.Blobsize), uint64(source.Length)); err != nil { + if _, err := node.CheckQuota(ctx, n.SpaceRoot, old.BlobID != "", uint64(old.Blobsize), uint64(source.Length)); err != nil { return nil, err } @@ -482,80 +477,49 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc return nil, errors.Wrap(err, "Decomposedfs: failed to read old mtime") } - if !fs.o.DisableVersioning && nodeExisted { - // TODO(OCISDEV-900): remove versionsPath reuse once CreateNodeForUpload is retired. - // CreateNodeForUpload pre-creates the version archive during FinishUploadDecomposed, - // so CommitUpload reuses it to avoid a duplicate. The coordinator never calls - // CreateNodeForUpload, so versionsPath is always empty on the coordinator path - // and this branch is already dead there. - versionPath := source.Metadata["versionsPath"] - if versionPath == "" { - versionPath = fs.lu.InternalPath(n.SpaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - } + if !fs.o.DisableVersioning && old.BlobID != "" { + versionPath := fs.lu.InternalPath(n.SpaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - _, errStat := os.Stat(versionPath) - if errStat != nil { - // Version file does not exist yet; create it. - revFile, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) + revFile, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + if !errors.Is(err, os.ErrExist) { + return nil, errors.Wrap(err, "Decomposedfs: failed to create revision file") + } + // EEXIST: a revision archive at this mtime already exists from a + // prior CommitUpload run. If the archive is byte-identical to the + // live node, it is a leftover from an idempotent retry and can be + // safely reset; otherwise we refuse rather than clobber history. + if err := validateRevisionChecksums(ctx, fs.lu, old, versionPath); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: existing revision archive does not match current node") + } + bID, _, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) if err != nil { - if !errors.Is(err, os.ErrExist) { - return nil, errors.Wrap(err, "Decomposedfs: failed to create revision file") - } - // EEXIST: a revision archive at this mtime already exists from a - // prior CommitUpload run. If the archive is byte-identical to the - // live node, it is a leftover from an idempotent retry and can be - // safely reset; otherwise we refuse rather than clobber history. - if err := validateRevisionChecksums(ctx, fs.lu, old, versionPath); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: existing revision archive does not match current node") - } - bID, _, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to read blob id of existing revision") - } - if err := fs.tp.DeleteBlob(&node.Node{BlobID: bID, SpaceID: n.SpaceID}); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to delete stale revision blob") - } - revFile, err = os.Create(versionPath) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to truncate revision file") - } - revFile.Close() - - if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, - func(name string, value []byte) ([]byte, bool) { - return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || - name == prefixes.TypeAttr || - name == prefixes.BlobIDAttr || - name == prefixes.BlobsizeAttr || - name == prefixes.MTimeAttr - }, f, true); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") - } + return nil, errors.Wrap(err, "Decomposedfs: failed to read blob id of existing revision") + } + if err := fs.tp.DeleteBlob(&node.Node{BlobID: bID, SpaceID: n.SpaceID}); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to delete stale revision blob") + } + revFile, err = os.Create(versionPath) + if err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to truncate revision file") + } + } + revFile.Close() - if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") - } - } else { - revFile.Close() - - if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, - func(name string, value []byte) ([]byte, bool) { - return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || - name == prefixes.TypeAttr || - name == prefixes.BlobIDAttr || - name == prefixes.BlobsizeAttr || - name == prefixes.MTimeAttr - }, f, true); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") - } + if err := fs.lu.CopyMetadataWithSourceLock(ctx, n.InternalPath(), versionPath, + func(name string, value []byte) ([]byte, bool) { + return value, strings.HasPrefix(name, prefixes.ChecksumPrefix) || + name == prefixes.TypeAttr || + name == prefixes.BlobIDAttr || + name == prefixes.BlobsizeAttr || + name == prefixes.MTimeAttr + }, f, true); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to archive current revision") + } - if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") - } - } + if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set revision mtime") } - // If the version file already exists (created by CreateNodeForUpload), - // its metadata was already copied there; nothing more to do. } sizeDiff = source.Length - old.Blobsize @@ -579,60 +543,22 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc } }() - // Determine whether to stomp existing node metadata or only update the - // blob pointer. We should NOT overwrite Blobsize/mtime when: - // (a) a newer in-flight session currently owns the processing slot, OR - // (b) this session's upload was for a new file (nodeExists=false) but the - // node has already been fully committed by a concurrent session. - // TODO(OCISDEV-900): remove newerSessionOwnsNode once CreateNodeForUpload is - // retired. The old path allowed two sessions for the same file to race through - // postprocessing simultaneously. The coordinator serializes this via - // MarkProcessing, so only one session can hold the slot at a time and the race - // described by cases (a) and (b) cannot occur. - sessionID := source.Metadata["sessionID"] - n.ResetXattrsCache() - curProcessingID, _ := n.ProcessingID(ctx) - // Case (a): another session is actively processing. - newerSessionInFlight := curProcessingID != "" && sessionID != "" && curProcessingID != sessionID - // Case (b): node was new at initiation but a concurrent session already - // committed a real blob (processing finished before us). - concurrentCommitWon := !nodeExisted && curProcessingID == "" && old.BlobID != "" && old.BlobID != sessionID - newerSessionOwnsNode := newerSessionInFlight || concurrentCommitWon - if newerSessionOwnsNode { - // Only update the blob pointer; preserve size/mtime set by the newer session. - limitedAttrs := node.Attributes{ - prefixes.BlobIDAttr: []byte(n.BlobID), - } - for k, v := range attrs { - if strings.HasPrefix(k, prefixes.ChecksumPrefix) { - limitedAttrs[k] = v - } - } - if err := n.SetXattrsWithContext(ctx, limitedAttrs, false); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to write blob reference metadata") - } - committed = true - } else { - if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") - } + if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") + } - if err := n.SetXattrsWithContext(ctx, attrs, false); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to write metadata") - } - // Durable commit point: the node metadata now references the new blob. - // Past here the file is committed, so the orphaned-blob cleanup must no - // longer run - a failure in the post-commit steps below leaves the file - // intact and must not delete the referenced blob. - committed = true + if err := n.SetXattrsWithContext(ctx, attrs, false); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to write metadata") + } + // Durable commit point: the node metadata now references the new blob. + // Past here the file is committed, so the orphaned-blob cleanup must no + // longer run - a failure in the post-commit steps below leaves the file + // intact and must not delete the referenced blob. + committed = true - if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") - } + if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to propagate") } - // Update parent tmtime so that etag changes propagate to folder listings - // after a successful commit (previously done in processEvent; absorbed here - // so that all drivers inheriting CommitUpload get it for free). if p, err := n.Parent(ctx); err == nil && p != nil { now := time.Now() _ = p.SetTMTime(ctx, &now) diff --git a/pkg/storage/utils/decomposedfs/upload/session.go b/pkg/storage/utils/decomposedfs/upload/session.go index bc69e59da5d..8d7a58ce68a 100644 --- a/pkg/storage/utils/decomposedfs/upload/session.go +++ b/pkg/storage/utils/decomposedfs/upload/session.go @@ -20,7 +20,6 @@ package upload import ( "context" - "encoding/hex" "encoding/json" "os" "path/filepath" @@ -35,7 +34,6 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/owncloud/reva/v2/pkg/appctx" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" - "github.com/owncloud/reva/v2/pkg/storage" "github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -227,11 +225,6 @@ func (s *OcisSession) Dir() string { return s.info.Storage["Dir"] } -// SpaceGid returns the numeric GID of the space owner, or "" if not set. -func (s *OcisSession) SpaceGid() string { - return s.info.Storage["SpaceGid"] -} - // Size returns the upload size func (s *OcisSession) Size() int64 { return s.info.Size @@ -318,11 +311,6 @@ func (s *OcisSession) binPath() string { return filepath.Join(s.store.root, "uploads", s.info.ID) } -// BinPath returns the path to the staged binary file. -func (s *OcisSession) BinPath() string { - return s.binPath() -} - // infoPath returns the path to the .info file storing the file's info. func (s *OcisSession) infoPath() string { return sessionPath(s.store.root, s.info.ID) @@ -339,40 +327,6 @@ func (s *OcisSession) SetScanData(result string, date time.Time) { s.info.MetaData["scanDate"] = date.Format(time.RFC3339) } -// SetChecksums stores pre-computed checksums so the coordinator can pass them -// to CommitUpload without re-reading the bin file. -func (s *OcisSession) SetChecksums(sha1, md5, adler32 []byte) { - s.info.MetaData["checksumSHA1"] = hex.EncodeToString(sha1) - s.info.MetaData["checksumMD5"] = hex.EncodeToString(md5) - s.info.MetaData["checksumAdler32"] = hex.EncodeToString(adler32) -} - -// Checksums returns the pre-computed checksums stored on the session. -func (s *OcisSession) Checksums() storage.UploadChecksums { - decode := func(key string) []byte { - b, _ := hex.DecodeString(s.info.MetaData[key]) - return b - } - return storage.UploadChecksums{ - SHA1: decode("checksumSHA1"), - MD5: decode("checksumMD5"), - Adler32: decode("checksumAdler32"), - } -} - -// Metadata returns metadata passed to CommitUpload. -// "versionsPath" is pre-created by CreateNodeForUpload so CommitUpload reuses it. -// "sessionID" lets CommitUpload detect whether this session still owns the processing slot. -func (s *OcisSession) Metadata() map[string]string { - return map[string]string{ - "providerID": s.info.MetaData["providerID"], - "mtime": s.info.MetaData["mtime"], - "nodeExists": s.info.Storage["NodeExists"], - "versionsPath": s.info.MetaData["versionsPath"], - "sessionID": s.info.ID, - } -} - // ScanData returns the virus scan data func (s *OcisSession) ScanData() (string, time.Time) { date := s.info.MetaData["scanDate"] diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index d59ce5e1ff1..0cc27b0a664 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -110,43 +110,6 @@ func (session *OcisSession) GetReader(ctx context.Context) (io.ReadCloser, error return os.Open(session.binPath()) } -// FinishBytesOnly computes and validates checksums and stores them on the session. -// It does NOT call CreateNodeForUpload, publish any event, or propagate size. -// The coordinator calls this from its coordinatedUpload.FinishUpload. -func (session *OcisSession) FinishBytesOnly(ctx context.Context) error { - ctx, span := tracer.Start(session.Context(ctx), "FinishBytesOnly") - defer span.End() - - sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) - if err != nil { - return err - } - - if session.info.MetaData["checksum"] != "" { - parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = checkHash(parts[1], sha1h) - case "md5": - err = checkHash(parts[1], md5h) - case "adler32": - err = checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if err != nil { - session.Cleanup(false, true, true, false) - return err - } - } - - session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) - return nil -} - // FinishUpload finishes an upload and moves the file to the internal destination // implements tusd.DataStore interface // returns tusd errors @@ -215,9 +178,6 @@ func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error { prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), } - // store on session so the coordinator can pass them to CommitUpload without re-reading the bin - session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) - // At this point we scope by the space to create the final file in the final location if session.store.um != nil && session.info.Storage["SpaceGid"] != "" { gid, err := strconv.Atoi(session.info.Storage["SpaceGid"]) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index ef7383d8e50..09c43084d77 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -72,50 +72,6 @@ func impersonatingUser(ctx context.Context) *user.User { var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) -// Session is the driver-agnostic view of an upload session the Coordinator -// needs. Implementations must be pure state (CRUD): protocol orchestration -// belongs to coordinatedUpload or the coordinator itself. -type Session interface { - storage.UploadSession - - // Data access — delegated to by coordinatedUpload for TUS reads/writes. - GetInfo(ctx context.Context) (tusd.FileInfo, error) - GetReader(ctx context.Context) (io.ReadCloser, error) - WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) - - // Internal coordinator plumbing. - ID() string - Filename() string - Size() int64 - SizeDiff() int64 - BinPath() string - ProviderID() string - SpaceID() string - NodeID() string - NodeExists() bool - Dir() string - SpaceOwner() *user.UserId - Executant() user.UserId - Reference() provider.Reference - URL(ctx context.Context) (string, error) - SetScanData(result string, date time.Time) - Checksums() storage.UploadChecksums - SetChecksums(sha1, md5, adler32 []byte) - Metadata() map[string]string - Persist(ctx context.Context) error - Cleanup(cleanBin, cleanInfo bool) - Context(ctx context.Context) context.Context - - // Typed setters used by Coordinator.InitiateUpload to populate a new session - // without knowing internal storage key names. - SetStorageValue(key, value string) - SetMetadata(key, value string) - SetSize(size int64) - SetSizeIsDeferred(value bool) - SetExecutant(u *user.User) - TouchBin() error -} - // rollback unmarks processing, cleans up session files, and deletes the placeholder // node if it was created by this upload (NodeExists=false at initiation). func (c *coordinator) rollback(ctx context.Context, session Session) { @@ -151,13 +107,6 @@ func (c *coordinator) commitSync(ctx context.Context, session Session) error { return nil } -// SessionStore abstracts upload-session persistence for the Coordinator. -type SessionStore interface { - New(ctx context.Context) Session - Get(ctx context.Context, id string) (Session, error) - List(ctx context.Context) ([]Session, error) -} - // Coordinator owns the full upload lifecycle: session initiation, TUS data transfer, // postprocessing event loop, and UploadReady publishing. type Coordinator interface { diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go index ed1f4766502..55be9057f16 100644 --- a/pkg/upload/filestore.go +++ b/pkg/upload/filestore.go @@ -43,6 +43,13 @@ type TokenOptions struct { TransferExpires int64 } +// SessionStore abstracts upload-session persistence for the Coordinator. +type SessionStore interface { + New(ctx context.Context) Session + Get(ctx context.Context, id string) (Session, error) + List(ctx context.Context) ([]Session, error) +} + // FileStore is a filesystem-backed SessionStore. Sessions are stored as a pair // of files under /uploads/: // diff --git a/pkg/upload/session.go b/pkg/upload/session.go index 5f3b88e9cde..43881f80767 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -30,7 +30,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "time" @@ -63,6 +62,43 @@ type FileSession struct { info tusd.FileInfo } +// Session is the driver-agnostic view of an upload session the Coordinator +// needs. Implementations must be pure state (CRUD): protocol orchestration +// belongs to coordinatedUpload or the coordinator itself. +type Session interface { + storage.UploadSession + + // Data access — delegated to by coordinatedUpload for TUS reads/writes. + GetInfo(ctx context.Context) (tusd.FileInfo, error) + GetReader(ctx context.Context) (io.ReadCloser, error) + WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) + + // Internal coordinator plumbing. + BinPath() string + ProviderID() string + SpaceID() string + NodeID() string + NodeExists() bool + Dir() string + URL(ctx context.Context) (string, error) + SetScanData(result string, date time.Time) + Checksums() storage.UploadChecksums + SetChecksums(sha1, md5, adler32 []byte) + Metadata() map[string]string + Persist(ctx context.Context) error + Cleanup(cleanBin, cleanInfo bool) + Context(ctx context.Context) context.Context + + // Typed setters used by Coordinator.InitiateUpload to populate a new session + // without knowing internal storage key names. + SetStorageValue(key, value string) + SetMetadata(key, value string) + SetSize(size int64) + SetSizeIsDeferred(value bool) + SetExecutant(u *userpb.User) + TouchBin() error +} + func (s *FileSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { return s.info, nil } @@ -117,12 +153,6 @@ func (s *FileSession) Size() int64 { return s.info.Size } -// SizeDiff returns the size difference computed after upload. -func (s *FileSession) SizeDiff() int64 { - v, _ := strconv.ParseInt(s.info.MetaData["sizeDiff"], 10, 64) - return v -} - // Offset returns the current upload offset. func (s *FileSession) Offset() int64 { return s.info.Offset @@ -226,7 +256,6 @@ func (s *FileSession) Metadata() map[string]string { "mtime": s.info.MetaData["mtime"], "nodeExists": s.info.Storage["NodeExists"], "versionsPath": s.info.MetaData["versionsPath"], - "sessionID": s.info.ID, } } From 54abd5817bb244cd0031c3efe8a62c69ba63c1cf Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Thu, 9 Jul 2026 13:02:59 +0200 Subject: [PATCH 12/20] bug fixes --- internal/http/services/owncloud/ocdav/copy.go | 7 +++ pkg/ocm/storage/received/upload.go | 4 +- pkg/upload/coordinated_upload.go | 17 ++---- pkg/upload/coordinator.go | 59 +++++++++++++------ 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/copy.go b/internal/http/services/owncloud/ocdav/copy.go index f7e3c0d2de2..e51f5b01d96 100644 --- a/internal/http/services/owncloud/ocdav/copy.go +++ b/internal/http/services/owncloud/ocdav/copy.go @@ -666,6 +666,13 @@ func (s *svc) prepareCopy(ctx context.Context, w http.ResponseWriter, r *http.Re return nil } + if !srcStatRes.GetInfo().GetPermissionSet().GetInitiateFileDownload() { + w.WriteHeader(http.StatusForbidden) + b, err := errors.Marshal(http.StatusForbidden, "no permission to copy from this resource", "", "") + errors.HandleWebdavError(log, w, b, err) + return nil + } + dstStatReq := &provider.StatRequest{Ref: dstRef} dstStatRes, err := client.Stat(ctx, dstStatReq) switch { diff --git a/pkg/ocm/storage/received/upload.go b/pkg/ocm/storage/received/upload.go index 1f9f6630e04..a122d77c505 100644 --- a/pkg/ocm/storage/received/upload.go +++ b/pkg/ocm/storage/received/upload.go @@ -76,8 +76,8 @@ func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, up }, nil } -func (d *driver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { - return errtypes.NotSupported("op not supported") +func (d *driver) MarkProcessing(_ context.Context, _ *provider.Reference, _ bool, _ string) error { + return nil } func (d *driver) CommitUpload(ctx context.Context, ref *provider.Reference, source storage.UploadSource) (*provider.ResourceInfo, error) { diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go index 755cd6d55c9..8b63a8a6437 100644 --- a/pkg/upload/coordinated_upload.go +++ b/pkg/upload/coordinated_upload.go @@ -25,10 +25,10 @@ import ( "os" "strings" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" tusd "github.com/tus/tusd/v2/pkg/handler" + ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" @@ -128,17 +128,12 @@ func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { u.coord.rollback(ctx, u.session) return err } + executingUser, _ := ctxpkg.ContextGetUser(ctx) if err := events.Publish(ctx, u.coord.pub, events.BytesReceived{ - UploadID: u.session.ID(), - URL: s, - SpaceOwner: u.session.SpaceOwner(), - ExecutingUser: &user.User{ - Id: &user.UserId{ - Type: u.session.Executant().Type, - Idp: u.session.Executant().Idp, - OpaqueId: u.session.Executant().OpaqueId, - }, - }, + UploadID: u.session.ID(), + URL: s, + SpaceOwner: u.session.SpaceOwner(), + ExecutingUser: executingUser, ResourceID: &provider.ResourceId{ StorageId: u.session.ProviderID(), SpaceId: u.session.SpaceID(), diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 09c43084d77..15f829c9082 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -230,13 +230,15 @@ func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() ref := session.Reference() - if _, err := c.fs.GetMD(ctx, &ref, []string{}, []string{}); err != nil { - log.Debug().Err(err).Msg("node no longer exists or not accessible; cleaning up") - session.Cleanup(true, true) - if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { - log.Error().Err(err).Msg("could not unmark processing during cleanup of inaccessible node") + if _, mdErr := c.fs.GetMD(ctx, &ref, []string{}, []string{}); mdErr != nil { + if _, notFound := mdErr.(errtypes.IsNotFound); notFound { + log.Debug().Err(mdErr).Msg("node deleted during postprocessing; cleaning up") + session.Cleanup(true, true) + if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during cleanup of deleted node") + } + return } - return } var ( @@ -482,6 +484,21 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference nodeName = existing.GetName() spaceOwner = existing.GetOwner() + diskLock, _ := c.fs.GetLock(ctx, ref) + contextLockID, _ := ctxpkg.ContextGetLockID(ctx) + if diskLock != nil { + switch contextLockID { + case "": + return nil, errtypes.Locked(diskLock.LockId) + case diskLock.LockId: + // ok + default: + return nil, errtypes.Aborted("mismatching lock") + } + } else if contextLockID != "" { + return nil, errtypes.Aborted("not locked") + } + // For overwrites the existing bytes will be freed on commit, so net required // space is uploadLength - existing.Size. Skip for size-deferred uploads. if uploadLength >= 0 { @@ -501,13 +518,18 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference } } else { if uploadLength > 0 { - if _, _, remaining, qErr := c.fs.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { + // ref points to the not-yet-existing file; resolve the space root instead so GetQuota finds an existing node. + spaceRef := &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: ref.GetResourceId().GetSpaceId()}} + if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil && remaining < uint64(uploadLength) { return nil, errtypes.InsufficientStorage("quota exceeded") } } result, tfErr := c.fs.TouchFile(ctx, ref, false, mtime) if tfErr != nil { + if _, ok := tfErr.(errtypes.IsNotFound); ok { + return nil, errtypes.PreconditionFailed(tfErr.Error()) + } return nil, tfErr } nodeID = result.ResourceID.GetOpaqueId() @@ -627,6 +649,14 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference if uploadLength == 0 { // Zero-length uploads complete immediately without postprocessing. + if err := checksumAndFinish(ctx, session); err != nil { + c.rollback(ctx, session) + return nil, fmt.Errorf("coordinator: zero-length checksums: %w", err) + } + if err := session.Persist(ctx); err != nil { + c.rollback(ctx, session) + return nil, fmt.Errorf("coordinator: zero-length persist: %w", err) + } commitRef := session.Reference() if _, err := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ Body: io.NopCloser(bytes.NewReader(nil)), @@ -693,17 +723,12 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff c.rollback(ctx, session) return nil, err } + executingUser, _ := ctxpkg.ContextGetUser(ctx) if err := events.Publish(ctx, c.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: session.SpaceOwner(), - ExecutingUser: &user.User{ - Id: &user.UserId{ - Type: session.Executant().Type, - Idp: session.Executant().Idp, - OpaqueId: session.Executant().OpaqueId, - }, - }, + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: executingUser, ResourceID: &provider.ResourceId{ StorageId: session.ProviderID(), SpaceId: session.SpaceID(), From af6094e95f6cfdaaaf8a97750b9b7b194ec6e1f6 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Thu, 9 Jul 2026 15:09:16 +0200 Subject: [PATCH 13/20] TouchFile after upload --- pkg/upload/coordinated_upload.go | 114 +---- pkg/upload/coordinator.go | 540 ++++++----------------- pkg/upload/coordinator_initiate_test.go | 82 +--- pkg/upload/coordinator_postprocessing.go | 327 ++++++++++++++ pkg/upload/coordinator_test.go | 14 +- pkg/upload/coordinator_upload_test.go | 43 +- pkg/upload/mock_fs_test.go | 5 +- pkg/upload/session.go | 1 + 8 files changed, 527 insertions(+), 599 deletions(-) create mode 100644 pkg/upload/coordinator_postprocessing.go diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go index 8b63a8a6437..da7c1828a05 100644 --- a/pkg/upload/coordinated_upload.go +++ b/pkg/upload/coordinated_upload.go @@ -23,30 +23,13 @@ import ( "fmt" "io" "os" - "strings" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" tusd "github.com/tus/tusd/v2/pkg/handler" - - ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" - "github.com/owncloud/reva/v2/pkg/errtypes" - "github.com/owncloud/reva/v2/pkg/events" - "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" ) -// coordinatedUpload is the object the TUS handler talks to during an upload. -// TUS is a resumable upload protocol: clients upload files in chunks via HTTP PATCH, -// can resume after interruption, and optionally delete (Terminate), declare size -// upfront (DeclareLength), or assemble partial uploads (ConcatUploads). -// -// To plug into the TUS handler, a type must implement: -// - tusd.Upload: GetInfo, GetReader, WriteChunk, FinishUpload (core read/write) -// - tusd.TerminatableUpload: Terminate (DELETE support) -// - tusd.LengthDeclarableUpload: DeclareLength (deferred-length uploads) -// - tusd.ConcatableUpload: ConcatUploads (parallel chunk concatenation) -// -// coordinatedUpload owns all upload lifecycle logic (checksums, MarkProcessing, -// event publishing). It delegates raw data access to the Session. +// coordinatedUpload is the TUS interface adapter between the tusd library and the coordinator. +// It implements tusd.Upload, TerminatableUpload, LengthDeclarableUpload, and ConcatableUpload. +// All real upload logic lives in coordinator; this type only translates tusd callbacks. type coordinatedUpload struct { session Session coord *coordinator @@ -65,11 +48,14 @@ func (u *coordinatedUpload) WriteChunk(ctx context.Context, offset int64, src io } func (u *coordinatedUpload) Terminate(ctx context.Context) error { - ref := u.session.Reference() u.session.Cleanup(true, true) - _ = u.coord.fs.MarkProcessing(ctx, &ref, false, u.session.ID()) - if !u.session.NodeExists() { - _, _ = u.coord.fs.Delete(ctx, &ref) + // Node may not exist if Terminate is called before FinishUpload. + ref := u.session.Reference() + if ref.ResourceId.GetOpaqueId() != "" { + _ = u.coord.fs.MarkProcessing(ctx, &ref, false, u.session.ID()) + if !u.session.NodeExists() { + _, _ = u.coord.fs.Delete(ctx, &ref) + } } return nil } @@ -105,85 +91,7 @@ func (u *coordinatedUpload) ConcatUploads(ctx context.Context, partials []tusd.U } func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { - if err := checksumAndFinish(ctx, u.session); err != nil { - u.coord.rollback(ctx, u.session) - return err - } - // Persist checksums so the postprocessing handler can read them after BytesReceived. - if err := u.session.Persist(ctx); err != nil { - u.coord.rollback(ctx, u.session) - return err - } - - metrics.UploadProcessing.Inc() - metrics.UploadSessionsBytesReceived.Inc() - - if !u.coord.async { - return u.coord.commitSync(ctx, u.session) - } - - if u.session.Size() > 0 { - s, err := u.session.URL(ctx) - if err != nil { - u.coord.rollback(ctx, u.session) - return err - } - executingUser, _ := ctxpkg.ContextGetUser(ctx) - if err := events.Publish(ctx, u.coord.pub, events.BytesReceived{ - UploadID: u.session.ID(), - URL: s, - SpaceOwner: u.session.SpaceOwner(), - ExecutingUser: executingUser, - ResourceID: &provider.ResourceId{ - StorageId: u.session.ProviderID(), - SpaceId: u.session.SpaceID(), - OpaqueId: u.session.NodeID(), - }, - Filename: u.session.Filename(), - Filesize: uint64(u.session.Size()), - ImpersonatingUser: impersonatingUser(ctx), - }); err != nil { - u.coord.rollback(ctx, u.session) - return err - } - } - return nil -} - -// checksumAndFinish computes and validates checksums, then stores them on the session. -// Used by both the TUS and simple PUT paths. -func checksumAndFinish(ctx context.Context, session Session) error { - sha1h, md5h, adler32h, err := calculateChecksums(ctx, session.BinPath()) - if err != nil { - return err - } - info, err := session.GetInfo(ctx) - if err != nil { - return err - } - if checksum := info.MetaData["checksum"]; checksum != "" { - parts := strings.SplitN(checksum, " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - var checkErr error - switch parts[0] { - case "sha1": - checkErr = checkHash(parts[1], sha1h) - case "md5": - checkErr = checkHash(parts[1], md5h) - case "adler32": - checkErr = checkHash(parts[1], adler32h) - default: - checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if checkErr != nil { - session.Cleanup(true, true) - return checkErr - } - } - session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) - return nil + return u.coord.finishUpload(ctx, u.session) } // UseIn registers the coordinator as the TUS data store in the composer. diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 15f829c9082..1de3290fee0 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -22,10 +22,8 @@ package upload import ( - "bytes" "context" "fmt" - "io" "os" "path/filepath" "strings" @@ -33,12 +31,12 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/google/uuid" "github.com/rs/zerolog" tusd "github.com/tus/tusd/v2/pkg/handler" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" - "github.com/owncloud/reva/v2/pkg/autoprop" ctxpkg "github.com/owncloud/reva/v2/pkg/ctx" "github.com/owncloud/reva/v2/pkg/errtypes" "github.com/owncloud/reva/v2/pkg/events" @@ -83,9 +81,9 @@ func (c *coordinator) rollback(ctx context.Context, session Session) { } } -// commitSync runs CommitUpload inline and cleans up the session. -// Used by the sync path (async=false) from FinishUpload (TUS) and Upload (simple PUT). -func (c *coordinator) commitSync(ctx context.Context, session Session) error { +// finishSync commits the upload inline, without postprocessing. +// Used when async=false or the upload is empty (size==0). +func (c *coordinator) finishSync(ctx context.Context, session Session) error { ref := session.Reference() f, err := os.Open(session.BinPath()) if err != nil { @@ -102,11 +100,134 @@ func (c *coordinator) commitSync(ctx context.Context, session Session) error { return err } _ = c.fs.MarkProcessing(ctx, &ref, false, session.ID()) - session.Cleanup(true, false) + session.Cleanup(true, true) metrics.UploadSessionsFinalized.Inc() return nil } +// triggerPostprocessing publishes BytesReceived to start async postprocessing. +func (c *coordinator) triggerPostprocessing(ctx context.Context, session Session) error { + s, err := session.URL(ctx) + if err != nil { + c.rollback(ctx, session) + return err + } + executingUser, _ := ctxpkg.ContextGetUser(ctx) + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: executingUser, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + ImpersonatingUser: impersonatingUser(ctx), + }); err != nil { + c.rollback(ctx, session) + return err + } + return nil +} + +// finishUpload is called after all bytes are received (TUS FinishUpload and simple PUT). +// It creates the node, validates checksums, then either commits inline or triggers postprocessing. +func (c *coordinator) finishUpload(ctx context.Context, session Session) error { + if err := c.touchAndMark(ctx, session); err != nil { + return err + } + if err := verifyAndStoreChecksums(ctx, session); err != nil { + c.rollback(ctx, session) + return err + } + if err := session.Persist(ctx); err != nil { + c.rollback(ctx, session) + return err + } + + metrics.UploadProcessing.Inc() + metrics.UploadSessionsBytesReceived.Inc() + + if !c.async || session.Size() == 0 { + return c.finishSync(ctx, session) + } + return c.triggerPostprocessing(ctx, session) +} + +// verifyAndStoreChecksums computes checksums from the staged binary, validates against +// any client-supplied checksum, and stores them on the session. +func verifyAndStoreChecksums(ctx context.Context, session Session) error { + sha1h, md5h, adler32h, err := calculateChecksums(ctx, session.BinPath()) + if err != nil { + return err + } + info, err := session.GetInfo(ctx) + if err != nil { + return err + } + if checksum := info.MetaData["checksum"]; checksum != "" { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + var checkErr error + switch parts[0] { + case "sha1": + checkErr = checkHash(parts[1], sha1h) + case "md5": + checkErr = checkHash(parts[1], md5h) + case "adler32": + checkErr = checkHash(parts[1], adler32h) + default: + checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if checkErr != nil { + session.Cleanup(true, true) + return checkErr + } + } + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + +// touchAndMark creates the node (new files only) and marks it as processing. +// Called from FinishUpload and Upload after all bytes have been received. +func (c *coordinator) touchAndMark(ctx context.Context, session Session) error { + if !session.NodeExists() { + pathRef := &provider.Reference{ + ResourceId: &provider.ResourceId{SpaceId: session.SpaceID()}, + Path: filepath.Join(session.Dir(), session.Filename()), + } + result, err := c.fs.TouchFile(ctx, pathRef, false, session.Metadata()["mtime"]) + if err != nil { + session.Cleanup(true, true) + if _, ok := err.(errtypes.IsNotFound); ok { + return errtypes.PreconditionFailed(err.Error()) + } + return err + } + session.SetStorageValue("NodeId", result.ResourceID.GetOpaqueId()) + session.SetStorageValue("SpaceRoot", result.SpaceID) + if result.SpaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", result.SpaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", result.SpaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(result.SpaceOwner.GetType())) + } + } + nodeRef := session.Reference() + if err := c.fs.MarkProcessing(ctx, &nodeRef, true, session.ID()); err != nil { + session.Cleanup(true, true) + if !session.NodeExists() { + _, _ = c.fs.Delete(ctx, &nodeRef) + } + return err + } + return session.Persist(ctx) +} + // Coordinator owns the full upload lifecycle: session initiation, TUS data transfer, // postprocessing event loop, and UploadReady publishing. type Coordinator interface { @@ -160,302 +281,6 @@ func NewCoordinator( }, nil } -// Start subscribes to the event stream and launches numConsumers goroutines -// that process postprocessing events. -func (c *coordinator)Start(stream events.Consumer) error { - ch, err := events.Consume( - stream, - c.conGroup, - events.PostprocessingFinished{}, - events.PostprocessingStepFinished{}, - events.RestartPostprocessing{}, - events.CleanUpload{}, - ) - if err != nil { - return err - } - for i := 0; i < c.numConc; i++ { - go c.postprocessingLoop(ch) - } - return nil -} - -func (c *coordinator)postprocessingLoop(ch <-chan events.Event) { - for event := range ch { - c.processEvent(context.Background(), event) - } -} - -func (c *coordinator)processEvent(evCtx context.Context, event events.Event) { - ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) - ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) - defer span.End() - - switch ev := event.Event.(type) { - case events.PostprocessingFinished: - c.handlePostprocessingFinished(ctx, ev) - case events.PostprocessingStepFinished: - c.handlePostprocessingStepFinished(ctx, ev) - case events.RestartPostprocessing: - c.handleRestartPostprocessing(ctx, ev) - case events.CleanUpload: - c.handleCleanUpload(ctx, ev) - default: - c.log.Error().Interface("event", ev).Msg("coordinator: unknown event") - } -} - -func (c *coordinator)handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { - log := c.log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { - log.Debug().Msg("ignoring event for different storage") - return - } - session, err := c.store.Get(ctx, ev.UploadID) - if err != nil { - log.Error().Err(err).Msg("Failed to get upload") - // Session file gone (e.g. coordinator restarted mid-postprocessing). - // Clear the processing flag directly using the node ID from the event so - // the node does not stay stuck returning 429 Too Early forever. - if ev.ResourceID != nil && ev.ResourceID.GetOpaqueId() != "" { - ref := provider.Reference{ResourceId: ev.ResourceID} - if mpErr := c.fs.MarkProcessing(ctx, &ref, false, ev.UploadID); mpErr != nil { - log.Error().Err(mpErr).Msg("could not unmark processing after lost session") - } - } - return - } - - ctx = session.Context(ctx) - - log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - ref := session.Reference() - if _, mdErr := c.fs.GetMD(ctx, &ref, []string{}, []string{}); mdErr != nil { - if _, notFound := mdErr.(errtypes.IsNotFound); notFound { - log.Debug().Err(mdErr).Msg("node deleted during postprocessing; cleaning up") - session.Cleanup(true, true) - if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { - log.Error().Err(err).Msg("could not unmark processing during cleanup of deleted node") - } - return - } - } - - var ( - failed bool - revertNodeMetadata bool - keepUpload bool - retryCommit bool - ) - - switch ev.Outcome { - default: - log.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - // Only revert node metadata for new files. For overwrites the node still - // holds the previous content. - revertNodeMetadata = !session.NodeExists() - keepUpload = true - metrics.UploadSessionsAborted.Inc() - case events.PPOutcomeContinue: - f, fopenErr := os.Open(session.BinPath()) - if fopenErr != nil { - log.Error().Err(fopenErr).Msg("could not open staged binary for CommitUpload") - failed = true - keepUpload = true - retryCommit = true - } else { - defer f.Close() - commitRef := session.Reference() - _, commitErr := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ - Body: f, - Length: session.Size(), - Metadata: session.Metadata(), - Checksums: session.Checksums(), - }) - if commitErr != nil { - log.Error().Err(commitErr).Msg("could not commit upload") - failed = true - keepUpload = true - retryCommit = true - } else { - metrics.UploadSessionsFinalized.Inc() - } - } - case events.PPOutcomeDelete: - failed = true - // Only revert node metadata for new files. For overwrites the node still - // holds the previous content. - revertNodeMetadata = !session.NodeExists() - metrics.UploadSessionsDeleted.Inc() - } - - now := time.Now() - - session.Cleanup(!keepUpload, !keepUpload) - - nodeRef := session.Reference() - if !retryCommit { - if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { - log.Error().Err(err).Msg("could not unmark processing after postprocessing finished") - } - if revertNodeMetadata { - if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { - if _, ok := delErr.(errtypes.NotFound); !ok { - log.Error().Err(delErr).Msg("could not delete placeholder node on abort") - } - } - } - } - - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionsPath"] != "" { - isVersion = true - } - } - - if err := events.Publish( - ctx, - c.pub, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - }, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Timestamp: utils.TimeToTS(now), - SpaceOwner: session.SpaceOwner(), - IsVersion: isVersion, - ImpersonatingUser: ev.ImpersonatingUser, - }, - ); err != nil { - log.Error().Err(err).Msg("Failed to publish UploadReady event") - } -} - -func (c *coordinator)handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { - log := c.log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() - session, err := c.store.Get(ctx, ev.UploadID) - if err != nil { - log.Error().Err(err).Msg("Failed to get upload") - return - } - ctx = session.Context(ctx) - log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - s, err := session.URL(ctx) - if err != nil { - log.Error().Err(err).Msg("could not create url") - return - } - - metrics.UploadSessionsRestarted.Inc() - - if err := events.Publish(ctx, c.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: session.SpaceOwner(), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, - ResourceID: &provider.ResourceId{ - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - }); err != nil { - log.Error().Err(err).Msg("Failed to publish BytesReceived event") - } -} - -func (c *coordinator)handleCleanUpload(ctx context.Context, ev events.CleanUpload) { - log := c.log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() - session, err := c.store.Get(ctx, ev.UploadID) - if err != nil { - log.Error().Err(err).Msg("Failed to get upload") - return - } - ctx = session.Context(ctx) - session.Cleanup(!ev.KeepUpload, !ev.KeepUpload) - nodeRef := session.Reference() - if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { - log.Error().Err(err).Msg("could not unmark processing during CleanUpload") - } - if !session.NodeExists() { - if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { - if _, ok := delErr.(errtypes.NotFound); !ok { - log.Error().Err(delErr).Msg("could not delete placeholder node during CleanUpload") - } - } - } -} - -func (c *coordinator)handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { - log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() - if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { - log.Debug().Msg("ignoring event for different storage") - return - } - if ev.FinishedStep != events.PPStepAntivirus { - return - } - - res, ok := ev.Result.(events.VirusscanResult) - if !ok { - log.Error().Msgf("coordinator: unexpected antivirus result type %T", ev.Result) - return - } - if res.ErrorMsg != "" { - return - } - log = c.log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() - - if ev.UploadID == "" { - // on-demand scanning not supported - return - } - - session, err := c.store.Get(ctx, ev.UploadID) - if err != nil { - log.Error().Err(err).Msg("Failed to get upload") - return - } - log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - - session.SetScanData(res.Description, res.Scandate) - if err := session.Persist(ctx); err != nil { - log.Error().Err(err).Msg("Failed to persist scan results") - } - - ctx = session.Context(ctx) - ref := session.Reference() - if err := c.fs.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ - Metadata: map[string]string{ - "scanstatus": res.Description, - "scandate": res.Scandate.Format(time.RFC3339Nano), - }, - }); err != nil { - log.Error().Err(err).Msg("Failed to write scan results to node") - } - - metrics.UploadSessionsScanned.Inc() -} - -// InitiateUpload creates a node placeholder via TouchFile and builds an upload session. func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) var nodeExists bool @@ -468,11 +293,6 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference return nil, err } - mtime := "" - if m, ok := metadata["mtime"]; ok && m != "null" { - mtime = m - } - var nodeID, spaceID, parentID, dir, nodeName string var spaceOwner *user.UserId @@ -518,33 +338,16 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference } } else { if uploadLength > 0 { - // ref points to the not-yet-existing file; resolve the space root instead so GetQuota finds an existing node. - spaceRef := &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: ref.GetResourceId().GetSpaceId()}} - if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil && remaining < uint64(uploadLength) { + if _, _, remaining, qErr := c.fs.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { return nil, errtypes.InsufficientStorage("quota exceeded") } } - result, tfErr := c.fs.TouchFile(ctx, ref, false, mtime) - if tfErr != nil { - if _, ok := tfErr.(errtypes.IsNotFound); ok { - return nil, errtypes.PreconditionFailed(tfErr.Error()) - } - return nil, tfErr - } - nodeID = result.ResourceID.GetOpaqueId() - spaceID = result.SpaceID - spaceOwner = result.SpaceOwner - // Derive dir and name from the ref path (ref must carry a path for new files). + // Pre-generate a node ID; the node is created in FinishUpload once bytes have arrived. + nodeID = uuid.New().String() + spaceID = ref.GetResourceId().GetSpaceId() dir = filepath.Dir(ref.GetPath()) nodeName = filepath.Base(ref.GetPath()) - parentRef := &provider.Reference{ - ResourceId: ref.ResourceId, - Path: filepath.Dir(ref.GetPath()), - } - if parentInfo, pErr := c.fs.GetMD(ctx, parentRef, []string{}, []string{}); pErr == nil { - parentID = parentInfo.GetId().GetOpaqueId() - } } if nodeName == "" { @@ -623,53 +426,20 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference } if err := session.TouchBin(); err != nil { - if !nodeExists { - _, _ = c.fs.Delete(ctx, ref) - } return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) } if err := session.Persist(ctx); err != nil { - session.Cleanup(true, false) - if !nodeExists { - _, _ = c.fs.Delete(ctx, ref) - } - return nil, fmt.Errorf("coordinator: could not persist session: %w", err) - } - - sessionRef := session.Reference() - if err := c.fs.MarkProcessing(ctx, &sessionRef, true, session.ID()); err != nil { session.Cleanup(true, true) - if !nodeExists { - _, _ = c.fs.Delete(ctx, ref) - } - return nil, fmt.Errorf("coordinator: could not mark processing: %w", err) + return nil, fmt.Errorf("coordinator: could not persist session: %w", err) } metrics.UploadSessionsInitiated.Inc() if uploadLength == 0 { // Zero-length uploads complete immediately without postprocessing. - if err := checksumAndFinish(ctx, session); err != nil { - c.rollback(ctx, session) - return nil, fmt.Errorf("coordinator: zero-length checksums: %w", err) - } - if err := session.Persist(ctx); err != nil { - c.rollback(ctx, session) - return nil, fmt.Errorf("coordinator: zero-length persist: %w", err) - } - commitRef := session.Reference() - if _, err := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ - Body: io.NopCloser(bytes.NewReader(nil)), - Length: 0, - Metadata: session.Metadata(), - Checksums: session.Checksums(), - }); err != nil { - c.rollback(ctx, session) - return nil, fmt.Errorf("coordinator: zero-length CommitUpload: %w", err) + if err := c.finishUpload(ctx, session); err != nil { + return nil, err } - _ = c.fs.MarkProcessing(ctx, &commitRef, false, session.ID()) - session.Cleanup(true, true) - metrics.UploadSessionsFinalized.Inc() return map[string]string{ "simple": session.ID(), "tus": session.ID(), @@ -701,48 +471,10 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff return nil, errtypes.PartialContent(req.Ref.String()) } - if err := checksumAndFinish(ctx, session); err != nil { - c.rollback(ctx, session) - return nil, err - } - if err := session.Persist(ctx); err != nil { - c.rollback(ctx, session) + if err := c.finishUpload(ctx, session); err != nil { return nil, err } - metrics.UploadProcessing.Inc() - metrics.UploadSessionsBytesReceived.Inc() - - if !c.async { - if err := c.commitSync(ctx, session); err != nil { - return nil, err - } - } else { - s, err := session.URL(ctx) - if err != nil { - c.rollback(ctx, session) - return nil, err - } - executingUser, _ := ctxpkg.ContextGetUser(ctx) - if err := events.Publish(ctx, c.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: session.SpaceOwner(), - ExecutingUser: executingUser, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - ImpersonatingUser: impersonatingUser(ctx), - }); err != nil { - c.rollback(ctx, session) - return nil, err - } - } - executant := session.Executant() uploadRef := &provider.Reference{ ResourceId: &provider.ResourceId{ diff --git a/pkg/upload/coordinator_initiate_test.go b/pkg/upload/coordinator_initiate_test.go index 7e1e33999e7..5ece11dd832 100644 --- a/pkg/upload/coordinator_initiate_test.go +++ b/pkg/upload/coordinator_initiate_test.go @@ -80,9 +80,6 @@ func TestInitiateUpload_NewFile(t *testing.T) { fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) fs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - fs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - fs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) ids, err := coord.InitiateUpload(ctx, r, 10, nil) require.NoError(t, err) @@ -120,19 +117,16 @@ func TestInitiateUpload_NewFile(t *testing.T) { fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) fs.On("GetQuota", mock.Anything, r).Return(uint64(0), uint64(0), uint64(0), errors.New("quota unavailable")) - fs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - fs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) _, err := coord.InitiateUpload(ctx, r, 10, nil) require.NoError(t, err) - fs.AssertCalled(t, "TouchFile", mock.Anything, r, false, "") + fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) }) } // TestInitiateUpload_Overwrite covers overwrite (existing node) scenarios. func TestInitiateUpload_Overwrite(t *testing.T) { - t.Run("happy path overwrite proceeds with MarkProcessing(true)", func(t *testing.T) { + t.Run("happy path overwrite creates session without touching node", func(t *testing.T) { root := t.TempDir() coord, fs, _ := newTestCoordinatorWithStore(t, root, false, nil) ctx := authedCtx() @@ -140,15 +134,15 @@ func TestInitiateUpload_Overwrite(t *testing.T) { existing := existingNodeInfo("node1", "space1", 20) fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) - // quota ref will be based on existing node id spaceRef := &provider.Reference{ResourceId: existing.GetId()} fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(50), uint64(50), nil) - fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) ids, err := coord.InitiateUpload(ctx, r, 30, nil) require.NoError(t, err) require.NotEmpty(t, ids["simple"]) fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + fs.AssertNotCalled(t, "MarkProcessing", mock.Anything, mock.Anything, mock.Anything, mock.Anything) }) t.Run("quota exceeded for overwrite", func(t *testing.T) { @@ -159,6 +153,7 @@ func TestInitiateUpload_Overwrite(t *testing.T) { existing := existingNodeInfo("node1", "space1", 5) fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) spaceRef := &provider.Reference{ResourceId: existing.GetId()} // remaining=90, net_required=100-5=95 > 90 fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(200), uint64(110), uint64(90), nil) @@ -180,7 +175,7 @@ func TestInitiateUpload_Overwrite(t *testing.T) { spaceRef := &provider.Reference{ResourceId: existing.GetId()} // remaining=0 but net_required=0, should still succeed fs.On("GetQuota", mock.Anything, spaceRef).Return(uint64(100), uint64(100), uint64(0), nil) - fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) _, err := coord.InitiateUpload(ctx, r, 30, nil) require.NoError(t, err) @@ -194,7 +189,7 @@ func TestInitiateUpload_Overwrite(t *testing.T) { existing := existingNodeInfo("node1", "space1", 20) fs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return(existing, nil) - fs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + fs.On("GetLock", mock.Anything, mock.Anything).Return((*provider.Lock)(nil), nil) _, err := coord.InitiateUpload(ctx, r, -1, map[string]string{"sizedeferred": "true"}) require.NoError(t, err) @@ -212,8 +207,7 @@ func TestInitiateUpload_ZeroLength(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResult("node1", "space1"), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) @@ -237,8 +231,7 @@ func TestInitiateUpload_ZeroLength(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResult("node1", "space1"), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), errors.New("commit failed")) // rollback: MarkProcessing(false), Delete (ref is session.Reference(), ResourceId-based) @@ -265,47 +258,7 @@ func TestInitiateUpload_ErrorPaths(t *testing.T) { fs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) }) - t.Run("TouchFile failure returns error, no session on disk", func(t *testing.T) { - root := t.TempDir() - coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) - ctx := authedCtx() - r := ref("/dir/file.txt") - - mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) - mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return((*storage.TouchFileResult)(nil), errors.New("touch failed")) - - _, err := coord.InitiateUpload(ctx, r, 10, nil) - require.Error(t, err) - - // No session files should exist. - infos, globErr := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) - require.NoError(t, globErr) - assert.Empty(t, infos) - }) - - t.Run("MarkProcessing(true) failure: session files cleaned, node deleted", func(t *testing.T) { - root := t.TempDir() - coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) - ctx := authedCtx() - r := ref("/dir/file.txt") - - mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) - mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(errors.New("mark failed")) - mockFs.On("Delete", mock.Anything, r).Return((*storage.DeleteResult)(nil), nil) - - _, err := coord.InitiateUpload(ctx, r, 10, nil) - require.Error(t, err) - - infos, globErr := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) - require.NoError(t, globErr) - assert.Empty(t, infos) - }) - - t.Run("TouchBin failure when uploads dir is missing: node deleted", func(t *testing.T) { + t.Run("TouchBin failure when uploads dir is missing: no node to delete", func(t *testing.T) { root := t.TempDir() coord, mockFs, store := newTestCoordinatorWithStore(t, root, false, nil) ctx := authedCtx() @@ -316,13 +269,11 @@ func TestInitiateUpload_ErrorPaths(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("Delete", mock.Anything, r).Return((*storage.DeleteResult)(nil), nil) _, err := coord.InitiateUpload(ctx, r, 10, nil) require.Error(t, err) - mockFs.AssertCalled(t, "Delete", mock.Anything, r) + mockFs.AssertNotCalled(t, "TouchFile", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockFs.AssertNotCalled(t, "Delete", mock.Anything, mock.Anything) }) } @@ -336,9 +287,6 @@ func TestInitiateUpload_Metadata(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ "checksum": "crc32 abc123", @@ -356,9 +304,6 @@ func TestInitiateUpload_Metadata(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ "checksum": "nospace", @@ -376,9 +321,6 @@ func TestInitiateUpload_Metadata(t *testing.T) { mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(touchFileResult("node1", "space1"), nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) _, err := coord.InitiateUpload(ctx, r, 10, map[string]string{ "checksum": "sha1 aabbccdd", diff --git a/pkg/upload/coordinator_postprocessing.go b/pkg/upload/coordinator_postprocessing.go new file mode 100644 index 00000000000..55000383747 --- /dev/null +++ b/pkg/upload/coordinator_postprocessing.go @@ -0,0 +1,327 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package upload + +import ( + "context" + "os" + "path/filepath" + "time" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/owncloud/reva/v2/pkg/autoprop" + "github.com/owncloud/reva/v2/pkg/errtypes" + "github.com/owncloud/reva/v2/pkg/events" + "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/utils" +) + +// Start subscribes to the event stream and launches numConsumers goroutines +// that process postprocessing events. +func (c *coordinator) Start(stream events.Consumer) error { + ch, err := events.Consume( + stream, + c.conGroup, + events.PostprocessingFinished{}, + events.PostprocessingStepFinished{}, + events.RestartPostprocessing{}, + events.CleanUpload{}, + ) + if err != nil { + return err + } + for i := 0; i < c.numConc; i++ { + go c.postprocessingLoop(ch) + } + return nil +} + +func (c *coordinator) postprocessingLoop(ch <-chan events.Event) { + for event := range ch { + c.processEvent(context.Background(), event) + } +} + +func (c *coordinator) processEvent(evCtx context.Context, event events.Event) { + ctx, span := events.TraceEventConsumerWithTracer(evCtx, tracer, event) + ctx = autoprop.SetMetaToContext(ctx, event.ExtraInfo) + defer span.End() + + switch ev := event.Event.(type) { + case events.PostprocessingFinished: + c.handlePostprocessingFinished(ctx, ev) + case events.PostprocessingStepFinished: + c.handlePostprocessingStepFinished(ctx, ev) + case events.RestartPostprocessing: + c.handleRestartPostprocessing(ctx, ev) + case events.CleanUpload: + c.handleCleanUpload(ctx, ev) + default: + c.log.Error().Interface("event", ev).Msg("coordinator: unknown event") + } +} + +func (c *coordinator) handlePostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished) { + log := c.log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + // Session file gone (e.g. coordinator restarted mid-postprocessing). + // Clear the processing flag directly using the node ID from the event so + // the node does not stay stuck returning 429 Too Early forever. + if ev.ResourceID != nil && ev.ResourceID.GetOpaqueId() != "" { + ref := provider.Reference{ResourceId: ev.ResourceID} + if mpErr := c.fs.MarkProcessing(ctx, &ref, false, ev.UploadID); mpErr != nil { + log.Error().Err(mpErr).Msg("could not unmark processing after lost session") + } + } + return + } + + ctx = session.Context(ctx) + + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + ref := session.Reference() + if _, mdErr := c.fs.GetMD(ctx, &ref, []string{}, []string{}); mdErr != nil { + if _, notFound := mdErr.(errtypes.IsNotFound); notFound { + log.Debug().Err(mdErr).Msg("node deleted during postprocessing; cleaning up") + session.Cleanup(true, true) + if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during cleanup of deleted node") + } + return + } + } + + var ( + failed bool + revertNodeMetadata bool + keepUpload bool + retryCommit bool + ) + + switch ev.Outcome { + default: + log.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") + fallthrough + case events.PPOutcomeAbort: + failed = true + revertNodeMetadata = !session.NodeExists() + keepUpload = true + metrics.UploadSessionsAborted.Inc() + case events.PPOutcomeContinue: + f, fopenErr := os.Open(session.BinPath()) + if fopenErr != nil { + log.Error().Err(fopenErr).Msg("could not open staged binary for CommitUpload") + failed = true + keepUpload = true + retryCommit = true + } else { + defer f.Close() + commitRef := session.Reference() + _, commitErr := c.fs.CommitUpload(ctx, &commitRef, storage.UploadSource{ + Body: f, + Length: session.Size(), + Metadata: session.Metadata(), + Checksums: session.Checksums(), + }) + if commitErr != nil { + log.Error().Err(commitErr).Msg("could not commit upload") + failed = true + keepUpload = true + retryCommit = true + } else { + metrics.UploadSessionsFinalized.Inc() + } + } + case events.PPOutcomeDelete: + failed = true + revertNodeMetadata = !session.NodeExists() + metrics.UploadSessionsDeleted.Inc() + } + + now := time.Now() + + session.Cleanup(!keepUpload, !keepUpload) + + nodeRef := session.Reference() + if !retryCommit { + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing after postprocessing finished") + } + if revertNodeMetadata { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node on abort") + } + } + } + } + + var isVersion bool + if session.NodeExists() { + info, err := session.GetInfo(ctx) + if err == nil && info.MetaData["versionsPath"] != "" { + isVersion = true + } + } + + if err := events.Publish( + ctx, + c.pub, + events.UploadReady{ + UploadID: ev.UploadID, + Failed: failed, + ExecutingUser: ev.ExecutingUser, + Filename: ev.Filename, + FileRef: &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + }, + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Timestamp: utils.TimeToTS(now), + SpaceOwner: session.SpaceOwner(), + IsVersion: isVersion, + ImpersonatingUser: ev.ImpersonatingUser, + }, + ); err != nil { + log.Error().Err(err).Msg("Failed to publish UploadReady event") + } +} + +func (c *coordinator) handleRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing) { + log := c.log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + ctx = session.Context(ctx) + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + s, err := session.URL(ctx) + if err != nil { + log.Error().Err(err).Msg("could not create url") + return + } + + metrics.UploadSessionsRestarted.Inc() + + if err := events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, + ResourceID: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + }); err != nil { + log.Error().Err(err).Msg("Failed to publish BytesReceived event") + } +} + +func (c *coordinator) handleCleanUpload(ctx context.Context, ev events.CleanUpload) { + log := c.log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + ctx = session.Context(ctx) + session.Cleanup(!ev.KeepUpload, !ev.KeepUpload) + nodeRef := session.Reference() + if err := c.fs.MarkProcessing(ctx, &nodeRef, false, session.ID()); err != nil { + log.Error().Err(err).Msg("could not unmark processing during CleanUpload") + } + if !session.NodeExists() { + if _, delErr := c.fs.Delete(ctx, &nodeRef); delErr != nil { + if _, ok := delErr.(errtypes.NotFound); !ok { + log.Error().Err(delErr).Msg("could not delete placeholder node during CleanUpload") + } + } + } +} + +func (c *coordinator) handlePostprocessingStepFinished(ctx context.Context, ev events.PostprocessingStepFinished) { + log := c.log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() + if ev.ResourceID != nil && ev.ResourceID.GetStorageId() != "" && ev.ResourceID.GetStorageId() != c.mountID { + log.Debug().Msg("ignoring event for different storage") + return + } + if ev.FinishedStep != events.PPStepAntivirus { + return + } + + res, ok := ev.Result.(events.VirusscanResult) + if !ok { + log.Error().Msgf("coordinator: unexpected antivirus result type %T", ev.Result) + return + } + if res.ErrorMsg != "" { + return + } + log = c.log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() + + if ev.UploadID == "" { + // on-demand scanning not supported + return + } + + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Msg("Failed to get upload") + return + } + log = c.log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() + + session.SetScanData(res.Description, res.Scandate) + if err := session.Persist(ctx); err != nil { + log.Error().Err(err).Msg("Failed to persist scan results") + } + + ctx = session.Context(ctx) + ref := session.Reference() + if err := c.fs.SetArbitraryMetadata(ctx, &ref, &provider.ArbitraryMetadata{ + Metadata: map[string]string{ + "scanstatus": res.Description, + "scandate": res.Scandate.Format(time.RFC3339Nano), + }, + }); err != nil { + log.Error().Err(err).Msg("Failed to write scan results to node") + } + + metrics.UploadSessionsScanned.Inc() +} diff --git a/pkg/upload/coordinator_test.go b/pkg/upload/coordinator_test.go index 511c97c0a45..d552f4c92e7 100644 --- a/pkg/upload/coordinator_test.go +++ b/pkg/upload/coordinator_test.go @@ -104,9 +104,9 @@ func TestRollback(t *testing.T) { }) } -// TestCommitSync covers commitSync: happy path, missing .bin, and CommitUpload failure. -func TestCommitSync(t *testing.T) { - t.Run("happy path: commits, unmarks, removes bin, keeps info", func(t *testing.T) { +// TestFinishSync covers finishSync: happy path, missing .bin, and CommitUpload failure. +func TestFinishSync(t *testing.T) { + t.Run("happy path: commits, unmarks, removes bin and info", func(t *testing.T) { root := t.TempDir() coord, fs, store := newTestCoordinatorWithStore(t, root, false, nil) session := newPopulatedSession(t, store, "/dir", "file.txt", "node1", "space1", false) @@ -120,13 +120,13 @@ func TestCommitSync(t *testing.T) { fs.On("CommitUpload", mock.Anything, &ref, mock.AnythingOfType("storage.UploadSource")).Return((*provider.ResourceInfo)(nil), nil) fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) - err = coord.(*coordinator).commitSync(context.Background(), loaded) + err = coord.(*coordinator).finishSync(context.Background(), loaded) require.NoError(t, err) fs.AssertExpectations(t) assert.NoFileExists(t, loaded.BinPath()) infoPath := fileSessionPath(store.root, loaded.ID()) - assert.FileExists(t, infoPath) + assert.NoFileExists(t, infoPath) }) t.Run("missing bin triggers rollback and returns error", func(t *testing.T) { @@ -140,7 +140,7 @@ func TestCommitSync(t *testing.T) { fs.On("MarkProcessing", mock.Anything, &ref, false, session.ID()).Return(nil) fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) - err := coord.(*coordinator).commitSync(context.Background(), session) + err := coord.(*coordinator).finishSync(context.Background(), session) require.Error(t, err) fs.AssertExpectations(t) }) @@ -159,7 +159,7 @@ func TestCommitSync(t *testing.T) { fs.On("MarkProcessing", mock.Anything, &ref, false, loaded.ID()).Return(nil) fs.On("Delete", mock.Anything, &ref).Return((*storage.DeleteResult)(nil), nil) - err = coord.(*coordinator).commitSync(context.Background(), loaded) + err = coord.(*coordinator).finishSync(context.Background(), loaded) require.Error(t, err) fs.AssertExpectations(t) }) diff --git a/pkg/upload/coordinator_upload_test.go b/pkg/upload/coordinator_upload_test.go index 613a0762925..2183e72d238 100644 --- a/pkg/upload/coordinator_upload_test.go +++ b/pkg/upload/coordinator_upload_test.go @@ -42,6 +42,8 @@ import ( // initiateAndGetID calls InitiateUpload on coord and returns the session ID. // It sets up the expected FS mock calls for a new-file happy path. +// TouchFile and MarkProcessing(true) are NOT registered here — they happen in +// FinishUpload/Upload, not InitiateUpload. Callers must register them as needed. func initiateAndGetID(t *testing.T, coord Coordinator, mockFs *mockFS, content string) string { t.Helper() ctx := ctxpkg.ContextSetUser(context.Background(), &userpb.User{ @@ -51,19 +53,21 @@ func initiateAndGetID(t *testing.T, coord Coordinator, mockFs *mockFS, content s mockFs.On("GetMD", mock.Anything, r, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")) mockFs.On("GetQuota", mock.Anything, r).Return(uint64(100), uint64(50), uint64(50), nil) - mockFs.On("TouchFile", mock.Anything, r, false, "").Return(&storage.TouchFileResult{ - ResourceID: &provider.ResourceId{OpaqueId: "node1"}, - SpaceID: "space1", - SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, - }, nil) - mockFs.On("GetMD", mock.Anything, mock.Anything, []string{}, []string{}).Return((*provider.ResourceInfo)(nil), errtypes.NotFound("")).Maybe() - mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) ids, err := coord.InitiateUpload(ctx, r, int64(len(content)), nil) require.NoError(t, err) return ids["simple"] } +// touchFileResult returns the mock TouchFile result used by upload tests. +func touchFileResultForUpload() *storage.TouchFileResult { + return &storage.TouchFileResult{ + ResourceID: &provider.ResourceId{OpaqueId: "node1", SpaceId: "space1"}, + SpaceID: "space1", + SpaceOwner: &userpb.UserId{OpaqueId: "owner1"}, + } +} + // newUploadStore creates a new FileStore at the same root as used by the coordinator. func newUploadStore(t *testing.T, root string) *FileStore { t.Helper() @@ -76,8 +80,8 @@ func newUploadStore(t *testing.T, root string) *FileStore { }, &log) } -// TestChecksumAndFinish tests the checksumAndFinish package-level function. -func TestChecksumAndFinish(t *testing.T) { +// TestVerifyAndStoreChecksums tests the verifyAndStoreChecksums package-level function. +func TestVerifyAndStoreChecksums(t *testing.T) { t.Run("no checksum in metadata: sets checksums on session", func(t *testing.T) { root := t.TempDir() _, _, store := newTestCoordinatorWithStore(t, root, false, nil) @@ -88,7 +92,7 @@ func TestChecksumAndFinish(t *testing.T) { require.NoError(t, err) require.Equal(t, int64(len(content)), n) - err = checksumAndFinish(context.Background(), session) + err = verifyAndStoreChecksums(context.Background(), session) require.NoError(t, err) cs := session.Checksums() @@ -112,7 +116,7 @@ func TestChecksumAndFinish(t *testing.T) { session.SetMetadata("checksum", "sha1 "+expected) require.NoError(t, session.Persist(context.Background())) - require.NoError(t, checksumAndFinish(context.Background(), session)) + require.NoError(t, verifyAndStoreChecksums(context.Background(), session)) }) t.Run("wrong sha1 checksum returns ChecksumMismatch and cleans bin", func(t *testing.T) { @@ -127,7 +131,7 @@ func TestChecksumAndFinish(t *testing.T) { session.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") require.NoError(t, session.Persist(context.Background())) - err = checksumAndFinish(context.Background(), session) + err = verifyAndStoreChecksums(context.Background(), session) require.Error(t, err) _, isMismatch := err.(errtypes.ChecksumMismatch) assert.True(t, isMismatch) @@ -143,6 +147,8 @@ func TestUpload_Sync(t *testing.T) { content := "hello" sessionID := initiateAndGetID(t, coord, mockFs, content) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, mock.AnythingOfType("string")).Return(nil) @@ -199,7 +205,7 @@ func TestUpload_Sync(t *testing.T) { require.Error(t, err) }) - t.Run("checksumAndFinish failure triggers rollback", func(t *testing.T) { + t.Run("verifyAndStoreChecksums failure triggers rollback", func(t *testing.T) { root := t.TempDir() coord, mockFs, _ := newTestCoordinatorWithStore(t, root, false, nil) content := "test" @@ -212,6 +218,8 @@ func TestUpload_Sync(t *testing.T) { sess.SetMetadata("checksum", "sha1 0000000000000000000000000000000000000000") require.NoError(t, sess.Persist(context.Background())) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) mockFs.On("Delete", mock.Anything, mock.Anything).Return((*storage.DeleteResult)(nil), nil) @@ -234,6 +242,9 @@ func TestUpload_Async(t *testing.T) { content := "asyncdata" sessionID := initiateAndGetID(t, coord, mockFs, content) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + ctx := context.Background() var uffCalled bool uff := func(_, _ *userpb.UserId, _ *provider.Reference) { uffCalled = true } @@ -270,6 +281,8 @@ func TestCoordinatedUpload_FinishUpload(t *testing.T) { _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) require.NoError(t, err) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, sessionID).Return(nil) @@ -291,6 +304,9 @@ func TestCoordinatedUpload_FinishUpload(t *testing.T) { _, err = up.WriteChunk(ctx, 0, strings.NewReader(content)) require.NoError(t, err) + mockFs.On("TouchFile", mock.Anything, mock.Anything, false, mock.Anything).Return(touchFileResultForUpload(), nil) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) + err = up.FinishUpload(ctx) require.NoError(t, err) @@ -313,6 +329,7 @@ func TestCoordinatedUpload_FinishUpload(t *testing.T) { up, err := coord.GetUpload(ctx, session.ID()) require.NoError(t, err) + mockFs.On("MarkProcessing", mock.Anything, mock.Anything, true, mock.AnythingOfType("string")).Return(nil) mockFs.On("CommitUpload", mock.Anything, mock.Anything, mock.Anything).Return((*provider.ResourceInfo)(nil), nil) mockFs.On("MarkProcessing", mock.Anything, mock.Anything, false, session.ID()).Return(nil) diff --git a/pkg/upload/mock_fs_test.go b/pkg/upload/mock_fs_test.go index f15122a9df5..b8c06e79ed2 100644 --- a/pkg/upload/mock_fs_test.go +++ b/pkg/upload/mock_fs_test.go @@ -173,8 +173,9 @@ func (m *mockFS) UnsetArbitraryMetadata(_ context.Context, _ *provider.Reference panic("not implemented: UnsetArbitraryMetadata") } -func (m *mockFS) GetLock(_ context.Context, _ *provider.Reference) (*provider.Lock, error) { - panic("not implemented: GetLock") +func (m *mockFS) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + args := m.Called(ctx, ref) + return args.Get(0).(*provider.Lock), args.Error(1) } func (m *mockFS) SetLock(_ context.Context, _ *provider.Reference, _ *provider.Lock) (*storage.SetLockResult, error) { diff --git a/pkg/upload/session.go b/pkg/upload/session.go index 43881f80767..d03ad5519df 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -256,6 +256,7 @@ func (s *FileSession) Metadata() map[string]string { "mtime": s.info.MetaData["mtime"], "nodeExists": s.info.Storage["NodeExists"], "versionsPath": s.info.MetaData["versionsPath"], + "sessionID": s.info.ID, } } From bd86383b2cce7e7c846969ab026bdd77602fe0dd Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Thu, 9 Jul 2026 15:49:52 +0200 Subject: [PATCH 14/20] Implement chunking v1 for coordinator --- .../storageprovider/storageprovider.go | 4 +- .../services/dataprovider/dataprovider.go | 4 +- pkg/storage/utils/chunking/chunking.go | 38 +++++++++ .../utils/decomposedfs/upload_async_test.go | 36 ++++++--- pkg/upload/coordinator.go | 80 ++++++++++++++----- pkg/upload/coordinator_test.go | 8 +- pkg/upload/filestore.go | 5 ++ pkg/upload/mock_fs_test.go | 4 +- pkg/upload/session.go | 6 ++ 9 files changed, 148 insertions(+), 37 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 061bb88acb3..f6774689d93 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "path" + "path/filepath" "sort" "strconv" "strings" @@ -228,7 +229,8 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. } async := evstream != nil coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, - c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log) + c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log, + filepath.Join(store.Root(), "chunks")) if err != nil { return nil, err } diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index c1ff6ae22fd..41d97474df5 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -21,6 +21,7 @@ package dataprovider import ( "fmt" "net/http" + "path/filepath" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" @@ -124,7 +125,8 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) } async := evstream != nil coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, - conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log) + conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log, + filepath.Join(store.Root(), "chunks")) if err != nil { return nil, err } diff --git a/pkg/storage/utils/chunking/chunking.go b/pkg/storage/utils/chunking/chunking.go index 97c21e030e6..bc3bdea3f1c 100644 --- a/pkg/storage/utils/chunking/chunking.go +++ b/pkg/storage/utils/chunking/chunking.go @@ -201,6 +201,44 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (bool, string, er return true, assembledFileName, nil } +// Assemble saves an intermediate chunk and, once all chunks have arrived, +// assembles them into a single temp file. Returns (reader, size, true, nil) +// when the transfer is complete. Returns (nil, 0, false, nil) when more +// chunks are still expected (partial transfer — caller should return PartialContent). +func (c *ChunkHandler) Assemble(chunkName string, body io.ReadCloser) (io.ReadCloser, int64, bool, error) { + origPath, assembledPath, err := c.WriteChunk(chunkName, body) + if err != nil { + return nil, 0, false, err + } + if origPath == "" { + return nil, 0, false, nil // partial + } + f, err := os.Open(assembledPath) + if err != nil { + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + return &assembledFile{File: f, path: assembledPath}, fi.Size(), true, nil +} + +// assembledFile wraps an *os.File and removes the temp file on Close. +type assembledFile struct { + *os.File + path string +} + +func (a *assembledFile) Close() error { + err := a.File.Close() + _ = os.RemoveAll(a.path) + return err +} + // WriteChunk saves an intermediate chunk temporarily and assembles all chunks // once the final one is received. func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 39e739958d0..1be2fefb1f3 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -204,7 +204,7 @@ var _ = Describe("Async file uploads", Ordered, func() { d := dfs.(*Decomposedfs) fileStore := pkgupload.NewFileStore(o.Root, pkgupload.TokenOptions{}, &zerolog.Logger{}) var coordErr error - coord, coordErr = pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, true, "", "dcfs", 1, &zerolog.Logger{}) + coord, coordErr = pkgupload.NewCoordinator(d, fileStore, aspects.EventStream, true, "", "dcfs", 1, &zerolog.Logger{}, "") Expect(coordErr).ToNot(HaveOccurred()) Expect(coord.Start(aspects.EventStream)).To(Succeed()) fs = d @@ -405,26 +405,40 @@ var _ = Describe("Async file uploads", Ordered, func() { }) When("a second upload is attempted while the first is still in postprocessing", func() { - // The coordinator serializes uploads via MarkProcessing: a second InitiateUpload - // while the first session holds the processing slot returns ResourceProcessing - // and cleans up immediately. + // The coordinator serializes uploads via MarkProcessing: a second FinishUpload + // while the first session holds the processing slot returns ResourceProcessing. + // InitiateUpload itself succeeds — the conflict is detected when bytes arrive. - It("rejects the second InitiateUpload with ResourceProcessing", func() { + It("rejects the second FinishUpload with ResourceProcessing", func() { // First upload is in postprocessing (BytesReceived consumed in BeforeEach). - _, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID := uploadIds["simple"] + + up, err := coord.GetUpload(ctx, secondUploadID) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) + Expect(err).ToNot(HaveOccurred()) + err = up.FinishUpload(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("resource is processing")) }) It("leaves no orphan session files after rejection", func() { - // InitiateUpload fails before TouchBin/Persist, so no .bin or .info are created. - _, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) - Expect(err).To(HaveOccurred()) + uploadIds, err := coord.InitiateUpload(ctx, ref, 20, map[string]string{}) + Expect(err).ToNot(HaveOccurred()) + secondUploadID := uploadIds["simple"] + + up, err := coord.GetUpload(ctx, secondUploadID) + Expect(err).ToNot(HaveOccurred()) + _, err = up.WriteChunk(ctx, 0, bytes.NewReader(secondContent)) + Expect(err).ToNot(HaveOccurred()) + _ = up.FinishUpload(ctx) // expected to fail - // No uploads directory entries should exist beyond the first session. + // touchAndMark rollback removes .bin and .info for the second session. + // Only the first upload's .bin and .info should remain. entries, readErr := os.ReadDir(filepath.Join(o.Root, "uploads")) Expect(readErr).ToNot(HaveOccurred()) - // Only the first upload's .bin and .info should be present. Expect(len(entries)).To(Equal(2)) }) }) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 1de3290fee0..3195f769884 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -42,6 +42,7 @@ import ( "github.com/owncloud/reva/v2/pkg/events" "github.com/owncloud/reva/v2/pkg/rhttp/datatx/metrics" "github.com/owncloud/reva/v2/pkg/storage" + "github.com/owncloud/reva/v2/pkg/storage/utils/chunking" "github.com/owncloud/reva/v2/pkg/utils" ) @@ -70,6 +71,16 @@ func impersonatingUser(ctx context.Context) *user.User { var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", 501) +// rewriteChunkedRef strips the chunk suffix from ref.Path and returns the chunk basename. +// Only called when chunking.IsChunked(ref.GetPath()) is true. +func rewriteChunkedRef(ref *provider.Reference) (*provider.Reference, string, error) { + ci, err := chunking.GetChunkBLOBInfo(ref.GetPath()) + if err != nil { + return nil, "", errtypes.BadRequest(err.Error()) + } + return &provider.Reference{ResourceId: ref.ResourceId, Path: ci.Path}, filepath.Base(ref.GetPath()), nil +} + // rollback unmarks processing, cleans up session files, and deletes the placeholder // node if it was created by this upload (NodeExists=false at initiation). func (c *coordinator) rollback(ctx context.Context, session Session) { @@ -241,18 +252,20 @@ type Coordinator interface { // coordinator is the concrete implementation of Coordinator. type coordinator struct { - fs storage.FS - store SessionStore - pub events.Publisher - async bool - mountID string - numConc int - conGroup string - log *zerolog.Logger + fs storage.FS + store SessionStore + pub events.Publisher + async bool + mountID string + numConc int + conGroup string + log *zerolog.Logger + chunkHandler *chunking.ChunkHandler // nil when legacy chunking v1 is not needed } // NewCoordinator constructs a Coordinator. Call Start to begin consuming events. // async=true requires a non-nil pub. +// chunkFolder enables legacy chunking v1 support; pass "" to disable it. func NewCoordinator( fs storage.FS, store SessionStore, @@ -262,6 +275,7 @@ func NewCoordinator( consumerGroup string, numConsumers int, log *zerolog.Logger, + chunkFolder string, ) (Coordinator, error) { if async && pub == nil { return nil, fmt.Errorf("need event stream for async upload processing") @@ -269,19 +283,33 @@ func NewCoordinator( if numConsumers <= 0 { numConsumers = 1 } + var ch *chunking.ChunkHandler + if chunkFolder != "" { + ch = chunking.NewChunkHandler(chunkFolder) + } return &coordinator{ - fs: fs, - store: store, - pub: pub, - async: async, - mountID: mountID, - numConc: numConsumers, - conGroup: consumerGroup, - log: log, + fs: fs, + store: store, + pub: pub, + async: async, + mountID: mountID, + numConc: numConsumers, + conGroup: consumerGroup, + log: log, + chunkHandler: ch, }, nil } -func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { +func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + var chunkName string + if chunking.IsChunked(ref.GetPath()) { // check legacy chunking v1 + var rerr error + ref, chunkName, rerr = rewriteChunkedRef(ref) + if rerr != nil { + return nil, rerr + } + } + existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) var nodeExists bool switch err.(type) { @@ -424,6 +452,9 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference if !mtimeSet { session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now())) } + if chunkName != "" { // check legacy chunking v1 + session.SetStorageValue("Chunk", chunkName) + } if err := session.TouchBin(); err != nil { return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) @@ -455,7 +486,7 @@ func (c *coordinator)InitiateUpload(ctx context.Context, ref *provider.Reference // Upload handles the simple (single-PUT) upload path so the coordinator owns // the complete upload lifecycle regardless of the datatx protocol used. // simple.go calls fs.Upload(); when fs is a *Coordinator this method intercepts. -func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { +func (c *coordinator) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { id := strings.TrimPrefix(req.Ref.GetPath(), "/") session, err := c.store.Get(ctx, id) if err != nil { @@ -463,6 +494,19 @@ func (c *coordinator)Upload(ctx context.Context, req storage.UploadRequest, uff } ctx = session.Context(ctx) + if session.Chunk() != "" { // check legacy chunking v1 + assembled, assembledSize, done, err := c.chunkHandler.Assemble(session.Chunk(), req.Body) + if err != nil { + return nil, err + } + if !done { + session.Cleanup(true, true) + return nil, errtypes.PartialContent(req.Ref.String()) + } + defer assembled.Close() + req.Body, req.Length = assembled, assembledSize + } + size, err := session.WriteChunk(ctx, 0, req.Body) if err != nil { return nil, err diff --git a/pkg/upload/coordinator_test.go b/pkg/upload/coordinator_test.go index d552f4c92e7..37fb5cc56d9 100644 --- a/pkg/upload/coordinator_test.go +++ b/pkg/upload/coordinator_test.go @@ -43,25 +43,25 @@ func TestNewCoordinator(t *testing.T) { fs := &mockFS{} t.Run("async without publisher returns error", func(t *testing.T) { - _, err := NewCoordinator(fs, store, nil, true, "m", "g", 1, &log) + _, err := NewCoordinator(fs, store, nil, true, "m", "g", 1, &log, "") require.Error(t, err) }) t.Run("sync without publisher succeeds", func(t *testing.T) { - coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 1, &log) + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 1, &log, "") require.NoError(t, err) require.NotNil(t, coord) }) t.Run("async with publisher succeeds", func(t *testing.T) { pub := &mockPublisher{} - coord, err := NewCoordinator(fs, store, pub, true, "m", "g", 1, &log) + coord, err := NewCoordinator(fs, store, pub, true, "m", "g", 1, &log, "") require.NoError(t, err) require.NotNil(t, coord) }) t.Run("numConsumers zero defaults to 1", func(t *testing.T) { - coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 0, &log) + coord, err := NewCoordinator(fs, store, nil, false, "m", "g", 0, &log, "") require.NoError(t, err) require.NotNil(t, coord) c := coord.(*coordinator) diff --git a/pkg/upload/filestore.go b/pkg/upload/filestore.go index 55be9057f16..d96ef138a13 100644 --- a/pkg/upload/filestore.go +++ b/pkg/upload/filestore.go @@ -128,6 +128,11 @@ func NewFileStore(root string, opts TokenOptions, log *zerolog.Logger) *FileStor return &FileStore{root: root, opts: opts, log: log} } +// Root returns the base directory of this FileStore. +func (fs *FileStore) Root() string { + return fs.root +} + // Setup creates the uploads directory eagerly so permission problems are caught // at startup rather than on the first upload. func (fs *FileStore) Setup() error { diff --git a/pkg/upload/mock_fs_test.go b/pkg/upload/mock_fs_test.go index b8c06e79ed2..533daf46a4d 100644 --- a/pkg/upload/mock_fs_test.go +++ b/pkg/upload/mock_fs_test.go @@ -236,7 +236,7 @@ func newTestCoordinator(t *testing.T, root string, async bool, pub events.Publis log := zerolog.Nop() store := newTestStore(t, root) fs := &mockFS{} - coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log) + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log, "") require.NoError(t, err) return coord, fs } @@ -247,7 +247,7 @@ func newTestCoordinatorWithStore(t *testing.T, root string, async bool, pub even log := zerolog.Nop() store := newTestStore(t, root) fs := &mockFS{} - coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log) + coord, err := NewCoordinator(fs, store, pub, async, "test-mount", "test-group", 1, &log, "") require.NoError(t, err) return coord, fs, store } diff --git a/pkg/upload/session.go b/pkg/upload/session.go index d03ad5519df..7abb58db6df 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -74,6 +74,7 @@ type Session interface { WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) // Internal coordinator plumbing. + Chunk() string BinPath() string ProviderID() string SpaceID() string @@ -158,6 +159,11 @@ func (s *FileSession) Offset() int64 { return s.info.Offset } +// Chunk returns the chunk basename stored in the session, or "" for non-chunked uploads. +func (s *FileSession) Chunk() string { + return s.info.Storage["Chunk"] +} + // BinPath returns the path to the staged binary file. func (s *FileSession) BinPath() string { return s.binPath() From 1f5c608ea3eda561e6d0f3ab33217e48e6989c44 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Thu, 9 Jul 2026 17:14:03 +0200 Subject: [PATCH 15/20] permission checks --- .../storageprovider/storageprovider.go | 2 +- .../services/dataprovider/dataprovider.go | 2 +- pkg/upload/coordinator.go | 28 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index f6774689d93..1ba99929b05 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -230,7 +230,7 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. async := evstream != nil coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, c.MountID, c.Events.ConsumerGroup, c.Events.NumConsumers, log, - filepath.Join(store.Root(), "chunks")) + filepath.Join(store.Root(), "uploads")) if err != nil { return nil, err } diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 41d97474df5..5a43601303b 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -126,7 +126,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) async := evstream != nil coord, err := pkgupload.NewCoordinator(fs, store, evstream, async, conf.MountID, conf.ConsumerGroup, conf.NumConsumers, log, - filepath.Join(store.Root(), "chunks")) + filepath.Join(store.Root(), "uploads")) if err != nil { return nil, err } diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 3195f769884..705bce7b292 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -378,6 +378,34 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc nodeName = filepath.Base(ref.GetPath()) } + if nodeExists { + if !existing.GetPermissionSet().GetInitiateFileUpload() { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } + if existing.GetType() == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + return nil, errtypes.PreconditionFailed("resource is not a file") + } + if metadata["if-none-match"] == "*" { + return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", parentID, nodeName, nodeID)) + } + } else { + parentRef := &provider.Reference{ + ResourceId: &provider.ResourceId{SpaceId: spaceID}, + Path: dir, + } + parentMD, pErr := c.fs.GetMD(ctx, parentRef, []string{}, []string{}) + switch pErr.(type) { + case nil: + case errtypes.IsNotFound: + return nil, errtypes.PreconditionFailed(pErr.Error()) + default: + return nil, pErr + } + if !parentMD.GetPermissionSet().GetInitiateFileUpload() { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } + } + if nodeName == "" { return nil, errtypes.BadRequest("coordinator: missing filename in ref") } From 604a55c8c9d1e2636b4926598fe740c9758840c7 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Fri, 10 Jul 2026 17:40:29 +0200 Subject: [PATCH 16/20] fixes --- pkg/ocm/storage/received/ocm.go | 3 ++ pkg/upload/coordinator.go | 64 ++++++++++++++++----------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go index fe2bb04ab8e..85f8fcdd757 100644 --- a/pkg/ocm/storage/received/ocm.go +++ b/pkg/ocm/storage/received/ocm.go @@ -539,6 +539,9 @@ func (d *driver) GetLock(ctx context.Context, ref *provider.Reference) (*provide return nil, err } + if token == "" { + return nil, errtypes.NotFound("no lock found") + } return &provider.Lock{LockId: token, Type: provider.LockType_LOCK_TYPE_EXCL}, nil } diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 705bce7b292..fc32d00ab43 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -31,7 +31,6 @@ import ( user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" "github.com/rs/zerolog" tusd "github.com/tus/tusd/v2/pkg/handler" "go.opentelemetry.io/otel" @@ -125,10 +124,10 @@ func (c *coordinator) triggerPostprocessing(ctx context.Context, session Session } executingUser, _ := ctxpkg.ContextGetUser(ctx) if err := events.Publish(ctx, c.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: session.SpaceOwner(), - ExecutingUser: executingUser, + UploadID: session.ID(), + URL: s, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: executingUser, ResourceID: &provider.ResourceId{ StorageId: session.ProviderID(), SpaceId: session.SpaceID(), @@ -324,6 +323,29 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc var nodeID, spaceID, parentID, dir, nodeName string var spaceOwner *user.UserId + // check quota + if uploadLength >= 0 { + spaceRef := &provider.Reference{ResourceId: &provider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + }} + if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil { + var existingSize uint64 + if nodeExists { + existingSize = existing.GetSize() + } + netRequired := uint64(uploadLength) + if existingSize < netRequired { + netRequired -= existingSize + } else { + netRequired = 0 + } + if remaining < netRequired { + return nil, errtypes.InsufficientStorage("quota exceeded") + } + } + } + if nodeExists { nodeID = existing.GetId().GetOpaqueId() spaceID = existing.GetId().GetSpaceId() @@ -346,33 +368,7 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc } else if contextLockID != "" { return nil, errtypes.Aborted("not locked") } - - // For overwrites the existing bytes will be freed on commit, so net required - // space is uploadLength - existing.Size. Skip for size-deferred uploads. - if uploadLength >= 0 { - spaceRef := &provider.Reference{ResourceId: existing.GetId()} - if _, _, remaining, qErr := c.fs.GetQuota(ctx, spaceRef); qErr == nil { - existingSize := existing.GetSize() - netRequired := uint64(uploadLength) - if existingSize < netRequired { - netRequired -= existingSize - } else { - netRequired = 0 - } - if remaining < netRequired { - return nil, errtypes.InsufficientStorage("quota exceeded") - } - } - } } else { - if uploadLength > 0 { - if _, _, remaining, qErr := c.fs.GetQuota(ctx, ref); qErr == nil && remaining < uint64(uploadLength) { - return nil, errtypes.InsufficientStorage("quota exceeded") - } - } - - // Pre-generate a node ID; the node is created in FinishUpload once bytes have arrived. - nodeID = uuid.New().String() spaceID = ref.GetResourceId().GetSpaceId() dir = filepath.Dir(ref.GetPath()) nodeName = filepath.Base(ref.GetPath()) @@ -418,9 +414,9 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc session.SetStorageValue("NodeName", nodeName) session.SetMetadata("dir", dir) session.SetStorageValue("Dir", dir) - session.SetStorageValue("NodeId", nodeID) session.SetStorageValue("SpaceRoot", spaceID) if nodeExists { + session.SetStorageValue("NodeId", nodeID) session.SetStorageValue("NodeExists", "true") } session.SetStorageValue("NodeParentId", parentID) @@ -533,6 +529,7 @@ func (c *coordinator) Upload(ctx context.Context, req storage.UploadRequest, uff } defer assembled.Close() req.Body, req.Length = assembled, assembledSize + session.SetSize(assembledSize) } size, err := session.WriteChunk(ctx, 0, req.Body) @@ -571,7 +568,7 @@ func (c *coordinator) Upload(ctx context.Context, req storage.UploadRequest, uff } // ListUploadSessions returns upload sessions matching the given filter. -func (c *coordinator)ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { +func (c *coordinator) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { if filter.ID != nil && *filter.ID != "" { session, err := c.store.Get(ctx, *filter.ID) if err != nil { @@ -614,4 +611,3 @@ func (c *coordinator)ListUploadSessions(ctx context.Context, filter storage.Uplo } return result, nil } - From bea99c08cb192e0be5da1c611592300fe4064323 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Mon, 13 Jul 2026 09:57:36 +0200 Subject: [PATCH 17/20] resolve permission issue --- pkg/upload/coordinated_upload.go | 2 +- pkg/upload/coordinator.go | 11 ++++++++--- pkg/upload/session.go | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go index da7c1828a05..2e00045d768 100644 --- a/pkg/upload/coordinated_upload.go +++ b/pkg/upload/coordinated_upload.go @@ -91,7 +91,7 @@ func (u *coordinatedUpload) ConcatUploads(ctx context.Context, partials []tusd.U } func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { - return u.coord.finishUpload(ctx, u.session) + return u.coord.finishUpload(u.session.Context(ctx), u.session) } // UseIn registers the coordinator as the TUS data store in the composer. diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index fc32d00ab43..92ecdf30220 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -208,8 +208,11 @@ func verifyAndStoreChecksums(ctx context.Context, session Session) error { func (c *coordinator) touchAndMark(ctx context.Context, session Session) error { if !session.NodeExists() { pathRef := &provider.Reference{ - ResourceId: &provider.ResourceId{SpaceId: session.SpaceID()}, - Path: filepath.Join(session.Dir(), session.Filename()), + ResourceId: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeParentID(), + }, + Path: session.Filename(), } result, err := c.fs.TouchFile(ctx, pathRef, false, session.Metadata()["mtime"]) if err != nil { @@ -386,7 +389,7 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc } } else { parentRef := &provider.Reference{ - ResourceId: &provider.ResourceId{SpaceId: spaceID}, + ResourceId: ref.GetResourceId(), Path: dir, } parentMD, pErr := c.fs.GetMD(ctx, parentRef, []string{}, []string{}) @@ -400,6 +403,8 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc if !parentMD.GetPermissionSet().GetInitiateFileUpload() { return nil, errtypes.PermissionDenied(ref.GetPath()) } + parentID = parentMD.GetId().GetOpaqueId() + spaceID = parentMD.GetId().GetSpaceId() } if nodeName == "" { diff --git a/pkg/upload/session.go b/pkg/upload/session.go index 7abb58db6df..ca3bbec61d0 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -79,6 +79,7 @@ type Session interface { ProviderID() string SpaceID() string NodeID() string + NodeParentID() string NodeExists() bool Dir() string URL(ctx context.Context) (string, error) @@ -189,6 +190,11 @@ func (s *FileSession) NodeID() string { return s.info.Storage["NodeId"] } +// NodeParentID returns the parent node ID for this upload. +func (s *FileSession) NodeParentID() string { + return s.info.Storage["NodeParentId"] +} + // NodeExists returns whether the target node existed when the upload was initiated. func (s *FileSession) NodeExists() bool { return s.info.Storage["NodeExists"] == "true" From dbb4878d779dac05daba04ad7a3091785e7b101b Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Mon, 13 Jul 2026 13:21:37 +0200 Subject: [PATCH 18/20] fix group permissions --- pkg/upload/coordinated_upload.go | 23 ++++++++++++++++- pkg/upload/coordinator.go | 14 ++++++++--- pkg/upload/session.go | 5 ++++ pkg/utils/etag.go | 42 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 pkg/utils/etag.go diff --git a/pkg/upload/coordinated_upload.go b/pkg/upload/coordinated_upload.go index 2e00045d768..381429627de 100644 --- a/pkg/upload/coordinated_upload.go +++ b/pkg/upload/coordinated_upload.go @@ -22,9 +22,12 @@ import ( "context" "fmt" "io" + "net/http" "os" tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/owncloud/reva/v2/pkg/errtypes" ) // coordinatedUpload is the TUS interface adapter between the tusd library and the coordinator. @@ -91,7 +94,25 @@ func (u *coordinatedUpload) ConcatUploads(ctx context.Context, partials []tusd.U } func (u *coordinatedUpload) FinishUpload(ctx context.Context) error { - return u.coord.finishUpload(u.session.Context(ctx), u.session) + err := u.coord.finishUpload(u.session.Context(ctx), u.session) + switch err.(type) { + case nil: + return nil + case errtypes.ResourceProcessing, errtypes.TooEarly: + return tusd.NewError("ERR_TOO_EARLY", err.Error(), http.StatusTooEarly) + case errtypes.Aborted: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed) + case errtypes.PreconditionFailed: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusMethodNotAllowed) + case errtypes.Locked: + return tusd.NewError("ERR_LOCKED", err.Error(), http.StatusLocked) + case errtypes.BadRequest: + return tusd.NewError("ERR_BAD_REQUEST", err.Error(), http.StatusBadRequest) + case errtypes.ChecksumMismatch: + return tusd.NewError("ERR_CHECKSUM_MISMATCH", err.Error(), errtypes.StatusChecksumMismatch) + default: + return err + } } // UseIn registers the coordinator as the TUS data store in the composer. diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 92ecdf30220..1696045b9e2 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -100,11 +100,12 @@ func (c *coordinator) finishSync(ctx context.Context, session Session) error { c.rollback(ctx, session) return err } + cs := session.Checksums() if _, err := c.fs.CommitUpload(ctx, &ref, storage.UploadSource{ Body: f, Length: session.Size(), Metadata: session.Metadata(), - Checksums: session.Checksums(), + Checksums: cs, }); err != nil { c.rollback(ctx, session) return err @@ -562,14 +563,21 @@ func (c *coordinator) Upload(ctx context.Context, req storage.UploadRequest, uff uff(session.SpaceOwner(), &executant, uploadRef) } - return &provider.ResourceInfo{ + ri := &provider.ResourceInfo{ Id: &provider.ResourceId{ StorageId: session.ProviderID(), SpaceId: session.SpaceID(), OpaqueId: session.NodeID(), }, Name: session.Filename(), - }, nil + } + if mt, ok := session.Metadata()["mtime"]; ok && mt != "" { + if t, err := utils.MTimeToTime(mt); err == nil { + ri.Etag, _ = utils.CalculateEtag(session.NodeID(), t) + ri.Mtime = utils.TimeToTS(t) + } + } + return ri, nil } // ListUploadSessions returns upload sessions matching the given filter. diff --git a/pkg/upload/session.go b/pkg/upload/session.go index ca3bbec61d0..4db4d665fec 100644 --- a/pkg/upload/session.go +++ b/pkg/upload/session.go @@ -315,6 +315,8 @@ func (s *FileSession) SetExecutant(u *userpb.User) { s.info.Storage["UserDisplayName"] = u.GetDisplayName() b, _ := json.Marshal(u.GetOpaque()) s.info.Storage["UserOpaque"] = string(b) + g, _ := json.Marshal(u.GetGroups()) + s.info.Storage["UserGroups"] = string(g) } // TouchBin creates the empty staging file. @@ -410,6 +412,8 @@ func (s *FileSession) infoPath() string { func (s *FileSession) executantUser() *userpb.User { var o *typespb.Opaque _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) + var groups []string + _ = json.Unmarshal([]byte(s.info.Storage["UserGroups"]), &groups) return &userpb.User{ Id: &userpb.UserId{ Type: utils.UserTypeMap(s.info.Storage["UserType"]), @@ -419,6 +423,7 @@ func (s *FileSession) executantUser() *userpb.User { Username: s.info.Storage["UserName"], DisplayName: s.info.Storage["UserDisplayName"], Opaque: o, + Groups: groups, } } diff --git a/pkg/utils/etag.go b/pkg/utils/etag.go new file mode 100644 index 00000000000..9293b463451 --- /dev/null +++ b/pkg/utils/etag.go @@ -0,0 +1,42 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package utils + +import ( + "crypto/md5" + "fmt" + "io" + "time" +) + +// CalculateEtag returns a hash of fileid + tmtime (or mtime) +func CalculateEtag(id string, t time.Time) (string, error) { + h := md5.New() + if _, err := io.WriteString(h, id); err != nil { + return "", err + } + if tb, err := t.UTC().MarshalBinary(); err == nil { + if _, err := h.Write(tb); err != nil { + return "", err + } + } else { + return "", err + } + return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil +} From 1c2f56d4f91db47dbe199d003d92cca2389e8f36 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Mon, 13 Jul 2026 17:50:32 +0200 Subject: [PATCH 19/20] handle processing flag in propfind --- internal/http/services/owncloud/ocdav/propfind/propfind.go | 4 ++++ pkg/storage/utils/decomposedfs/decomposedfs.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/internal/http/services/owncloud/ocdav/propfind/propfind.go b/internal/http/services/owncloud/ocdav/propfind/propfind.go index 4ec66360802..aaf1e317335 100644 --- a/internal/http/services/owncloud/ocdav/propfind/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind/propfind.go @@ -587,6 +587,10 @@ func (p *Handler) getResourceInfos(ctx context.Context, w http.ResponseWriter, r var status *rpc.Status info, status, err = p.statSpace(ctx, spaceRef, metadataKeys, fieldMaskPaths) if err != nil || status.GetCode() != rpc.Code_CODE_OK { + if status.GetCode() == rpc.Code_CODE_TOO_EARLY { + w.WriteHeader(http.StatusTooEarly) + return nil, false, false + } continue } } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index f81cdaa9ffa..eadc31c35de 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -613,6 +613,10 @@ func (fs *Decomposedfs) GetMD(ctx context.Context, ref *provider.Reference, mdKe return } + if node.IsProcessing(ctx) { + return nil, errtypes.ResourceProcessing(ref.String()) + } + rp, err := fs.p.AssemblePermissions(ctx, node) switch { case err != nil: From 6a4660cea11de8f3e2d0d7f7fdb4e95d83065220 Mon Sep 17 00:00:00 2001 From: "lars.jurgensen" Date: Mon, 13 Jul 2026 21:07:10 +0200 Subject: [PATCH 20/20] test fixes --- pkg/upload/coordinator.go | 19 ++++++++++++++++++- pkg/upload/coordinator_postprocessing.go | 8 +------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/upload/coordinator.go b/pkg/upload/coordinator.go index 1696045b9e2..86a1c4f9161 100644 --- a/pkg/upload/coordinator.go +++ b/pkg/upload/coordinator.go @@ -354,7 +354,7 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc nodeID = existing.GetId().GetOpaqueId() spaceID = existing.GetId().GetSpaceId() parentID = existing.GetParentId().GetOpaqueId() - dir = filepath.Dir(existing.GetPath()) + dir = filepath.Dir(ref.GetPath()) nodeName = existing.GetName() spaceOwner = existing.GetOwner() @@ -397,6 +397,23 @@ func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Referenc switch pErr.(type) { case nil: case errtypes.IsNotFound: + // RFC 4918: missing intermediate dir → 409, no permission → 404. + // GetMD returns NotFound for both (hides resources from unauthorized callers). + // Walk up the path: if an ancestor is visible, the dir is truly missing (409). + // If nothing is visible up to the root, caller has no access (404). + ancestor := dir + permDenied := true + for ancestor != "." && ancestor != "/" { + ancestor = filepath.Dir(ancestor) + ancestorRef := &provider.Reference{ResourceId: ref.GetResourceId(), Path: ancestor} + if _, aErr := c.fs.GetMD(ctx, ancestorRef, []string{}, []string{}); aErr == nil { + permDenied = false + break + } + } + if permDenied { + return nil, errtypes.PermissionDenied(ref.GetPath()) + } return nil, errtypes.PreconditionFailed(pErr.Error()) default: return nil, pErr diff --git a/pkg/upload/coordinator_postprocessing.go b/pkg/upload/coordinator_postprocessing.go index 55000383747..c999a9448e5 100644 --- a/pkg/upload/coordinator_postprocessing.go +++ b/pkg/upload/coordinator_postprocessing.go @@ -181,13 +181,7 @@ func (c *coordinator) handlePostprocessingFinished(ctx context.Context, ev event } } - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionsPath"] != "" { - isVersion = true - } - } + isVersion := session.NodeExists() if err := events.Publish( ctx,