diff --git a/DriveClient/Helpers/KDriveJsonContext.cs b/DriveClient/Helpers/KDriveJsonContext.cs index aaebc90..fbda3eb 100644 --- a/DriveClient/Helpers/KDriveJsonContext.cs +++ b/DriveClient/Helpers/KDriveJsonContext.cs @@ -1,5 +1,6 @@ using kDriveClient.Models; using kDriveClient.Models.Exceptions; + namespace kDriveClient.Helpers { /// @@ -11,8 +12,10 @@ namespace kDriveClient.Helpers [JsonSerializable(typeof(KDriveFile))] [JsonSerializable(typeof(KDriveChunk))] [JsonSerializable(typeof(KDriveUploadResponseWraper))] + [JsonSerializable(typeof(KDriveFinishRequest))] + [JsonSerializable(typeof(KDriveUploadDataResponse))] [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] public partial class KDriveJsonContext : JsonSerializerContext { } -} +} \ No newline at end of file diff --git a/DriveClient/Helpers/KDriveJsonHelper.cs b/DriveClient/Helpers/KDriveJsonHelper.cs index 0991f38..4cd29c3 100644 --- a/DriveClient/Helpers/KDriveJsonHelper.cs +++ b/DriveClient/Helpers/KDriveJsonHelper.cs @@ -16,7 +16,7 @@ public static class KDriveJsonHelper /// public static KDriveUploadResponse DeserializeUploadResponse(string json) { - return JsonSerializer.Deserialize(json, KDriveJsonContext.Default.KDriveUploadResponseWraper)?.Data ?? throw new InvalidOperationException("Failed to parse upload response"); + return JsonSerializer.Deserialize(json, KDriveJsonContext.Default.KDriveUploadResponseWraper)?.Data?.File ?? throw new InvalidOperationException("Failed to parse upload response"); } /// @@ -54,19 +54,18 @@ public static async Task DeserializeResponseAsync(HttpRespo { if (!response.IsSuccessStatusCode) { - var json = await response.Content.ReadAsStringAsync(ct); - KDriveErrorResponse? error = null; try { - error = JsonSerializer.Deserialize(json, KDriveJsonContext.Default.KDriveErrorResponse); + await using var stream = await response.Content.ReadAsStreamAsync(ct); + error = await JsonSerializer.DeserializeAsync(stream, KDriveJsonContext.Default.KDriveErrorResponse, cancellationToken: ct); } catch { response.EnsureSuccessStatusCode(); } - if (error != null) + if (error is not null) { throw new KDriveApiException(error); } diff --git a/DriveClient/Helpers/KDriveRequestFactory.cs b/DriveClient/Helpers/KDriveRequestFactory.cs index 5b62359..5187c72 100644 --- a/DriveClient/Helpers/KDriveRequestFactory.cs +++ b/DriveClient/Helpers/KDriveRequestFactory.cs @@ -1,4 +1,5 @@ using kDriveClient.Models; +using System.Net; using System.Net.Http.Headers; namespace kDriveClient.Helpers @@ -19,7 +20,7 @@ public static HttpRequestMessage CreateUploadDirectRequest(long driveId, KDriveF var url = $"/3/drive/{driveId}/upload?" + string.Join("&", BuildUploadQueryParams(file).ToList().ConvertAll(e => $"{e.Key}={e.Value}")); var request = new HttpRequestMessage(HttpMethod.Post, url) { - Content = new StreamContent(new MemoryStream(file.Chunks.First().Content)) + Content = new StreamContent(new MemoryStream(file.Chunks.First().Content!)) }; request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); @@ -55,14 +56,14 @@ public static HttpRequestMessage CreateStartSessionRequest(long driveId, KDriveF /// An HttpRequestMessage configured for the chunk upload operation. public static HttpRequestMessage CreateChunkUploadRequest(string baseUrl, string token, long driveId, KDriveChunk chunk) { - var request = new HttpRequestMessage(HttpMethod.Post, $"{baseUrl}/3/drive/{driveId}/upload/session/{token}/chunk?chunk_number={chunk.ChunkNumber + 1}&chunk_size={chunk.ChunkSize}&chunk_hash=sha256:{chunk.ChunkHash.ToLowerInvariant()}") + var content = new ReadOnlyMemoryContent(new ReadOnlyMemory(chunk.Content, 0, chunk.ChunkSize)); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + return new HttpRequestMessage(HttpMethod.Post, $"{baseUrl}/3/drive/{driveId}/upload/session/{token}/chunk?chunk_number={chunk.ChunkNumber + 1}&chunk_size={chunk.ChunkSize}&chunk_hash=sha256:{chunk.ChunkHash.ToLowerInvariant()}") { - Content = new ByteArrayContent(chunk.Content) + Content = content, + Version = HttpVersion.Version20, + VersionPolicy = HttpVersionPolicy.RequestVersionOrLower, }; - request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - request.Content.Headers.ContentLength = chunk.ChunkSize; - - return request; } /// @@ -74,10 +75,10 @@ public static HttpRequestMessage CreateChunkUploadRequest(string baseUrl, string /// An HttpRequestMessage configured to finish the upload session. public static HttpRequestMessage CreateFinishSessionRequest(long driveId, string sessionToken, string totalChunkHash) { - var content = new StringContent(JsonSerializer.Serialize(new + var content = new StringContent(JsonSerializer.Serialize(new KDriveFinishRequest { - total_chunk_hash = $"sha256:{totalChunkHash.ToLowerInvariant()}" - }, KDriveJsonContext.Default.Object)); + TotalChunkHash = $"sha256:{totalChunkHash.ToLowerInvariant()}" + }, KDriveJsonContext.Default.KDriveFinishRequest)); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); diff --git a/DriveClient/Models/ConflictChoice.cs b/DriveClient/Models/ConflictChoice.cs index af688a4..f8f4f8e 100644 --- a/DriveClient/Models/ConflictChoice.cs +++ b/DriveClient/Models/ConflictChoice.cs @@ -1,7 +1,7 @@ namespace kDriveClient.Models { /// - /// + /// /// public enum ConflictChoice { @@ -9,13 +9,15 @@ public enum ConflictChoice /// Throw an error /// Error, + /// /// Add a new version to file /// Version, + /// /// Create a new file with an automated renaming /// Rename, } -} +} \ No newline at end of file diff --git a/DriveClient/Models/KDriveChunk.cs b/DriveClient/Models/KDriveChunk.cs index 5700cda..e153bef 100644 --- a/DriveClient/Models/KDriveChunk.cs +++ b/DriveClient/Models/KDriveChunk.cs @@ -3,7 +3,7 @@ /// /// kDriveChunk represents a chunk of Data in a kDrive file. /// - public class KDriveChunk(byte[] content, int chunkNumber, byte[] hash) + public class KDriveChunk(byte[] content, int chunkNumber, byte[] hash) : IDisposable { /// /// ChunkHash is the SHA-256 hash of the chunk content. @@ -18,11 +18,28 @@ public class KDriveChunk(byte[] content, int chunkNumber, byte[] hash) /// /// ChunkSize is the size of the chunk in bytes. /// - public long ChunkSize => this.Content.Length; + public int ChunkSize { get; set; } = content.Length; /// /// Content is the actual byte content of the chunk. /// - public byte[] Content { get; init; } = content; + public byte[]? Content { get; private set; } = content; + + /// + /// Frees resources used by the KDriveChunk. + /// + public void Dispose() + { + GC.SuppressFinalize(this); + } + + /// + /// Deletes the content of the chunk to free memory. + /// + internal void Clean() + { + this.Content = null; + GC.Collect(); + } } } \ No newline at end of file diff --git a/DriveClient/Models/KDriveFile.cs b/DriveClient/Models/KDriveFile.cs index 848447a..ef73688 100644 --- a/DriveClient/Models/KDriveFile.cs +++ b/DriveClient/Models/KDriveFile.cs @@ -1,10 +1,14 @@ -namespace kDriveClient.Models +using System.Text; + +namespace kDriveClient.Models { /// /// KDriveFile represents a file in the kDrive system. /// - public class KDriveFile + public class KDriveFile : IDisposable { + private Int64 totalSize; + /// /// CreatedAt is the timestamp when the file was created. /// @@ -38,20 +42,23 @@ public class KDriveFile /// /// TotalChunkHash is the SHA-256 hash of the entire file content, computed from all chunks. /// - public string TotalChunkHash + public string TotalChunkHash { get; set; } = string.Empty; + + /// + /// TotalSize is the total size of the file in bytes, calculated as the sum of all chunk sizes. + /// + public long TotalSize { get { - return Convert.ToHexString( - SHA256.HashData(this.Content)); + if (totalSize == 0) + { + totalSize = this.Chunks.Sum(c => (long)c.ChunkSize); + } + return totalSize; } } - /// - /// TotalSize is the total size of the file in bytes, calculated as the sum of all chunk sizes. - /// - public long TotalSize => this.Chunks.Sum(c => c.ChunkSize); - /// /// Chunks is a list of KDriveChunk objects representing the file's content split into chunks. /// @@ -60,7 +67,7 @@ public string TotalChunkHash /// /// Content is a stream representing the file's content. /// - public required Stream Content { get; init; } + public Stream? Content { get; init; } /// /// In case of conflict with an existing file, it define how to manage the conflict @@ -73,19 +80,31 @@ public string TotalChunkHash /// Define the size of each chunk (except the last one) public void SplitIntoChunks(int chunkSize) { + if (this.Content == null) + { + throw new InvalidOperationException("Content stream is null."); + } + var buffer = new byte[chunkSize]; int chunkNumber = 0; int bytesRead; this.Content.Position = 0; - + using var fileSha256 = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); while ((bytesRead = this.Content.Read(buffer, 0, chunkSize)) > 0) { byte[] content = [.. buffer.Take(bytesRead)]; - this.Chunks.Add(new KDriveChunk(content, chunkNumber++, SHA256.HashData(content))); + var chunkHash = SHA256.HashData(content); + this.Chunks.Add(new KDriveChunk(content, chunkNumber++, chunkHash)); + fileSha256.AppendData(Encoding.UTF8.GetBytes(Convert.ToHexString(chunkHash).ToLowerInvariant())); } - this.Content.Position = 0; + this.Content.Dispose(); + GC.Collect(); + + this.TotalChunkHash = this.Chunks.Count > 1 ? + this.TotalChunkHash = Convert.ToHexString(fileSha256.GetHashAndReset()) + : this.Chunks.First().ChunkHash; } /// @@ -111,5 +130,15 @@ public string ConvertConflictChoice() _ => "error" }; } + + /// + /// Frees resources used by the KDriveFile instance. + /// + public void Dispose() + { + this.Chunks.ForEach(c => c.Dispose()); + + GC.SuppressFinalize(this); + } } } \ No newline at end of file diff --git a/DriveClient/Models/KDriveFinishRequest.cs b/DriveClient/Models/KDriveFinishRequest.cs new file mode 100644 index 0000000..834c794 --- /dev/null +++ b/DriveClient/Models/KDriveFinishRequest.cs @@ -0,0 +1,14 @@ +namespace kDriveClient.Models +{ + /// + /// Represents the request body for finishing an upload session. + /// + public class KDriveFinishRequest + { + /// + /// The total chunk hash of the uploaded file. + /// + [JsonPropertyName("total_chunk_hash")] + public string TotalChunkHash { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/DriveClient/Models/KDriveUploadResponse.cs b/DriveClient/Models/KDriveUploadResponse.cs index 0441688..72e10f3 100644 --- a/DriveClient/Models/KDriveUploadResponse.cs +++ b/DriveClient/Models/KDriveUploadResponse.cs @@ -13,7 +13,33 @@ public class KDriveUploadResponseWraper : ApiResultBase /// /// Data containing the details of the uploaded file. /// - public KDriveUploadResponse? Data { get; set; } + public KDriveUploadDataResponse? Data { get; set; } + } + + /// + /// KDrive upload data response model. + /// + public class KDriveUploadDataResponse : ApiResultBase + { + /// + /// KDrive file upload response details. + /// + public KDriveUploadResponse? File { get; set; } + + /// + /// Token for the uploaded request. + /// + public string? Token { get; set; } + + /// + /// Status of the upload operation. + /// + public bool Result { get; set; } + + /// + /// Additional message regarding the upload operation. + /// + public string? Message { get; set; } } /// @@ -65,5 +91,80 @@ public class KDriveUploadResponse : ApiResultBase /// File hash. /// public string? Hash { get; set; } + + /// + /// File or folder type. + /// + public required string Type { get; set; } + + /// + /// Current status of the file. + /// + public required string Status { get; set; } + + /// + /// Visibility of the file. + /// + public required string Visibility { get; set; } + + /// + /// Drive ID associated with the file. + /// + public int Drive_id { get; set; } + + /// + /// Depth of the file in the directory structure. + /// + public int Depth { get; set; } + + /// + /// Creator user ID. + /// + public int Created_by { get; set; } + + /// + /// Timestamp when the file was created. + /// + public int Created_at { get; set; } + + /// + /// Timestamp when the file was added to the drive. + /// + public int Added_at { get; set; } + + /// + /// Timestamp when the file was last modified. + /// + public int Last_modified_at { get; set; } + + /// + /// Timestamp when the file was last edited. + /// + public int Last_modified_by { get; set; } + + /// + /// Timestamp of last file version. + /// + public int Revised_at { get; set; } + + /// + /// Timestamp when the file was last updated. + /// + public int Updated_at { get; set; } + + /// + /// Id of parent directory. + /// + public int Parent_id { get; set; } + + /// + /// Extension type of the file. + /// + public string? Extension_type { get; set; } + + /// + /// Antivirus scan status of the file. + /// + public string? Scan_status { get; set; } } } \ No newline at end of file diff --git a/DriveClient/kDriveClient.csproj b/DriveClient/kDriveClient.csproj index 7953db7..68c15a0 100644 --- a/DriveClient/kDriveClient.csproj +++ b/DriveClient/kDriveClient.csproj @@ -23,14 +23,26 @@ Native .NET logging support README.md https://github.com/anthonychaussin/DriveClient infomaniak;kDrive;wrapper;c#;DotNet - ✨ New features -- Added support for per-file `conflict_mode` (`error`, `rename`, `version`) via `KDriveFile.ConflictMode` + + ✨ New features + - Improve chunk size estimation + - Improve memory management + - Complete kDrive File Object + - Fix upload response wrapper + - fix hash computation + - bypass speed test if custom chunk size is provided + - fix user-agent version + - Add custom chunk size support and fix total_chunk_hash calculation + - Add optional chunkSize parameter to KDriveClient constructor + - Fix total_chunk_hash calculation to hash concatenated chunk hash hex strings + - Use IncrementalHash for efficient hash computation during file chunking + en LICENSE.txt True True link - 1.0.2 + 1.0.3 diff --git a/DriveClient/kDriveClient/KDriveClient.cs b/DriveClient/kDriveClient/KDriveClient.cs index b3d0b06..fb94cb8 100644 --- a/DriveClient/kDriveClient/KDriveClient.cs +++ b/DriveClient/kDriveClient/KDriveClient.cs @@ -1,5 +1,7 @@ using kDriveClient.Helpers; using kDriveClient.Models; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Net.Http.Headers; using System.Reflection; using System.Threading.RateLimiting; @@ -36,7 +38,7 @@ public partial class KDriveClient : IKDriveClient /// private RateLimiter RateLimiter { get; set; } = new FixedWindowRateLimiter(new FixedWindowRateLimiterOptions { - PermitLimit = 60, + PermitLimit = 59, Window = TimeSpan.FromMinutes(1), AutoReplenishment = true }); @@ -81,7 +83,7 @@ public KDriveClient(string token, long driveId, ILogger? logger) : /// Choose if we should make a speed test to optimize chunks /// Number of parrallels threads /// Logger - public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger) : this(token, driveId, autoChunk, parallelism, logger, null) + public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger) : this(token, driveId, autoChunk, parallelism, logger, null, null) { } /// @@ -93,7 +95,8 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, /// Number of parrallels threads /// Logger /// Custome HttpClient - public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger, HttpClient? httpClient = null) + /// Optional custom chunk size in bytes. If not specified, will be calculated dynamically based on speed test when autoChunk is true. + public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger, HttpClient? httpClient = null, int? chunkSize = null) { DriveId = driveId; Parallelism = parallelism; @@ -101,14 +104,25 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, string version = Assembly.GetEntryAssembly()?.GetName().Version?.ToString() ?? "unknown"; HttpClient = httpClient ?? new HttpClient { BaseAddress = new Uri("https://api.infomaniak.com") }; HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); - HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd("kDriveClient.NET/version"); + HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd("kDriveClient.NET/" + GetVersion()); this.Logger?.LogInformation("KDriveClient initialized with Drive ID: {DriveId}", DriveId); + if (autoChunk) { this.Logger?.LogInformation("Auto chunking enabled, initializing upload strategy..."); - InitializeUploadStrategyAsync().GetAwaiter().GetResult(); + InitializeUploadStrategyAsync(chunkSize).GetAwaiter().GetResult(); this.Logger?.LogInformation("Upload strategy initialized with direct upload threshold: {Threshold} bytes and dynamic chunk size: {ChunkSize} bytes", DirectUploadThresholdBytes, DynamicChunkSizeBytes); } + else if (chunkSize is null) // If autoChunk is disabled and no custom chunk size provided, use a default + { + DynamicChunkSizeBytes = 1024 * 1024; // Default to 1MB chunks + this.Logger?.LogInformation("Using default chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes); + } + else + { + DynamicChunkSizeBytes = chunkSize.Value; + this.Logger?.LogInformation("Using custom chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes); + } } /// @@ -141,10 +155,19 @@ public async Task UploadAsync(KDriveFile file, Cancellatio /// protected virtual async Task SendAsync(HttpRequestMessage request, CancellationToken ct = default) { - if (!(await RateLimiter.AcquireAsync(1, ct)).IsAcquired) + using var lease = await RateLimiter.AcquireAsync(1, ct); + if (!lease.IsAcquired) { - Logger?.LogWarning("Rate limit exceeded for request: {RequestMethod} {RequestUri}", request.Method, request.RequestUri); - throw new HttpRequestException("Rate limit exceeded"); + if (lease.TryGetMetadata("RETRY_AFTER", out var obj) && obj is TimeSpan retryAfter) + { + Logger?.LogInformation("Rate limit reached. Retry-After {RetryAfter}", retryAfter); + await Task.Delay(retryAfter, ct); + } + else + { + Logger?.LogWarning("Rate limit exceeded for request: {RequestMethod} {RequestUri}", request.Method, request.RequestUri); + throw new HttpRequestException("Rate limit exceeded"); + } } Logger?.LogInformation("Sending request: {RequestMethod} {RequestUri}", request.Method, request.RequestUri); @@ -159,9 +182,35 @@ protected virtual async Task SendAsync(HttpRequestMessage r /// private async Task SendWithErrorHandlingAsync(HttpRequestMessage request, CancellationToken ct = default) { - var response = await HttpClient.SendAsync(request, ct); + var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); return await KDriveJsonHelper.DeserializeResponseAsync(response, ct); } + + /// + /// Gets the version of the assembly. + /// + /// The verstion of the assembly + [RequiresAssemblyFiles("Calls System.Reflection.Assembly.Location")] + private static string GetVersion() + { + var asm = typeof(KDriveClient).Assembly; + var infoVer = asm.GetCustomAttribute()?.InformationalVersion; + if (!string.IsNullOrWhiteSpace(infoVer)) return infoVer; + var asmVer = asm.GetName().Version?.ToString(); + if (!string.IsNullOrWhiteSpace(asmVer)) return asmVer; + try + { + var loc = asm.Location; + if (!string.IsNullOrWhiteSpace(loc)) + { + var fvi = FileVersionInfo.GetVersionInfo(loc); + if (!string.IsNullOrWhiteSpace(fvi.FileVersion)) return fvi.FileVersion!; + } + } + catch { } + + return "unknown"; + } } -} +} \ No newline at end of file diff --git a/DriveClient/kDriveClient/SpeedTest.cs b/DriveClient/kDriveClient/SpeedTest.cs index 753c025..f16569e 100644 --- a/DriveClient/kDriveClient/SpeedTest.cs +++ b/DriveClient/kDriveClient/SpeedTest.cs @@ -8,13 +8,29 @@ namespace kDriveClient.kDriveClient /// public partial class KDriveClient { + private const int MaxRequestsPerMinute = 60; + private const int MinChunkBytes = 100 * 1024 * 1024; + private const long MaxChunkBytes = 1L * 1024 * 1024 * 1024; + private const double Safety = 1.10; + /// /// Initializes the upload strategy by performing a speed test. /// + /// Optional custom chunk size in bytes. If specified, only DirectUploadThresholdBytes will be calculated from speed test. /// Cancellation token to cancel the operation. /// A task that represents the asynchronous operation. - private async Task InitializeUploadStrategyAsync(CancellationToken ct = default) + private async Task InitializeUploadStrategyAsync(int? customChunkSize = null, CancellationToken ct = default) { + if (customChunkSize != null) + { + DynamicChunkSizeBytes = (int)customChunkSize; + DirectUploadThresholdBytes = (int)customChunkSize; + this.Logger?.LogInformation("Custom chunk size is provided. Speed test is no longer needed"); + this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (custom)", + DirectUploadThresholdBytes, DynamicChunkSizeBytes); + return; + } + this.Logger?.LogInformation("Starting upload strategy initialization..."); var buffer = new byte[1024 * 1024]; RandomNumberGenerator.Fill(buffer); @@ -62,11 +78,16 @@ private async Task InitializeUploadStrategyAsync(CancellationToken ct = default) await CancelUploadSessionRequest(SessionToken, ct); this.Logger?.LogInformation("Upload session finalized successfully."); - var speedBytesPerSec = buffer.Length / (sw.ElapsedMilliseconds / 1000.0); + var v = buffer.Length / Math.Max(0.001, sw.Elapsed.TotalSeconds); + Logger?.LogInformation("Measured upload speed: {SpeedF2} MB/s", v / (1024 * 1024.0)); + + var target = Math.Min(Math.Max((long)Math.Ceiling((long)Math.Ceiling(Parallelism * v * 60.0 / MaxRequestsPerMinute) * Safety), MinChunkBytes), MaxChunkBytes); + + DynamicChunkSizeBytes = (int)target; + + DirectUploadThresholdBytes = (long)(target * 1.5); - DirectUploadThresholdBytes = (long)speedBytesPerSec; - DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9); - this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes}", + this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (calculated from speed test)", DirectUploadThresholdBytes, DynamicChunkSizeBytes); } } diff --git a/DriveClient/kDriveClient/Upload.cs b/DriveClient/kDriveClient/Upload.cs index 45c02b5..c52da1f 100644 --- a/DriveClient/kDriveClient/Upload.cs +++ b/DriveClient/kDriveClient/Upload.cs @@ -68,19 +68,28 @@ public async Task UploadFileChunkedAsync(KDriveFile file, this.Logger?.LogInformation("Starting chunked upload for file '{FileName}' with size {FileSize} bytes...", file.Name, file.TotalSize); var (sessionToken, uploadUrl) = await StartUploadSessionAsync(file, ct); this.Logger?.LogInformation("Upload session started with token '{SessionToken}' and URL '{UploadUrl}' for file '{FileName}'.", sessionToken, uploadUrl, file.Name); - - await Parallel.ForEachAsync(file.Chunks, new ParallelOptions + try { - MaxDegreeOfParallelism = Parallelism, - CancellationToken = ct - }, async (chunk, token) => + await Parallel.ForEachAsync(file.Chunks, new ParallelOptions + { + MaxDegreeOfParallelism = Parallelism, + CancellationToken = ct + }, async (chunk, token) => + { + this.Logger?.LogInformation("Uploading chunk {ChunkNumber}/{TotalChunks} for file '{FileName}'...", chunk.ChunkNumber + 1, file.Chunks.Count, file.Name); + await UploadChunkAsync(uploadUrl, sessionToken, chunk, file, token); + chunk.Clean(); + }); + + this.Logger?.LogInformation("All chunks uploaded successfully for file '{FileName}'.", file.Name); + return await FinishUploadSessionAsync(sessionToken, file.TotalChunkHash, ct); + } + catch (Exception ex) { - this.Logger?.LogInformation("Uploading chunk {ChunkNumber}/{TotalChunks} for file '{FileName}'...", chunk.ChunkNumber + 1, file.Chunks.Count, file.Name); - await UploadChunkAsync(uploadUrl, sessionToken, chunk, file, token); - }); - - this.Logger?.LogInformation("All chunks uploaded successfully for file '{FileName}'.", file.Name); - return await FinishUploadSessionAsync(sessionToken, file.TotalChunkHash, ct); + this.Logger?.LogError("Error while uploading file: {error}.", ex.Message); + await this.CancelUploadSessionRequest(sessionToken, ct); + throw; + } } /// @@ -120,10 +129,13 @@ private async Task UploadChunkAsync(string uploadUrl, string sessionToken, KDriv { this.Logger?.LogInformation("Uploading chunk {ChunkNumber}/{TotalChunks} for file '{FileName}' with size {ChunkSize} bytes...", chunk.ChunkNumber + 1, file.Chunks.Count, file.Name, chunk.ChunkSize); + var response = await SendAsync(KDriveRequestFactory.CreateChunkUploadRequest(uploadUrl, sessionToken, this.DriveId, chunk), ct); try { response = await KDriveJsonHelper.DeserializeResponseAsync(response, ct); + + chunk.Clean(); } catch (HttpRequestException ex) { diff --git a/DriveClientTests/KDriveFileTests.cs b/DriveClientTests/KDriveFileTests.cs index 0eee323..a6697d1 100644 --- a/DriveClientTests/KDriveFileTests.cs +++ b/DriveClientTests/KDriveFileTests.cs @@ -33,7 +33,14 @@ public void SplitIntoChunksTest(int totalSize, int nbChunk) foreach (var chunk in file.Chunks) { Assert.IsFalse(string.IsNullOrWhiteSpace(chunk.ChunkHash)); - Assert.AreEqual(chunk.Content.Length, chunk.ChunkSize); + if(chunk != file.Chunks.Last()) + { + Assert.AreEqual(1000, chunk.ChunkSize); + } + else + { + Assert.AreEqual(totalSize % 1000 == 0 ? 1000 : totalSize % 1000, chunk.ChunkSize); + } } Assert.IsFalse(string.IsNullOrWhiteSpace(file.TotalChunkHash)); diff --git a/DriveClientTests/kDriveClient/KDriveClientTests.cs b/DriveClientTests/kDriveClient/KDriveClientTests.cs index 1833239..e863908 100644 --- a/DriveClientTests/kDriveClient/KDriveClientTests.cs +++ b/DriveClientTests/kDriveClient/KDriveClientTests.cs @@ -13,7 +13,7 @@ public async Task UploadDirect_Should_ConstructProperRequest() { var response = new HttpResponseMessage(HttpStatusCode.OK) { - Content = new StringContent("{\"Result\":\"success\",\"Data\":{\"id\":123,\"name\":\"example.txt\",\"path\":\"/Private/\",\"hash\":\"test\",\"mime_type\":\"text/text\"}}") + Content = new StringContent("""{"result":"success","data":{"file":{"id":123,"name":"example.txt","path":"/Private","hash":"test","mime_type":"text/text", "visibility": "private_folder", "status": "ok", "type": "file"}}}""") }; var handler = new FakeHttpMessageHandler(response); @@ -25,7 +25,7 @@ public async Task UploadDirect_Should_ConstructProperRequest() DirectoryPath = "/documents", Content = new MemoryStream([1, 2, 3]) }; - file.SplitIntoChunks(1000); + var result = await client.UploadAsync(file); Assert.AreEqual(123, result.Id);