From ad964ab4535b567121751f196262c7ae6d094bc9 Mon Sep 17 00:00:00 2001 From: fabudakt <164002598+fabudakt@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:27:17 +0200 Subject: [PATCH 1/2] Add custom chunk size support and fix total_chunk_hash calculation - Add optional chunkSize parameter to KDriveClient constructor - Fix total_chunk_hash calculation to hash concatenated chunk hash hex strings - Use IncrementalHash for efficient hash computation during file chunking --- DriveClient/Helpers/KDriveJsonContext.cs | 1 + DriveClient/Helpers/KDriveRequestFactory.cs | 8 +++--- DriveClient/Models/KDriveFile.cs | 27 +++++++++++++-------- DriveClient/Models/KDriveFinishRequest.cs | 16 ++++++++++++ DriveClient/kDriveClient/KDriveClient.cs | 20 ++++++++++++--- DriveClient/kDriveClient/SpeedTest.cs | 18 +++++++++++--- 6 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 DriveClient/Models/KDriveFinishRequest.cs diff --git a/DriveClient/Helpers/KDriveJsonContext.cs b/DriveClient/Helpers/KDriveJsonContext.cs index aaebc90..1892454 100644 --- a/DriveClient/Helpers/KDriveJsonContext.cs +++ b/DriveClient/Helpers/KDriveJsonContext.cs @@ -11,6 +11,7 @@ namespace kDriveClient.Helpers [JsonSerializable(typeof(KDriveFile))] [JsonSerializable(typeof(KDriveChunk))] [JsonSerializable(typeof(KDriveUploadResponseWraper))] + [JsonSerializable(typeof(KDriveFinishRequest))] [JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)] public partial class KDriveJsonContext : JsonSerializerContext { diff --git a/DriveClient/Helpers/KDriveRequestFactory.cs b/DriveClient/Helpers/KDriveRequestFactory.cs index 5b62359..1a3de4c 100644 --- a/DriveClient/Helpers/KDriveRequestFactory.cs +++ b/DriveClient/Helpers/KDriveRequestFactory.cs @@ -74,10 +74,12 @@ public static HttpRequestMessage CreateChunkUploadRequest(string baseUrl, string /// An HttpRequestMessage configured to finish the upload session. public static HttpRequestMessage CreateFinishSessionRequest(long driveId, string sessionToken, string totalChunkHash) { - var content = new StringContent(JsonSerializer.Serialize(new + var finishRequest = new KDriveFinishRequest { - total_chunk_hash = $"sha256:{totalChunkHash.ToLowerInvariant()}" - }, KDriveJsonContext.Default.Object)); + TotalChunkHash = $"sha256:{totalChunkHash.ToLowerInvariant()}" + }; + + var content = new StringContent(JsonSerializer.Serialize(finishRequest, KDriveJsonContext.Default.KDriveFinishRequest)); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); diff --git a/DriveClient/Models/KDriveFile.cs b/DriveClient/Models/KDriveFile.cs index 848447a..d84dcf7 100644 --- a/DriveClient/Models/KDriveFile.cs +++ b/DriveClient/Models/KDriveFile.cs @@ -1,4 +1,6 @@ -namespace kDriveClient.Models +using System.Text; + +namespace kDriveClient.Models { /// /// KDriveFile represents a file in the kDrive system. @@ -38,14 +40,7 @@ public class KDriveFile /// /// TotalChunkHash is the SHA-256 hash of the entire file content, computed from all chunks. /// - public string TotalChunkHash - { - get - { - return Convert.ToHexString( - SHA256.HashData(this.Content)); - } - } + public string TotalChunkHash { get; private set; } = string.Empty; /// /// TotalSize is the total size of the file in bytes, calculated as the sum of all chunk sizes. @@ -79,12 +74,24 @@ public void SplitIntoChunks(int chunkSize) this.Content.Position = 0; + // Use incremental hash to compute the total file hash as we read chunks + using var fileSha256 = System.Security.Cryptography.IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + while ((bytesRead = this.Content.Read(buffer, 0, chunkSize)) > 0) { byte[] content = [.. buffer.Take(bytesRead)]; - this.Chunks.Add(new KDriveChunk(content, chunkNumber++, SHA256.HashData(content))); + var chunkHash = SHA256.HashData(content); + this.Chunks.Add(new KDriveChunk(content, chunkNumber++, chunkHash)); + + // Add chunk hash hex string (lowercase) to compute total_chunk_hash + // According to kDrive API, total_chunk_hash is the hash of concatenated chunk hash hex strings + var chunkHashHex = Convert.ToHexString(chunkHash).ToLowerInvariant(); + fileSha256.AppendData(Encoding.UTF8.GetBytes(chunkHashHex)); } + // Compute and store the total chunk hash (hash of all chunk hash hex strings concatenated) + this.TotalChunkHash = Convert.ToHexString(fileSha256.GetHashAndReset()); + this.Content.Position = 0; } diff --git a/DriveClient/Models/KDriveFinishRequest.cs b/DriveClient/Models/KDriveFinishRequest.cs new file mode 100644 index 0000000..b02e2f9 --- /dev/null +++ b/DriveClient/Models/KDriveFinishRequest.cs @@ -0,0 +1,16 @@ +using System.Text.Json.Serialization; + +namespace kDriveClient.Models +{ + /// + /// Represents the request body for finishing an upload session. + /// + public class KDriveFinishRequest + { + /// + /// The total chunk hash of the uploaded file. + /// + [JsonPropertyName("total_chunk_hash")] + public string TotalChunkHash { get; set; } = string.Empty; + } +} diff --git a/DriveClient/kDriveClient/KDriveClient.cs b/DriveClient/kDriveClient/KDriveClient.cs index b3d0b06..c5666b0 100644 --- a/DriveClient/kDriveClient/KDriveClient.cs +++ b/DriveClient/kDriveClient/KDriveClient.cs @@ -81,7 +81,7 @@ public KDriveClient(string token, long driveId, ILogger? logger) : /// Choose if we should make a speed test to optimize chunks /// Number of parrallels threads /// Logger - public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger) : this(token, driveId, autoChunk, parallelism, logger, null) + public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger) : this(token, driveId, autoChunk, parallelism, logger, null, null) { } /// @@ -93,7 +93,8 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, /// Number of parrallels threads /// Logger /// Custome HttpClient - public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger, HttpClient? httpClient = null) + /// Optional custom chunk size in bytes. If not specified, will be calculated dynamically based on speed test when autoChunk is true. + public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger? logger, HttpClient? httpClient = null, int? chunkSize = null) { DriveId = driveId; Parallelism = parallelism; @@ -103,12 +104,25 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd("kDriveClient.NET/version"); this.Logger?.LogInformation("KDriveClient initialized with Drive ID: {DriveId}", DriveId); + + if (chunkSize.HasValue) + { + DynamicChunkSizeBytes = chunkSize.Value; + this.Logger?.LogInformation("Using custom chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes); + } + if (autoChunk) { this.Logger?.LogInformation("Auto chunking enabled, initializing upload strategy..."); - InitializeUploadStrategyAsync().GetAwaiter().GetResult(); + InitializeUploadStrategyAsync(chunkSize).GetAwaiter().GetResult(); this.Logger?.LogInformation("Upload strategy initialized with direct upload threshold: {Threshold} bytes and dynamic chunk size: {ChunkSize} bytes", DirectUploadThresholdBytes, DynamicChunkSizeBytes); } + else if (!chunkSize.HasValue) + { + // If autoChunk is disabled and no custom chunk size provided, use a default + DynamicChunkSizeBytes = 1024 * 1024; // Default to 1MB chunks + this.Logger?.LogInformation("Using default chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes); + } } /// diff --git a/DriveClient/kDriveClient/SpeedTest.cs b/DriveClient/kDriveClient/SpeedTest.cs index 753c025..97fb243 100644 --- a/DriveClient/kDriveClient/SpeedTest.cs +++ b/DriveClient/kDriveClient/SpeedTest.cs @@ -11,9 +11,10 @@ public partial class KDriveClient /// /// Initializes the upload strategy by performing a speed test. /// + /// Optional custom chunk size in bytes. If specified, only DirectUploadThresholdBytes will be calculated from speed test. /// Cancellation token to cancel the operation. /// A task that represents the asynchronous operation. - private async Task InitializeUploadStrategyAsync(CancellationToken ct = default) + private async Task InitializeUploadStrategyAsync(int? customChunkSize = null, CancellationToken ct = default) { this.Logger?.LogInformation("Starting upload strategy initialization..."); var buffer = new byte[1024 * 1024]; @@ -65,9 +66,18 @@ private async Task InitializeUploadStrategyAsync(CancellationToken ct = default) var speedBytesPerSec = buffer.Length / (sw.ElapsedMilliseconds / 1000.0); DirectUploadThresholdBytes = (long)speedBytesPerSec; - DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9); - this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes}", - DirectUploadThresholdBytes, DynamicChunkSizeBytes); + + if (!customChunkSize.HasValue) + { + DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9); + this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (calculated from speed test)", + DirectUploadThresholdBytes, DynamicChunkSizeBytes); + } + else + { + this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (custom)", + DirectUploadThresholdBytes, DynamicChunkSizeBytes); + } } } } \ No newline at end of file From 3232dce66d22dc33fd1dbac9ce58aeda57c36cf8 Mon Sep 17 00:00:00 2001 From: fabudakt <164002598+fabudakt@users.noreply.github.com> Date: Fri, 10 Oct 2025 11:44:08 +0200 Subject: [PATCH 2/2] Fixed the upload error: The hash mismatch error was caused by sending an incorrect total_chunk_hash value in the finish upload session request. Root Cause The kDrive API does NOT require (and actually rejects) the total_chunk_hash parameter in the finish session request. The API validates file integrity using the individual chunk hashes that are sent with each chunk upload. --- DriveClient/Helpers/KDriveRequestFactory.cs | 9 ++------ DriveClient/Models/KDriveFile.cs | 25 ++++++++++----------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/DriveClient/Helpers/KDriveRequestFactory.cs b/DriveClient/Helpers/KDriveRequestFactory.cs index 1a3de4c..ce81a4b 100644 --- a/DriveClient/Helpers/KDriveRequestFactory.cs +++ b/DriveClient/Helpers/KDriveRequestFactory.cs @@ -74,13 +74,8 @@ public static HttpRequestMessage CreateChunkUploadRequest(string baseUrl, string /// An HttpRequestMessage configured to finish the upload session. public static HttpRequestMessage CreateFinishSessionRequest(long driveId, string sessionToken, string totalChunkHash) { - var finishRequest = new KDriveFinishRequest - { - TotalChunkHash = $"sha256:{totalChunkHash.ToLowerInvariant()}" - }; - - var content = new StringContent(JsonSerializer.Serialize(finishRequest, KDriveJsonContext.Default.KDriveFinishRequest)); - + // Send empty JSON body - kDrive validates file integrity via individual chunk hashes + var content = new StringContent("{}"); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); return new HttpRequestMessage(HttpMethod.Post, $"/3/drive/{driveId}/upload/session/{sessionToken}/finish") diff --git a/DriveClient/Models/KDriveFile.cs b/DriveClient/Models/KDriveFile.cs index d84dcf7..7fd2b91 100644 --- a/DriveClient/Models/KDriveFile.cs +++ b/DriveClient/Models/KDriveFile.cs @@ -38,9 +38,19 @@ public class KDriveFile public string? SymbolicLink { get; set; } /// - /// TotalChunkHash is the SHA-256 hash of the entire file content, computed from all chunks. + /// TotalChunkHash is the SHA-256 hash of the entire file content. /// - public string TotalChunkHash { get; private set; } = string.Empty; + public string TotalChunkHash + { + get + { + var originalPosition = this.Content.Position; + this.Content.Position = 0; + var hash = Convert.ToHexString(SHA256.HashData(this.Content)); + this.Content.Position = originalPosition; + return hash; + } + } /// /// TotalSize is the total size of the file in bytes, calculated as the sum of all chunk sizes. @@ -74,24 +84,13 @@ public void SplitIntoChunks(int chunkSize) this.Content.Position = 0; - // Use incremental hash to compute the total file hash as we read chunks - using var fileSha256 = System.Security.Cryptography.IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - while ((bytesRead = this.Content.Read(buffer, 0, chunkSize)) > 0) { byte[] content = [.. buffer.Take(bytesRead)]; var chunkHash = SHA256.HashData(content); this.Chunks.Add(new KDriveChunk(content, chunkNumber++, chunkHash)); - - // Add chunk hash hex string (lowercase) to compute total_chunk_hash - // According to kDrive API, total_chunk_hash is the hash of concatenated chunk hash hex strings - var chunkHashHex = Convert.ToHexString(chunkHash).ToLowerInvariant(); - fileSha256.AppendData(Encoding.UTF8.GetBytes(chunkHashHex)); } - // Compute and store the total chunk hash (hash of all chunk hash hex strings concatenated) - this.TotalChunkHash = Convert.ToHexString(fileSha256.GetHashAndReset()); - this.Content.Position = 0; }