Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions DriveClient/Helpers/KDriveJsonContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace kDriveClient.Helpers
[JsonSerializable(typeof(KDriveFile))]
[JsonSerializable(typeof(KDriveChunk))]
[JsonSerializable(typeof(KDriveUploadResponseWraper))]
[JsonSerializable(typeof(KDriveFinishRequest))]
[JsonSourceGenerationOptions(PropertyNameCaseInsensitive = true)]
public partial class KDriveJsonContext : JsonSerializerContext
{
Expand Down
7 changes: 2 additions & 5 deletions DriveClient/Helpers/KDriveRequestFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,8 @@ public static HttpRequestMessage CreateChunkUploadRequest(string baseUrl, string
/// <returns>An HttpRequestMessage configured to finish the upload session.</returns>
public static HttpRequestMessage CreateFinishSessionRequest(long driveId, string sessionToken, string totalChunkHash)
{
var content = new StringContent(JsonSerializer.Serialize(new
{
total_chunk_hash = $"sha256:{totalChunkHash.ToLowerInvariant()}"
}, KDriveJsonContext.Default.Object));

// Send empty JSON body - kDrive validates file integrity via individual chunk hashes
var content = new StringContent("{}");
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");

return new HttpRequestMessage(HttpMethod.Post, $"/3/drive/{driveId}/upload/session/{sessionToken}/finish")
Expand Down
16 changes: 11 additions & 5 deletions DriveClient/Models/KDriveFile.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
namespace kDriveClient.Models
using System.Text;

namespace kDriveClient.Models
{
/// <summary>
/// KDriveFile represents a file in the kDrive system.
Expand Down Expand Up @@ -36,14 +38,17 @@ public class KDriveFile
public string? SymbolicLink { get; set; }

/// <summary>
/// TotalChunkHash is the SHA-256 hash of the entire file content, computed from all chunks.
/// TotalChunkHash is the SHA-256 hash of the entire file content.
/// </summary>
public string TotalChunkHash
{
get
{
return Convert.ToHexString(
SHA256.HashData(this.Content));
var originalPosition = this.Content.Position;
this.Content.Position = 0;
var hash = Convert.ToHexString(SHA256.HashData(this.Content));
this.Content.Position = originalPosition;
return hash;
}
}

Expand Down Expand Up @@ -82,7 +87,8 @@ public void SplitIntoChunks(int chunkSize)
while ((bytesRead = this.Content.Read(buffer, 0, chunkSize)) > 0)
{
byte[] content = [.. buffer.Take(bytesRead)];
this.Chunks.Add(new KDriveChunk(content, chunkNumber++, SHA256.HashData(content)));
var chunkHash = SHA256.HashData(content);
this.Chunks.Add(new KDriveChunk(content, chunkNumber++, chunkHash));
}

this.Content.Position = 0;
Expand Down
16 changes: 16 additions & 0 deletions DriveClient/Models/KDriveFinishRequest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
using System.Text.Json.Serialization;

namespace kDriveClient.Models
{
/// <summary>
/// Represents the request body for finishing an upload session.
/// </summary>
public class KDriveFinishRequest
{
/// <summary>
/// The total chunk hash of the uploaded file.
/// </summary>
[JsonPropertyName("total_chunk_hash")]
public string TotalChunkHash { get; set; } = string.Empty;
}
}
20 changes: 17 additions & 3 deletions DriveClient/kDriveClient/KDriveClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public KDriveClient(string token, long driveId, ILogger<KDriveClient>? logger) :
/// <param name="autoChunk">Choose if we should make a speed test to optimize chunks</param>
/// <param name="parallelism">Number of parrallels threads</param>
/// <param name="logger">Logger</param>
public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger<KDriveClient>? logger) : this(token, driveId, autoChunk, parallelism, logger, null)
public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger<KDriveClient>? logger) : this(token, driveId, autoChunk, parallelism, logger, null, null)
{ }

/// <summary>
Expand All @@ -93,7 +93,8 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism,
/// <param name="parallelism">Number of parrallels threads</param>
/// <param name="logger">Logger</param>
/// <param name="httpClient">Custome HttpClient</param>
public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger<KDriveClient>? logger, HttpClient? httpClient = null)
/// <param name="chunkSize">Optional custom chunk size in bytes. If not specified, will be calculated dynamically based on speed test when autoChunk is true.</param>
public KDriveClient(string token, long driveId, bool autoChunk, int parallelism, ILogger<KDriveClient>? logger, HttpClient? httpClient = null, int? chunkSize = null)
{
DriveId = driveId;
Parallelism = parallelism;
Expand All @@ -103,12 +104,25 @@ public KDriveClient(string token, long driveId, bool autoChunk, int parallelism,
HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
HttpClient.DefaultRequestHeaders.UserAgent.ParseAdd("kDriveClient.NET/version");
this.Logger?.LogInformation("KDriveClient initialized with Drive ID: {DriveId}", DriveId);

if (chunkSize.HasValue)
{
DynamicChunkSizeBytes = chunkSize.Value;
this.Logger?.LogInformation("Using custom chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes);
}

if (autoChunk)
{
this.Logger?.LogInformation("Auto chunking enabled, initializing upload strategy...");
InitializeUploadStrategyAsync().GetAwaiter().GetResult();
InitializeUploadStrategyAsync(chunkSize).GetAwaiter().GetResult();
this.Logger?.LogInformation("Upload strategy initialized with direct upload threshold: {Threshold} bytes and dynamic chunk size: {ChunkSize} bytes", DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}
else if (!chunkSize.HasValue)
{
// If autoChunk is disabled and no custom chunk size provided, use a default
DynamicChunkSizeBytes = 1024 * 1024; // Default to 1MB chunks
this.Logger?.LogInformation("Using default chunk size: {ChunkSize} bytes", DynamicChunkSizeBytes);
}
}

/// <summary>
Expand Down
18 changes: 14 additions & 4 deletions DriveClient/kDriveClient/SpeedTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ public partial class KDriveClient
/// <summary>
/// Initializes the upload strategy by performing a speed test.
/// </summary>
/// <param name="customChunkSize">Optional custom chunk size in bytes. If specified, only DirectUploadThresholdBytes will be calculated from speed test.</param>
/// <param name="ct">Cancellation token to cancel the operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private async Task InitializeUploadStrategyAsync(CancellationToken ct = default)
private async Task InitializeUploadStrategyAsync(int? customChunkSize = null, CancellationToken ct = default)
{
this.Logger?.LogInformation("Starting upload strategy initialization...");
var buffer = new byte[1024 * 1024];
Expand Down Expand Up @@ -65,9 +66,18 @@ private async Task InitializeUploadStrategyAsync(CancellationToken ct = default)
var speedBytesPerSec = buffer.Length / (sw.ElapsedMilliseconds / 1000.0);

DirectUploadThresholdBytes = (long)speedBytesPerSec;
DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9);
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes}",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);

if (!customChunkSize.HasValue)
{
DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9);
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (calculated from speed test)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}
else
{
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (custom)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}
Comment on lines +70 to +80
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (!customChunkSize.HasValue)
{
DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9);
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (calculated from speed test)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}
else
{
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (custom)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}
private async Task InitializeUploadStrategyAsync(int? customChunkSize = null, CancellationToken ct = default)
{
if (customChunkSize != null)
{
DynamicChunkSizeBytes = (int)customChunkSize;
DirectUploadThresholdBytes = (int)customChunkSize;
this.Logger?.LogInformation("Custom chunk size is provided. Speed test is no longer needed");
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (custom)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
return;
}
this.Logger?.LogInformation("Starting upload strategy initialization...");
var buffer = new byte[1024 * 1024];
RandomNumberGenerator.Fill(buffer);
this.Logger?.LogInformation("Generated test Data of size {Size} bytes.", buffer.Length);
var testFile = new Models.KDriveFile
{
Name = "speedtest.dat",
DirectoryPath = "/Private",
Content = new ByteArrayContent(buffer)
{
Headers =
{
ContentType = new MediaTypeHeaderValue("application/octet-stream"),
ContentLength = buffer.Length
}
}.ReadAsStream(ct)
};
testFile.SplitIntoChunks(buffer.Length);
this.Logger?.LogInformation("Test file created with {ChunkCount} chunks of size {ChunkSize} bytes.", testFile.Chunks.Count, buffer.Length);
this.Logger?.LogInformation("Starting upload session for speed test...");
var (SessionToken, UploadUrl) = await StartUploadSessionAsync(testFile, ct);
this.Logger?.LogInformation("Upload session started with token: {SessionToken} and URL: {UploadUrl}", SessionToken, UploadUrl);
this.Logger?.LogInformation("Uploading first chunk of size {ChunkSize} bytes...", testFile.Chunks.First().ChunkSize);
var chunkRequest = KDriveRequestFactory.CreateChunkUploadRequest(UploadUrl, SessionToken, this.DriveId, testFile.Chunks.First());
var sw = System.Diagnostics.Stopwatch.StartNew();
var response = await SendAsync(chunkRequest, ct);
sw.Stop();
this.Logger?.LogInformation("Chunk upload completed in {ElapsedMilliseconds} ms.", sw.ElapsedMilliseconds);
try
{
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException ex)
{
this.Logger?.LogError(ex, "Failed to upload chunk: {Message}", ex.Message);
throw;
}
this.Logger?.LogInformation("Chunk uploaded successfully. Response: {Response}", await response.Content.ReadAsStringAsync(ct));
this.Logger?.LogInformation("Finalizing upload session...");
await CancelUploadSessionRequest(SessionToken, ct);
this.Logger?.LogInformation("Upload session finalized successfully.");
var speedBytesPerSec = buffer.Length / (sw.ElapsedMilliseconds / 1000.0);
DirectUploadThresholdBytes = (long)speedBytesPerSec;
DynamicChunkSizeBytes = (int)(speedBytesPerSec * 0.9);
this.Logger?.LogInformation("Upload strategy initialized: DirectUploadThresholdBytes = {DirectUploadThresholdBytes}, DynamicChunkSizeBytes = {DynamicChunkSizeBytes} (calculated from speed test)",
DirectUploadThresholdBytes, DynamicChunkSizeBytes);
}

Hi,

I think that since the speed test is mainly used to determine the size of the chunks,
if we provide a custom size, we can bypass the speed test.
What do you think?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, sure that's the way better solution.

}
}
}