Changelogs: S3 upload adjustments

docs-builder.slnx

@@ -74,6 +74,7 @@
     <Project Path="src/services/Elastic.Changelog/Elastic.Changelog.csproj" />
     <Project Path="src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj" />
     <Project Path="src/services/Elastic.Documentation.Isolated/Elastic.Documentation.Isolated.csproj" />
+    <Project Path="src/services/Elastic.Documentation.Integrations/Elastic.Documentation.Integrations.csproj" />
     <Project Path="src/services/Elastic.Documentation.Services/Elastic.Documentation.Services.csproj" />
 	<Project Path="src/services/Elastic.Documentation.Search/Elastic.Documentation.Search.csproj" />
   </Folder>
@@ -99,6 +100,7 @@
     <Project Path="tests/Elastic.Markdown.Tests/Elastic.Markdown.Tests.csproj" />
     <Project Path="tests/Navigation.Tests/Navigation.Tests.csproj" />
     <Project Path="tests/Elastic.Changelog.Tests/Elastic.Changelog.Tests.csproj" />
+    <Project Path="tests/Elastic.Documentation.Integrations.Tests/Elastic.Documentation.Integrations.Tests.csproj" />
   </Folder>
   <Project Path=".github/.github.csproj">
     <Build Project="false" />

src/services/Elastic.Changelog/Elastic.Changelog.csproj

@@ -28,6 +28,7 @@
     <ProjectReference Include="..\..\Elastic.Documentation\Elastic.Documentation.csproj"/>
     <ProjectReference Include="..\..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj"/>
     <ProjectReference Include="..\Elastic.Documentation.Services\Elastic.Documentation.Services.csproj"/>
+    <ProjectReference Include="..\Elastic.Documentation.Integrations\Elastic.Documentation.Integrations.csproj"/>
   </ItemGroup>
 
 </Project>

src/services/Elastic.Changelog/Uploading/ChangelogUploadService.cs

@@ -0,0 +1,169 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using System.Text.RegularExpressions;
+using Amazon.S3;
+using Elastic.Changelog.Configuration;
+using Elastic.Documentation.Configuration;
+using Elastic.Documentation.Configuration.Changelog;
+using Elastic.Documentation.Configuration.ReleaseNotes;
+using Elastic.Documentation.Diagnostics;
+using Elastic.Documentation.Integrations.S3;
+using Elastic.Documentation.ReleaseNotes;
+using Elastic.Documentation.Services;
+using Microsoft.Extensions.Logging;
+using Nullean.ScopedFileSystem;
+
+namespace Elastic.Changelog.Uploading;
+
+public enum ArtifactType { Changelog, Bundle }
+
+public enum UploadTargetKind { S3, Elasticsearch }
+
+public record ChangelogUploadArguments
+{
+	public required ArtifactType ArtifactType { get; init; }
+	public required UploadTargetKind Target { get; init; }
+	public required string S3BucketName { get; init; }
+	public string? Config { get; init; }
+	public string? Directory { get; init; }
+}
+
+public partial class ChangelogUploadService(
+	ILoggerFactory logFactory,
+	IConfigurationContext? configurationContext = null,
+	ScopedFileSystem? fileSystem = null,
+	IAmazonS3? s3Client = null
+) : IService
+{
+	private readonly ILogger _logger = logFactory.CreateLogger<ChangelogUploadService>();
+	private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.RealRead;
+	private readonly ChangelogConfigurationLoader? _configLoader = configurationContext != null
+		? new ChangelogConfigurationLoader(logFactory, configurationContext, fileSystem ?? FileSystemFactory.RealRead)
+		: null;
+
+	[GeneratedRegex(@"^[a-zA-Z0-9_-]+$")]
+	private static partial Regex ProductNameRegex();
+
+	private static readonly YamlDotNet.Serialization.IDeserializer EntryDeserializer =
+		ReleaseNotesSerialization.GetEntryDeserializer();
+
+	public async Task<bool> Upload(IDiagnosticsCollector collector, ChangelogUploadArguments args, Cancel ctx)
+	{
+		if (args.Target == UploadTargetKind.Elasticsearch)
+		{
+			_logger.LogWarning("Elasticsearch upload target is not yet implemented; skipping");
+			return true;
+		}
+
+		if (args.ArtifactType == ArtifactType.Bundle)
+		{
+			_logger.LogWarning("Bundle artifact upload is not yet implemented; skipping");
+			return true;
+		}
+
+		var changelogDir = await ResolveChangelogDirectory(collector, args, ctx);
+		if (changelogDir == null)
+			return false;
+
+		if (!_fileSystem.Directory.Exists(changelogDir))
+		{
+			_logger.LogInformation("Changelog directory {Directory} does not exist; nothing to upload", changelogDir);
+			return true;
+		}
+
+		var targets = DiscoverUploadTargets(collector, changelogDir);
+		if (targets.Count == 0)
+		{
+			_logger.LogInformation("No changelog files found to upload in {Directory}", changelogDir);
+			return true;
+		}
+
+		_logger.LogInformation("Found {Count} upload target(s) from {Directory}", targets.Count, changelogDir);
+
+		using var defaultClient = s3Client == null ? new AmazonS3Client() : null;
+		var client = s3Client ?? defaultClient!;
+		var etagCalculator = new S3EtagCalculator(logFactory, _fileSystem);
+		var uploader = new S3IncrementalUploader(logFactory, client, _fileSystem, etagCalculator, args.S3BucketName);
+		var result = await uploader.Upload(targets, ctx);
+
+		_logger.LogInformation("Upload complete: {Uploaded} uploaded, {Skipped} skipped, {Failed} failed", result.Uploaded, result.Skipped, result.Failed);
+
+		if (result.Failed > 0)
+			collector.EmitError(string.Empty, $"{result.Failed} file(s) failed to upload");
+
+		return result.Failed == 0;
+	}
+
+	internal IReadOnlyList<UploadTarget> DiscoverUploadTargets(IDiagnosticsCollector collector, string changelogDir)
+	{
+		var yamlFiles = _fileSystem.Directory.GetFiles(changelogDir, "*.yaml", SearchOption.TopDirectoryOnly)
+			.Concat(_fileSystem.Directory.GetFiles(changelogDir, "*.yml", SearchOption.TopDirectoryOnly))
+			.ToList();
+
+		var targets = new List<UploadTarget>();
+
+		foreach (var filePath in yamlFiles)
+		{
+			var products = ReadProductsFromFragment(filePath);
+			if (products.Count == 0)
+			{
+				_logger.LogDebug("No products found in {File}, skipping", filePath);
+				continue;
+			}
+
+			var fileName = _fileSystem.Path.GetFileName(filePath);
+
+			foreach (var product in products)
+			{
+				if (!ProductNameRegex().IsMatch(product))
+				{
+					collector.EmitWarning(filePath, $"Skipping invalid product name \"{product}\" (must match [a-zA-Z0-9_-]+)");
+					continue;
+				}
+
+				var s3Key = $"{product}/changelogs/{fileName}";
+				targets.Add(new UploadTarget(filePath, s3Key));
+			}
+		}
+
+		return targets;
+	}
+
+	private List<string> ReadProductsFromFragment(string filePath)
+	{
+		try
+		{
+			var content = _fileSystem.File.ReadAllText(filePath);
+			var normalized = ReleaseNotesSerialization.NormalizeYaml(content);
+			var entry = EntryDeserializer.Deserialize<ChangelogEntryDto>(normalized);
+			if (entry?.Products == null)
+				return [];
+
+			return entry.Products
+				.Select(p => p?.Product)
+				.Where(p => !string.IsNullOrWhiteSpace(p))
+				.Select(p => p!)
+				.ToList();
+		}
+		catch (Exception ex)
+		{
+			_logger.LogWarning(ex, "Could not read products from {File}", filePath);
+			return [];
+		}
+	}
+
+	private async Task<string?> ResolveChangelogDirectory(IDiagnosticsCollector collector, ChangelogUploadArguments args, Cancel ctx)
+	{
+		if (!string.IsNullOrWhiteSpace(args.Directory))
+			return args.Directory;
+
+		if (_configLoader == null)
+			return "docs/changelog";
+
+		var config = await _configLoader.LoadChangelogConfiguration(collector, args.Config, ctx);
+		return config?.Bundle?.Directory ?? "docs/changelog";
+	}
+}

src/services/Elastic.Documentation.Assembler/Deploying/IncrementalDeployService.cs

@@ -9,6 +9,7 @@
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Diagnostics;
+using Elastic.Documentation.Integrations.S3;
 using Elastic.Documentation.Services;
 using Microsoft.Extensions.Logging;
 using Nullean.ScopedFileSystem;

src/services/Elastic.Documentation.Assembler/Deploying/Synchronization/AwsS3SyncPlanStrategy.cs

@@ -3,76 +3,13 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Concurrent;
-using System.Diagnostics.CodeAnalysis;
-using System.IO.Abstractions;
-using System.Security.Cryptography;
 using Amazon.S3;
 using Amazon.S3.Model;
+using Elastic.Documentation.Integrations.S3;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Documentation.Assembler.Deploying.Synchronization;
 
-public interface IS3EtagCalculator
-{
-	Task<string> CalculateS3ETag(string filePath, Cancel ctx = default);
-}
-
-public class S3EtagCalculator(ILoggerFactory logFactory, IFileSystem readFileSystem) : IS3EtagCalculator
-{
-	private readonly ILogger<AwsS3SyncPlanStrategy> _logger = logFactory.CreateLogger<AwsS3SyncPlanStrategy>();
-
-	private static readonly ConcurrentDictionary<string, string> EtagCache = new();
-
-	internal const long PartSize = 5 * 1024 * 1024; // 5MB
-
-	[SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")]
-	public async Task<string> CalculateS3ETag(string filePath, Cancel ctx = default)
-	{
-		if (EtagCache.TryGetValue(filePath, out var cachedEtag))
-		{
-			_logger.LogDebug("Using cached ETag for {Path}", filePath);
-			return cachedEtag;
-		}
-
-		var fileInfo = readFileSystem.FileInfo.New(filePath);
-		var fileSize = fileInfo.Length;
-
-		// For files under 5MB, use simple MD5 (matching TransferUtility behavior)
-		if (fileSize <= PartSize)
-		{
-			await using var stream = readFileSystem.FileStream.New(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-			var smallBuffer = new byte[fileSize];
-			var bytesRead = await stream.ReadAsync(smallBuffer.AsMemory(0, (int)fileSize), ctx);
-			var hash = MD5.HashData(smallBuffer.AsSpan(0, bytesRead));
-			var etag = Convert.ToHexStringLower(hash);
-			EtagCache[filePath] = etag;
-			return etag;
-		}
-
-		// For files over 5MB, use multipart format with 5MB parts (matching TransferUtility)
-		var parts = (int)Math.Ceiling((double)fileSize / PartSize);
-
-		await using var fileStream = readFileSystem.FileStream.New(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
-		var partBuffer = new byte[PartSize];
-		var partHashes = new List<byte[]>();
-
-		for (var i = 0; i < parts; i++)
-		{
-			var bytesRead = await fileStream.ReadAsync(partBuffer.AsMemory(0, partBuffer.Length), ctx);
-			var partHash = MD5.HashData(partBuffer.AsSpan(0, bytesRead));
-			partHashes.Add(partHash);
-		}
-
-		// Concatenate all part hashes
-		var concatenatedHashes = partHashes.SelectMany(h => h).ToArray();
-		var finalHash = MD5.HashData(concatenatedHashes);
-
-		var multipartEtag = $"{Convert.ToHexStringLower(finalHash)}-{parts}";
-		EtagCache[filePath] = multipartEtag;
-		return multipartEtag;
-	}
-}
-
 public class AwsS3SyncPlanStrategy(
 	ILoggerFactory logFactory,
 	IAmazonS3 s3Client,

src/services/Elastic.Documentation.Assembler/Elastic.Documentation.Assembler.csproj

@@ -27,6 +27,7 @@
     <ProjectReference Include="..\..\Elastic.Documentation.Links\Elastic.Documentation.Links.csproj" />
     <ProjectReference Include="..\..\Elastic.Markdown\Elastic.Markdown.csproj" />
     <ProjectReference Include="..\Elastic.Documentation.Services\Elastic.Documentation.Services.csproj" />
+    <ProjectReference Include="..\Elastic.Documentation.Integrations\Elastic.Documentation.Integrations.csproj" />
   </ItemGroup>
 
 </Project>

src/services/Elastic.Documentation.Integrations/Elastic.Documentation.Integrations.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+	<PropertyGroup>
+		<TargetFramework>net10.0</TargetFramework>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<Nullable>enable</Nullable>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="AWSSDK.S3" />
+		<PackageReference Include="Microsoft.Extensions.Logging" />
+		<PackageReference Include="System.IO.Abstractions" />
+	</ItemGroup>
+
+</Project>

src/services/Elastic.Documentation.Integrations/S3/S3EtagCalculator.cs

@@ -0,0 +1,67 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Concurrent;
+using System.Diagnostics.CodeAnalysis;
+using System.IO.Abstractions;
+using System.Security.Cryptography;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Documentation.Integrations.S3;
+
+public interface IS3EtagCalculator
+{
+	Task<string> CalculateS3ETag(string filePath, Cancel ctx = default);
+}
+
+public class S3EtagCalculator(ILoggerFactory logFactory, IFileSystem readFileSystem) : IS3EtagCalculator
+{
+	private readonly ILogger _logger = logFactory.CreateLogger<S3EtagCalculator>();
+
+	private readonly ConcurrentDictionary<string, string> _etagCache = new();
+
+	public const long PartSize = 5 * 1024 * 1024; // 5MB — matches TransferUtility default
+
+	[SuppressMessage("Security", "CA5351:Do Not Use Broken Cryptographic Algorithms")]
+	public async Task<string> CalculateS3ETag(string filePath, Cancel ctx = default)
+	{
+		if (_etagCache.TryGetValue(filePath, out var cachedEtag))
+		{
+			_logger.LogDebug("Using cached ETag for {Path}", filePath);
+			return cachedEtag;
+		}
+
+		var fileInfo = readFileSystem.FileInfo.New(filePath);
+		var fileSize = fileInfo.Length;
+
+		if (fileSize <= PartSize)
+		{
+			await using var stream = readFileSystem.FileStream.New(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+			var smallBuffer = new byte[fileSize];
+			var bytesRead = await stream.ReadAsync(smallBuffer.AsMemory(0, (int)fileSize), ctx);
+			var hash = MD5.HashData(smallBuffer.AsSpan(0, bytesRead));
+			var etag = Convert.ToHexStringLower(hash);
+			_etagCache[filePath] = etag;
+			return etag;
+		}
+
+		var parts = (int)Math.Ceiling((double)fileSize / PartSize);
+		await using var fileStream = readFileSystem.FileStream.New(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+		var partBuffer = new byte[PartSize];
+		var partHashes = new List<byte[]>();
+
+		for (var i = 0; i < parts; i++)
+		{
+			var bytesRead = await fileStream.ReadAsync(partBuffer.AsMemory(0, partBuffer.Length), ctx);
+			var partHash = MD5.HashData(partBuffer.AsSpan(0, bytesRead));
+			partHashes.Add(partHash);
+		}
+
+		var concatenatedHashes = partHashes.SelectMany(h => h).ToArray();
+		var finalHash = MD5.HashData(concatenatedHashes);
+		var multipartEtag = $"{Convert.ToHexStringLower(finalHash)}-{parts}";
+		_etagCache[filePath] = multipartEtag;
+		return multipartEtag;
+	}
+}