Ingestion re-implement on updated Elastic.Ingest.Elasticsearch

Directory.Packages.props

@@ -48,7 +48,8 @@
     <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="FakeItEasy" Version="9.0.1" />
-    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.17.1" />
+    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.34.5" />
+    <PackageVersion Include="Elastic.Mapping" Version="0.34.5" />
     <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
     <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.7.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />

docs/cli/assembler/assembler-index.md

@@ -29,9 +29,6 @@ docs-builder assembler index [options...] [-h|--help] [--version]
 `--password` `<string>`
 :   Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional)
 
-`--no-semantic` `<bool?>`
-:   Index without semantic fields (optional)
-
 `--search-num-threads` `<int?>`
 :   The number of search threads the inference endpoint should use. Defaults:   8 (optional)
 

docs/cli/docset/index-command.md

@@ -25,9 +25,6 @@ docs-builder index [options...] [-h|--help] [--version]
 `--password` `<string>`
 :   Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional)
 
-`--no-semantic` `<bool?>`
-:   Index without semantic fields (optional)
-
 `--search-num-threads` `<int?>`
 :   The number of search threads the inference endpoint should use. Defaults:   8 (optional)
 

src/Elastic.Codex/Building/CodexBuildService.cs

@@ -89,7 +89,8 @@ public async Task<CodexBuildResult> BuildAll(
 		if (exporters is not null && buildContexts.Count > 0)
 		{
 			var firstContext = buildContexts[0].BuildContext;
-			sharedExporters = exporters.CreateMarkdownExporters(logFactory, firstContext, context.IndexNamespace).ToArray();
+			firstContext.Endpoints.BuildType = "codex";
+			sharedExporters = exporters.CreateMarkdownExporters(logFactory, firstContext).ToArray();
 			var startTasks = sharedExporters.Select(async e => await e.StartAsync(ctx));
 			await Task.WhenAll(startTasks);
 		}

src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs

@@ -9,6 +9,17 @@ namespace Elastic.Documentation.Configuration;
 public class DocumentationEndpoints
 {
 	public required ElasticsearchEndpoint Elasticsearch { get; init; }
+
+	/// <summary>
+	/// Index namespace for environment isolation. Maps to the <c>{env}</c> placeholder
+	/// in index name templates. Overridden by the assembler <c>--environment</c> flag.
+	/// </summary>
+	public string Environment { get; set; } = "dev";
+
+	/// <summary>
+	/// Build type identifier (assembler, isolated, codex). Controlled by DOCS_BUILD_TYPE env var.
+	/// </summary>
+	public string BuildType { get; set; } = "isolated";
 }
 
 public class ElasticsearchEndpoint
@@ -25,11 +36,8 @@ public class ElasticsearchEndpoint
 	public int IndexNumThreads { get; set; } = 4; // Reduced for Serverless rate limits
 	public bool NoElasticInferenceService { get; set; }
 
-	// index options
-	public string IndexNamePrefix { get; set; } = "semantic-docs";
-
 	// channel buffer options
-	public int BufferSize { get; set; } = 50; // Reduced for Serverless rate limits
+	public int BufferSize { get; set; } = 100; // Reduced for Serverless rate limits
 	public int MaxRetries { get; set; } = 5; // Increased for 429 retries
 
 	// connection options
@@ -43,12 +51,11 @@ public class ElasticsearchEndpoint
 	public X509Certificate? Certificate { get; set; }
 	public bool CertificateIsNotRoot { get; set; }
 	public int? BootstrapTimeout { get; set; }
-	public bool NoSemantic { get; set; }
 	public bool ForceReindex { get; set; }
 
 	/// <summary>
 	/// Enable AI enrichment of documents using LLM-generated metadata.
 	/// When enabled, documents are enriched with summaries, search queries, and questions.
 	/// </summary>
-	public bool EnableAiEnrichment { get; set; }
+	public bool EnableAiEnrichment { get; set; } = true;
 }

src/Elastic.Documentation.Configuration/ElasticsearchEndpointConfigurator.cs

@@ -21,15 +21,13 @@ public record ElasticsearchIndexOptions
 	public string? Password { get; init; }
 
 	// inference options
-	public bool? NoSemantic { get; init; }
-	public bool? EnableAiEnrichment { get; init; }
+	public bool? NoAiEnrichment { get; init; }
 	public int? SearchNumThreads { get; init; }
 	public int? IndexNumThreads { get; init; }
 	public bool? NoEis { get; init; }
 	public int? BootstrapTimeout { get; init; }
 
 	// index options
-	public string? IndexNamePrefix { get; init; }
 	public bool? ForceReindex { get; init; }
 
 	// channel buffer options
@@ -85,8 +83,6 @@ public static async Task ApplyAsync(
 			cfg.IndexNumThreads = options.IndexNumThreads.Value;
 		if (options.NoEis.HasValue)
 			cfg.NoElasticInferenceService = options.NoEis.Value;
-		if (!string.IsNullOrEmpty(options.IndexNamePrefix))
-			cfg.IndexNamePrefix = options.IndexNamePrefix;
 		if (options.BufferSize.HasValue)
 			cfg.BufferSize = options.BufferSize.Value;
 		if (options.MaxRetries.HasValue)
@@ -117,10 +113,8 @@ public static async Task ApplyAsync(
 		if (options.BootstrapTimeout.HasValue)
 			cfg.BootstrapTimeout = options.BootstrapTimeout.Value;
 
-		if (options.NoSemantic.HasValue)
-			cfg.NoSemantic = options.NoSemantic.Value;
-		if (options.EnableAiEnrichment.HasValue)
-			cfg.EnableAiEnrichment = options.EnableAiEnrichment.Value;
+		if (options.NoAiEnrichment == true)
+			cfg.EnableAiEnrichment = false;
 		if (options.ForceReindex.HasValue)
 			cfg.ForceReindex = options.ForceReindex.Value;
 	}

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -45,6 +45,9 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 		_ = builder.Services.AddElasticDocumentationLogging(globalArgs.LogLevel, noConsole: globalArgs.IsMcp);
 		_ = services.AddSingleton(globalArgs);
 
+		var endpoints = ElasticsearchEndpointFactory.Create(builder.Configuration);
+		_ = services.AddSingleton(endpoints);
+
 		return builder.AddServiceDefaults();
 	}
 

src/Elastic.Documentation.ServiceDefaults/Elastic.Documentation.ServiceDefaults.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="OpenTelemetry.Instrumentation.Http"/>
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime"/>
     <PackageReference Include="Crayon"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets"/>
     <PackageReference Include="GitHub.Actions.Core" />
   </ItemGroup>