Add chunking strategies using Docling

api/v1alpha1/unstructureddataproduct_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -27,15 +29,30 @@ type (
 )
 
 const (
-	SourceTypeS3                       UnstructuredDataSourceType      = "s3"
-	DestinationTypeInternalStage       UnstructuredDataDestinationType = "snowflakeInternalStage"
-	ChunkingStrategyRecursiveCharacter ChunkingStrategy                = "recursiveCharacterTextSplitter"
-	ChunkingStrategyMarkdown           ChunkingStrategy                = "markdownTextSplitter"
-	ChunkingStrategyToken              ChunkingStrategy                = "tokenTextSplitter"
+	SourceTypeS3                        UnstructuredDataSourceType      = "s3"
+	DestinationTypeInternalStage        UnstructuredDataDestinationType = "snowflakeInternalStage"
+	ChunkingStrategyRecursiveCharacter  ChunkingStrategy                = "recursiveCharacterTextSplitter"
+	ChunkingStrategyMarkdown            ChunkingStrategy                = "markdownTextSplitter"
+	ChunkingStrategyToken               ChunkingStrategy                = "tokenTextSplitter"
+	ChunkingStrategyDoclingHierarchical ChunkingStrategy                = "doclingHierarchicalChunker"
+	ChunkingStrategyDoclingHybrid       ChunkingStrategy                = "doclingHybridChunker"
 
 	UnstructuredDataProductCondition = "UnstructuredDataProductReady"
 )
 
+func IsDoclingChunkingStrategy(strategy ChunkingStrategy) bool {
+	return strategy == ChunkingStrategyDoclingHierarchical || strategy == ChunkingStrategyDoclingHybrid
+}
+
+func (s *UnstructuredDataProductSpec) ValidateSpec() error {
+	if IsDoclingChunkingStrategy(s.ChunksGeneratorConfig.Strategy) {
+		if s.DocumentProcessorConfig != nil {
+			return fmt.Errorf("documentProcessorConfig must not be set when using Docling chunking strategy %q; Docling performs conversion and chunking in a single step", s.ChunksGeneratorConfig.Strategy)
+		}
+	}
+	return nil
+}
+
 type DocumentProcessorConfig struct {
 	Type          string        `json:"type,omitempty"`
 	DoclingConfig DoclingConfig `json:"doclingConfig,omitempty"`
@@ -59,6 +76,8 @@ type ChunksGeneratorConfig struct {
 	RecursiveCharacterSplitterConfig RecursiveCharacterSplitterConfig `json:"recursiveCharacterSplitterConfig,omitempty"`
 	MarkdownSplitterConfig           MarkdownSplitterConfig           `json:"markdownSplitterConfig,omitempty"`
 	TokenSplitterConfig              TokenSplitterConfig              `json:"tokenSplitterConfig,omitempty"`
+	DoclingHierarchicalChunkerConfig DoclingHierarchicalChunkerConfig `json:"doclingHierarchicalChunkerConfig,omitempty"`
+	DoclingHybridChunkerConfig       DoclingHybridChunkerConfig       `json:"doclingHybridChunkerConfig,omitempty"`
 }
 
 type RecursiveCharacterSplitterConfig struct {
@@ -86,12 +105,22 @@ type TokenSplitterConfig struct {
 	DisallowedSpecial []string `json:"disallowedSpecial,omitempty"`
 }
 
+type DoclingHierarchicalChunkerConfig struct {
+	MergeListItems *bool `json:"mergeListItems,omitempty"`
+}
+
+type DoclingHybridChunkerConfig struct {
+	Tokenizer  string `json:"tokenizer,omitempty"`
+	MaxTokens  int    `json:"maxTokens,omitempty"`
+	MergePeers *bool  `json:"mergePeers,omitempty"`
+}
+
 // UnstructuredDataProductSpec defines the desired state of UnstructuredDataProduct
 type UnstructuredDataProductSpec struct {
-	SourceConfig            SourceConfig            `json:"sourceConfig,omitempty"`
-	DestinationConfig       DestinationConfig       `json:"destinationConfig,omitempty"`
-	DocumentProcessorConfig DocumentProcessorConfig `json:"documentProcessorConfig,omitempty"`
-	ChunksGeneratorConfig   ChunksGeneratorConfig   `json:"chunksGeneratorConfig,omitempty"`
+	SourceConfig            SourceConfig             `json:"sourceConfig,omitempty"`
+	DestinationConfig       DestinationConfig        `json:"destinationConfig,omitempty"`
+	DocumentProcessorConfig *DocumentProcessorConfig `json:"documentProcessorConfig,omitempty"`
+	ChunksGeneratorConfig   ChunksGeneratorConfig    `json:"chunksGeneratorConfig,omitempty"`
 }
 
 type SourceConfig struct {

api/v1alpha1/unstructureddataproduct_types_test.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"testing"
+)
+
+func TestValidateSpec(t *testing.T) {
+	tests := []struct {
+		name    string
+		spec    UnstructuredDataProductSpec
+		wantErr bool
+	}{
+		{
+			name: "docling hierarchical with DocumentProcessorConfig set returns error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyDoclingHierarchical,
+				},
+				DocumentProcessorConfig: &DocumentProcessorConfig{
+					Type: "docling",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "docling hybrid with DocumentProcessorConfig set returns error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyDoclingHybrid,
+				},
+				DocumentProcessorConfig: &DocumentProcessorConfig{
+					Type: "docling",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "docling hierarchical without DocumentProcessorConfig returns no error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyDoclingHierarchical,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "docling hybrid without DocumentProcessorConfig returns no error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyDoclingHybrid,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "non-docling strategy with DocumentProcessorConfig set returns no error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyRecursiveCharacter,
+				},
+				DocumentProcessorConfig: &DocumentProcessorConfig{
+					Type: "docling",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "non-docling strategy without DocumentProcessorConfig returns no error",
+			spec: UnstructuredDataProductSpec{
+				ChunksGeneratorConfig: ChunksGeneratorConfig{
+					Strategy: ChunkingStrategyMarkdown,
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.spec.ValidateSpec()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ValidateSpec() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestIsDoclingChunkingStrategy(t *testing.T) {
+	tests := []struct {
+		name     string
+		strategy ChunkingStrategy
+		want     bool
+	}{
+		{
+			name:     "docling hierarchical",
+			strategy: ChunkingStrategyDoclingHierarchical,
+			want:     true,
+		},
+		{
+			name:     "docling hybrid",
+			strategy: ChunkingStrategyDoclingHybrid,
+			want:     true,
+		},
+		{
+			name:     "recursive character",
+			strategy: ChunkingStrategyRecursiveCharacter,
+			want:     false,
+		},
+		{
+			name:     "markdown",
+			strategy: ChunkingStrategyMarkdown,
+			want:     false,
+		},
+		{
+			name:     "token",
+			strategy: ChunkingStrategyToken,
+			want:     false,
+		},
+		{
+			name:     "empty string",
+			strategy: "",
+			want:     false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := IsDoclingChunkingStrategy(tt.strategy)
+			if got != tt.want {
+				t.Errorf("IsDoclingChunkingStrategy(%q) = %v, want %v", tt.strategy, got, tt.want)
+			}
+		})
+	}
+}

config/crd/bases/operator.dataverse.redhat.com_chunksgenerators.yaml

@@ -48,6 +48,20 @@ spec:
             properties:
               config:
                 properties:
+                  doclingHierarchicalChunkerConfig:
+                    properties:
+                      mergeListItems:
+                        type: boolean
+                    type: object
+                  doclingHybridChunkerConfig:
+                    properties:
+                      maxTokens:
+                        type: integer
+                      mergePeers:
+                        type: boolean
+                      tokenizer:
+                        type: string
+                    type: object
                   markdownSplitterConfig:
                     properties:
                       chunkOverlap:

config/crd/bases/operator.dataverse.redhat.com_unstructureddataproducts.yaml

@@ -50,6 +50,20 @@ spec:
             properties:
               chunksGeneratorConfig:
                 properties:
+                  doclingHierarchicalChunkerConfig:
+                    properties:
+                      mergeListItems:
+                        type: boolean
+                    type: object
+                  doclingHybridChunkerConfig:
+                    properties:
+                      maxTokens:
+                        type: integer
+                      mergePeers:
+                        type: boolean
+                      tokenizer:
+                        type: string
+                    type: object
                   markdownSplitterConfig:
                     properties:
                       chunkOverlap: