From 76b1288c645ce9f754655ce3500788341a0bf07a Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Thu, 9 Apr 2026 10:58:59 +0530 Subject: [PATCH 1/4] Add Data Format aware engine Signed-off-by: Mohit Godwani --- .../CompositeIndexingExecutionEngine.java | 74 +- .../opensearch/composite/CompositeWriter.java | 7 + .../parquet/engine/ParquetIndexingEngine.java | 2 +- .../parquet/writer/ParquetWriter.java | 18 + .../opensearch/index/shard/IndexShardIT.java | 2 +- .../org/opensearch/index/IndexModule.java | 20 +- .../org/opensearch/index/IndexService.java | 17 +- .../org/opensearch/index/IndexSettings.java | 5 + .../index/engine/DataFormatBasedEngine.java | 1068 +++++++++++++++++ .../org/opensearch/index/engine/Engine.java | 4 +- .../index/engine/EngineBackedIndexer.java | 6 +- .../opensearch/index/engine/EngineConfig.java | 39 + .../index/engine/EngineConfigFactory.java | 19 +- .../index/engine/IndexingThrottler.java | 65 + .../index/engine/InternalEngine.java | 42 +- .../LastRefreshedCheckpointListener.java | 48 + .../engine/dataformat/IndexingConfig.java | 12 + .../dataformat/IndexingExecutionEngine.java | 3 +- .../index/engine/dataformat/RefreshInput.java | 10 +- .../index/engine/dataformat/Writer.java | 9 +- .../engine/dataformat/commit/Committer.java | 10 + .../dataformat/commit/CommitterFactory.java | 7 + .../exec/DataFormatAwareEngineFactory.java | 46 +- .../exec/DataFormatAwareIndexerFactory.java | 12 + .../exec/EngineBackedIndexerFactory.java | 19 + .../index/engine/exec/IndexerFactory.java | 12 + .../coord/DataformatAwareCatalogSnapshot.java | 2 +- .../opensearch/index/shard/IndexShard.java | 20 +- .../opensearch/indices/IndicesService.java | 15 +- .../org/opensearch/plugins/EnginePlugin.java | 6 + .../opensearch/index/IndexModuleTests.java | 18 +- .../shard/IndexShardRetentionLeaseTests.java | 5 +- .../index/shard/IndexShardTests.java | 2 +- ...overyWithRemoteTranslogOnPrimaryTests.java | 2 +- .../indices/IndicesServiceTests.java | 4 +- .../index/shard/IndexShardTestCase.java | 64 +- .../test/engine/MockIndexerFactory.java | 21 + 37 files changed, 1504 insertions(+), 231 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java create mode 100644 server/src/main/java/org/opensearch/index/engine/IndexingThrottler.java create mode 100644 server/src/main/java/org/opensearch/index/engine/LastRefreshedCheckpointListener.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/IndexingConfig.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java create mode 100644 server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java create mode 100644 server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java create mode 100644 server/src/main/java/org/opensearch/index/engine/exec/EngineBackedIndexerFactory.java create mode 100644 server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java create mode 100644 test/framework/src/main/java/org/opensearch/test/engine/MockIndexerFactory.java diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index 9383112b7a8b4..b35234760a56d 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -13,6 +13,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.queue.LockablePool; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatPlugin; @@ -60,8 +61,6 @@ public class CompositeIndexingExecutionEngine implements IndexingExecutionEngine private final IndexingExecutionEngine primaryEngine; private final Set> secondaryEngines; private final CompositeDataFormat compositeDataFormat; - private final LockablePool writerPool; - private final AtomicLong writerGenerationCounter; /** * Constructs a CompositeIndexingExecutionEngine by reading index settings to @@ -112,14 +111,6 @@ public CompositeIndexingExecutionEngine( this.secondaryEngines = Set.copyOf(secondaries); this.compositeDataFormat = new CompositeDataFormat(allFormats); - - // Create the writer pool internally, matching the reference code pattern - writerGenerationCounter = new AtomicLong(0); - this.writerPool = new LockablePool<>( - () -> new CompositeWriter(this, writerGenerationCounter.getAndIncrement()), - LinkedList::new, - Runtime.getRuntime().availableProcessors() - ); } /** @@ -178,64 +169,17 @@ public Merger getMerger() { @Override public RefreshResult refresh(RefreshInput refreshInput) throws IOException { - List dataFormatWriters = writerPool.checkoutAll(); - - // Mark each writer as flush-pending before flushing - for (CompositeWriter writer : dataFormatWriters) { - writer.setFlushPending(); - } - - List refreshedSegments = new ArrayList<>(refreshInput.existingSegments()); - List newSegmentList = new ArrayList<>(); - - logger.debug( - "Refreshing composite engine: flushing {} writers, existing segments={}", - dataFormatWriters.size(), - refreshedSegments.size() - ); - - // Flush each writer to disk and build segments from the file infos - for (CompositeWriter writer : dataFormatWriters) { - FileInfos fileInfos = writer.flush(); - Segment.Builder segmentBuilder = Segment.builder(writer.getWriterGeneration()); - boolean hasFiles = false; - for (Map.Entry entry : fileInfos.writerFilesMap().entrySet()) { - logger.debug( - "Writer gen={} flushed format=[{}] files={}", - writer.getWriterGeneration(), - entry.getKey().name(), - entry.getValue().files() - ); - segmentBuilder.addSearchableFiles(entry.getKey(), entry.getValue()); - hasFiles = true; - } - writer.close(); - if (hasFiles) { - newSegmentList.add(segmentBuilder.build()); - } - } - - if (newSegmentList.isEmpty()) { - logger.debug("No new segments produced from flush"); - return null; - } - - logger.debug("Produced {} new segments from flush", newSegmentList.size()); - refreshedSegments.addAll(newSegmentList); - - // Delegate refresh to each per-format engine - RefreshInput emptyInput = RefreshInput.builder().build(); - primaryEngine.refresh(emptyInput); + RefreshResult primary = primaryEngine.refresh(refreshInput); + List secResults = new ArrayList<>(); for (IndexingExecutionEngine engine : secondaryEngines) { - engine.refresh(emptyInput); + secResults.add(engine.refresh(refreshInput)); } - - return new RefreshResult(refreshedSegments); + return null; } @Override public long getNextWriterGeneration() { - return writerGenerationCounter.getAndIncrement(); + throw new UnsupportedOperationException(); } @Override @@ -288,6 +232,12 @@ public CompositeDocumentInput newDocumentInput() { return new CompositeDocumentInput(primaryEngine.getDataFormat(), primaryInput, secondaryInputMap); } + @Override + public void close() throws IOException { + IOUtils.closeWhileHandlingException(primaryEngine); + secondaryEngines.forEach(IOUtils::closeWhileHandlingException); + } + /** * Returns the primary delegate engine. * diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java index 63c68dbbea0cd..7c61500a4c115 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeWriter.java @@ -105,6 +105,7 @@ public WriteResult addDoc(CompositeDocumentInput doc) throws IOException { throw new IllegalStateException("Cannot add document to writer in state " + state.get()); } // Write to primary first + doc.setRowId("__row_id__", rowIdGenerator.currentRowId()); WriteResult primaryResult = primaryWriter.addDoc(doc.getPrimaryInput()); switch (primaryResult) { case WriteResult.Success s -> logger.trace("Successfully added document in primary format [{}]", primaryFormat.name()); @@ -138,6 +139,7 @@ public WriteResult addDoc(CompositeDocumentInput doc) throws IOException { @Override public FileInfos flush() throws IOException { + setFlushPending(); FileInfos.Builder builder = FileInfos.builder(); // Flush primary Optional primaryWfs = primaryWriter.flush().getWriterFileSet(primaryFormat); @@ -161,6 +163,11 @@ public void sync() throws IOException { } } + @Override + public long generation() { + return getWriterGeneration(); + } + @Override public void close() throws IOException { primaryWriter.close(); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java index 9f183ab85e679..60713a2d05b5e 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java @@ -52,7 +52,7 @@ * time, where writer-specific settings (e.g., {@code parquet.max_rows_per_vsr}) are * extracted and applied. */ -public class ParquetIndexingEngine implements IndexingExecutionEngine, Closeable { +public class ParquetIndexingEngine implements IndexingExecutionEngine { private static final Logger logger = LogManager.getLogger(ParquetIndexingEngine.class); diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java index f65628379fe50..a169e89c51082 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/writer/ParquetWriter.java @@ -97,6 +97,24 @@ public void sync() throws IOException { vsrManager.sync(); } + @Override + public long generation() { + return writerGeneration; + } + + @Override + public void lock() { + } + + @Override + public boolean tryLock() { + return false; + } + + @Override + public void unlock() { + } + @Override public void close() throws IOException { vsrManager.close(); diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 0aa358fc71f89..82a812cb4bb56 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -704,7 +704,7 @@ public static final IndexShard newIndexShard( indexService.cache(), indexService.mapperService(), indexService.similarityService(), - shard.getEngineFactory(), + shard.getIndexerFactory(), shard.getEngineConfigFactory(), indexService.getIndexEventListener(), wrapper, diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index c5f5871dfa687..986a5a0ca3801 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -74,9 +74,9 @@ import org.opensearch.index.compositeindex.CompositeIndexSettings; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.EngineConfigFactory; -import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.exec.DataFormatAwareEngineFactory; +import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; @@ -266,7 +266,7 @@ public final class IndexModule { private final IndexSettings indexSettings; private final AnalysisRegistry analysisRegistry; - private final EngineFactory engineFactory; + private final IndexerFactory indexerFactory; private final EngineConfigFactory engineConfigFactory; private SetOnce>> indexReaderWrapper = new SetOnce<>(); @@ -291,14 +291,14 @@ public final class IndexModule { * * @param indexSettings the index settings * @param analysisRegistry the analysis registry - * @param engineFactory the engine factory + * @param indexerFactory the engine factory * @param directoryFactories the available store types */ @InternalApi public IndexModule( final IndexSettings indexSettings, final AnalysisRegistry analysisRegistry, - final EngineFactory engineFactory, + final IndexerFactory indexerFactory, final EngineConfigFactory engineConfigFactory, final Map directoryFactories, final Map compositeDirectoryFactories, @@ -311,7 +311,7 @@ public IndexModule( ) { this.indexSettings = indexSettings; this.analysisRegistry = analysisRegistry; - this.engineFactory = Objects.requireNonNull(engineFactory); + this.indexerFactory = Objects.requireNonNull(indexerFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); this.searchOperationListeners.add(new SearchSlowLog(indexSettings)); this.indexOperationListeners.add(new IndexingSlowLog(indexSettings)); @@ -329,7 +329,7 @@ public IndexModule( public IndexModule( final IndexSettings indexSettings, final AnalysisRegistry analysisRegistry, - final EngineFactory engineFactory, + final IndexerFactory indexerFactory, final EngineConfigFactory engineConfigFactory, final Map directoryFactories, final BooleanSupplier allowExpensiveQueries, @@ -339,7 +339,7 @@ public IndexModule( this( indexSettings, analysisRegistry, - engineFactory, + indexerFactory, engineConfigFactory, directoryFactories, Collections.emptyMap(), @@ -393,8 +393,8 @@ public Index getIndex() { * * @return the engine factory */ - EngineFactory getEngineFactory() { - return engineFactory; + IndexerFactory getIndexerFactory() { + return indexerFactory; } /** @@ -919,7 +919,7 @@ public IndexService newIndexService( new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers, - engineFactory, + indexerFactory, engineConfigFactory, circuitBreakerService, bigArrays, diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index ad475b69fdee0..1f632fe22ceeb 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -79,6 +79,7 @@ import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.engine.MergedSegmentWarmerFactory; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.MapperService; @@ -167,7 +168,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final NamedXContentRegistry xContentRegistry; private final NamedWriteableRegistry namedWriteableRegistry; private final SimilarityService similarityService; - private final EngineFactory engineFactory; + private final IndexerFactory indexerFactory; private final EngineConfigFactory engineConfigFactory; private final IndexWarmer warmer; private volatile Map shards = emptyMap(); @@ -221,7 +222,7 @@ public IndexService( SimilarityService similarityService, ShardStoreDeleter shardStoreDeleter, IndexAnalyzers indexAnalyzers, - EngineFactory engineFactory, + IndexerFactory indexerFactory, EngineConfigFactory engineConfigFactory, CircuitBreakerService circuitBreakerService, BigArrays bigArrays, @@ -329,7 +330,7 @@ public IndexService( this.compositeDirectoryFactory = compositeDirectoryFactory; this.remoteDirectoryFactory = remoteDirectoryFactory; this.recoveryStateFactory = recoveryStateFactory; - this.engineFactory = Objects.requireNonNull(engineFactory); + this.indexerFactory = Objects.requireNonNull(indexerFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE this.readerWrapper = wrapperFactory.apply(this); @@ -381,7 +382,7 @@ public IndexService( SimilarityService similarityService, ShardStoreDeleter shardStoreDeleter, IndexAnalyzers indexAnalyzers, - EngineFactory engineFactory, + IndexerFactory indexerFactory, EngineConfigFactory engineConfigFactory, CircuitBreakerService circuitBreakerService, BigArrays bigArrays, @@ -422,7 +423,7 @@ public IndexService( similarityService, shardStoreDeleter, indexAnalyzers, - engineFactory, + indexerFactory, engineConfigFactory, circuitBreakerService, bigArrays, @@ -789,7 +790,7 @@ protected void closeInternal() { indexCache, mapperService, similarityService, - engineFactory, + indexerFactory, engineConfigFactory, eventListener, readerWrapper, @@ -1370,8 +1371,8 @@ public interface ShardStoreDeleter { void addPendingDelete(ShardId shardId, IndexSettings indexSettings); } - public final EngineFactory getEngineFactory() { - return engineFactory; + public final IndexerFactory getIndexerFactory() { + return indexerFactory; } final CheckedFunction getReaderWrapper() { diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 66a1c29d6a9de..9dd15ad81bc69 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -2371,6 +2371,7 @@ public boolean isDerivedSourceEnabled() { return derivedSourceEnabled; } + /** * Returns whether the pluggable data format feature is enabled for this index. * Requires both the experimental feature flag and the index-level setting. @@ -2380,4 +2381,8 @@ public boolean isDerivedSourceEnabled() { public boolean isPluggableDataFormatEnabled() { return pluggableDataFormatEnabled; } + + public String pluggableDataFormat() { + return "composite"; + } } diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java new file mode 100644 index 0000000000000..cd2945578a207 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java @@ -0,0 +1,1068 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.engine; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.common.Nullable; +import org.opensearch.common.SetOnce; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.concurrent.GatedCloseable; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.queue.LockablePool; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ReleasableLock; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.index.AppendOnlyIndexOperationRetryException; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.VersionType; +import org.opensearch.index.engine.dataformat.*; +import org.opensearch.index.engine.dataformat.commit.Committer; +import org.opensearch.index.engine.exec.*; +import org.opensearch.index.engine.exec.Segment; +import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.engine.exec.coord.CatalogSnapshotManager; +import org.opensearch.index.mapper.*; +import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.seqno.LocalCheckpointTracker; +import org.opensearch.index.seqno.SeqNoStats; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.store.Store; +import org.opensearch.index.translog.DefaultTranslogDeletionPolicy; +import org.opensearch.index.translog.InternalTranslogManager; +import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.TranslogCorruptedException; +import org.opensearch.index.translog.TranslogDeletionPolicy; +import org.opensearch.index.translog.TranslogException; +import org.opensearch.index.translog.TranslogManager; +import org.opensearch.index.translog.TranslogOperationHelper; +import org.opensearch.index.translog.listener.TranslogEventListener; +import org.opensearch.indices.pollingingest.PollingIngestStats; +import org.opensearch.search.suggest.completion.CompletionStats; + +import java.io.Closeable; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiFunction; + +import org.apache.lucene.index.Term; + +import static org.opensearch.index.engine.Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID; + +/** + * An {@link Indexer} implementation that delegates to an {@link IndexingExecutionEngine}. + * This engine manages the full lifecycle of indexing, + * deleting, and searching documents on a single shard by coordinating between the + * underlying data format engine and the translog. + *

+ * Mirrors the responsibilities of {@link InternalEngine} but is decoupled from Lucene: + *

    + *
  • Sequence number generation and local checkpoint tracking
    • + *
  • Translog management for durability and recovery
    • + *
  • Refresh to make documents searchable via catalog snapshots
    • + *
  • Flush to commit catalog snapshots and trim the translog
    • + *
  • Throttling when merges fall behind or indexing buffer is full
    • + *
  • Soft-deletes policy for peer-recovery retention
    • + *
+ * + * @opensearch.experimental + */ +@ExperimentalApi +public class DataFormatBasedEngine implements Indexer { + + private final Logger logger; + private final EngineConfig engineConfig; + private final ShardId shardId; + private final Store store; + + private final IndexingExecutionEngine indexingExecutionEngine; + private final IndexingStrategyPlanner indexingStrategyPlanner; + private final LockablePool> writerPool; + private final AtomicLong writerGenerationCounter; + + private final Map> readerManagers; + private volatile CatalogSnapshotManager catalogSnapshotManager; + private Committer committer; + + // Translog for durability and recovery + private final TranslogManager translogManager; + + // Sequence number tracking + private final LocalCheckpointTracker localCheckpointTracker; + + // Throttling + private final IndexingThrottler throttle; + private final AtomicInteger throttleRequestCount = new AtomicInteger(); + + // Timestamps and seq-no markers + private final AtomicLong maxUnsafeAutoIdTimestamp = new AtomicLong(-1); + private final AtomicLong maxSeenAutoIdTimestamp = new AtomicLong(-1); + private volatile long lastWriteNanos = System.nanoTime(); + + // Lifecycle + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final SetOnce failedEngine = new SetOnce<>(); + private final CountDownLatch closedLatch = new CountDownLatch(1); + + // Concurrency locks + private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); + private final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); + private final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); + private final Lock flushLock = new ReentrantLock(); + private final ReentrantLock failEngineLock = new ReentrantLock(); + + + // Refresh tracker + private final LastRefreshedCheckpointListener lastRefreshedCheckpointListener; + + @Nullable + private final String historyUUID; + + /** + * Constructs a DataFormatBasedEngine. + * + * @param engineConfig the engine configuration + */ + public DataFormatBasedEngine(EngineConfig engineConfig) { + this.logger = Loggers.getLogger(DataFormatBasedEngine.class, engineConfig.getShardId()); + this.engineConfig = engineConfig; + this.shardId = engineConfig.getShardId(); + this.store = engineConfig.getStore(); + this.throttle = new IndexingThrottler(); + + if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { + updateAutoIdTimestamp(Long.MAX_VALUE, true); + } + + boolean success = false; + TranslogManager translogManagerRef = null; + + try { + store.incRef(); + + // init committer with store + Committer constructingCommitter = engineConfig.getCommitterFactory().getCommitter(store); + + // Read history UUID and translog UUID from last commit + final Map userData = constructingCommitter.readLastCommittedUserData(); + String translogUUID = Objects.requireNonNull(userData.get(Translog.TRANSLOG_UUID_KEY)); + + // Initialize translog + // TODO: Once file deleter is merged, we will add relevant listeners + final TranslogEventListener translogEventListener = createInternalTranslogEventListener(); + translogManagerRef = createTranslogManager(translogUUID, translogEventListener); + this.translogManager = translogManagerRef; + + // Initialize local checkpoint tracker from last committed segment infos + this.localCheckpointTracker = createLocalCheckpointTracker(LocalCheckpointTracker::new); + + // Initialize from commit data + this.historyUUID = userData.get(Engine.HISTORY_UUID_KEY); + updateAutoIdTimestamp(Long.parseLong(userData.get(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID)), true); + + + // Move to data format aware writers and readers. + DataFormatRegistry registry = engineConfig.getDataFormatRegistry(); + // Create indexing engine + // Pass committer here as well. + this.indexingExecutionEngine = registry.getIndexingEngine(registry.format(engineConfig.getIndexSettings().pluggableDataFormat()), + engineConfig.getMapperService(), store.shardPath(), engineConfig.getIndexSettings()); + this.writerGenerationCounter = new AtomicLong(committer.getLastCommittedGeneration()); + this.writerPool = new LockablePool<>( + () -> indexingExecutionEngine.createWriter(writerGenerationCounter.getAndIncrement()), + LinkedList::new, + Runtime.getRuntime().availableProcessors() + ); + // Create Reader managers + // We will pass IndexViewProvider to this, which would contain store + // and any index specific attributes useful for reads. + this.readerManagers = registry.getReaderManagers(engineConfig.getMapperService(), + engineConfig.getIndexSettings(), + store.shardPath()); + + this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker); + this.indexingStrategyPlanner = new IndexingStrategyPlanner( + engineConfig.getIndexSettings(), + engineConfig.getShardId(), + new LiveVersionMap(), + maxUnsafeAutoIdTimestamp::get, + () -> 0L, + localCheckpointTracker::getProcessedCheckpoint, + this::hasBeenProcessedBefore, + op -> OpVsEngineDocStatus.OP_NEWER, + (a, b) -> null, + this::updateAutoIdTimestamp, + (a, b) -> null + ); + success = true; + logger.trace("created new DataFormatBasedEngine"); + } catch (IOException | TranslogCorruptedException e) { + throw new EngineCreationFailureException(shardId, "failed to create engine", e); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(translogManagerRef); + if (isClosed.get() == false) { + store.decRef(); + } + } + } + } + + private LocalCheckpointTracker createLocalCheckpointTracker( + BiFunction supplier + ) throws IOException { + final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit( + store.readLastCommittedSegmentsInfo().getUserData().entrySet() + ); + logger.trace("recovered max_seq_no [{}] and local_checkpoint [{}]", seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); + return supplier.apply(seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); + } + + private TranslogEventListener createInternalTranslogEventListener() { + return new TranslogEventListener() { + @Override + public void onAfterTranslogSync() { + try { + // TODO: Handle file deletion policy + translogManager.trimUnreferencedReaders(); + } catch (IOException ex) { + throw new TranslogException(shardId, "failed to trim translog on sync", ex); + } + } + + @Override + public void onAfterTranslogRecovery() { + flush(false, true); + translogManager.trimUnreferencedTranslogFiles(); + } + + @Override + public void onFailure(String reason, Exception ex) { + if (ex instanceof AlreadyClosedException) { + failOnTragicEvent((AlreadyClosedException) ex); + } else { + failEngine(reason, ex); + } + } + }; + } + + private TranslogManager createTranslogManager( + String translogUUID, + TranslogEventListener translogEventListener + ) throws IOException { + TranslogDeletionPolicy deletionPolicy = getTranslogDeletionPolicy(); + return new InternalTranslogManager( + engineConfig.getTranslogConfig(), + engineConfig.getPrimaryTermSupplier(), + engineConfig.getGlobalCheckpointSupplier(), + deletionPolicy, + shardId, + readLock, + () -> localCheckpointTracker, + translogUUID, + translogEventListener, + this::ensureOpen, + engineConfig.getTranslogFactory(), + engineConfig.getStartedPrimarySupplier(), + TranslogOperationHelper.create(engineConfig) + ); + } + + private TranslogDeletionPolicy getTranslogDeletionPolicy() { + TranslogDeletionPolicy custom = null; + if (engineConfig.getCustomTranslogDeletionPolicyFactory() != null) { + custom = engineConfig.getCustomTranslogDeletionPolicyFactory() + .create(engineConfig.getIndexSettings(), engineConfig.retentionLeasesSupplier()); + } + return Objects.requireNonNullElseGet(custom, () -> new DefaultTranslogDeletionPolicy( + engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(), + engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis(), + engineConfig.getIndexSettings().getTranslogRetentionTotalFiles() + )); + } + + @Override + public Engine.IndexResult index(Engine.Index index) throws IOException { + assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field(); + final boolean doThrottle = index.origin().isRecovery() == false; + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + try (Releasable indexThrottle = doThrottle ? throttle.acquireThrottle() : () -> {}) { + lastWriteNanos = index.startTime(); + final IndexingStrategy plan = indexingStrategyPlanner.planOperationAsPrimary(index); + final Engine.IndexResult indexResult; + if (plan.earlyResultOnPreFlightError.isPresent()) { + assert index.origin() == Engine.Operation.Origin.PRIMARY : index.origin(); + indexResult = (Engine.IndexResult) plan.earlyResultOnPreFlightError.get(); + assert indexResult.getResultType() == Engine.Result.Type.FAILURE : indexResult.getResultType(); + } else { + if (index.origin() == Engine.Operation.Origin.PRIMARY) { + index = new Engine.Index( + index.uid(), + index.parsedDoc(), + generateSeqNoForOperationOnPrimary(index), + index.primaryTerm(), + index.version(), + index.versionType(), + index.origin(), + index.startTime(), + index.getAutoGeneratedIdTimestamp(), + index.isRetry(), + index.getIfSeqNo(), + index.getIfPrimaryTerm() + ); + } else { + markSeqNoAsSeen(index.seqNo()); + } + + assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); + + if (plan.executeOpOnEngine) { + logger.debug("Indexing doc id=[{}] seqNo=[{}] primaryTerm=[{}] — writing to engine", + index.id(), index.seqNo(), index.primaryTerm()); + indexResult = indexIntoEngine(index); + } else { + indexResult = + new Engine.IndexResult(plan.version, index.primaryTerm(), index.seqNo(), plan.currentNotFoundOrDeleted); + } + } + return indexResult; + } + } catch (RuntimeException | IOException e) { + maybeFailEngine("index id[" + index.id() + "] origin[" + index.origin() + "]", e); + throw e; + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOException { + Engine.IndexResult indexResult; + if (index.origin() == Engine.Operation.Origin.PRIMARY) { + // Assign a sequence number on the primary + final long seqNo = localCheckpointTracker.generateSeqNo(); + index = new Engine.Index( + index.uid(), index.parsedDoc(), seqNo, index.primaryTerm(), + index.version(), index.versionType(), index.origin(), index.startTime(), + index.getAutoGeneratedIdTimestamp(), index.isRetry(), + index.getIfSeqNo(), index.getIfPrimaryTerm() + ); + } else { + // On replica, advance max seq no + localCheckpointTracker.advanceMaxSeqNo(index.seqNo()); + } + + assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); + + // Convert ParsedDocument to DocumentInput and write via the execution engine's writer + Writer currentWriter = null; + try { + currentWriter = writerPool.getAndLock(); + + //Replace with index#documentInput + DocumentInput docInput = indexingExecutionEngine.newDocumentInput(); + + WriteResult result = currentWriter.addDoc(docInput); + + if (result instanceof WriteResult.Success(long version, long term, long seqNo)) { + indexResult = new Engine.IndexResult(version, term, seqNo, true); + } else { + WriteResult.Failure f = (WriteResult.Failure) result; + indexResult = new Engine.IndexResult(f.cause(), index.version(), index.primaryTerm(), index.seqNo()); + } + } catch (Exception e) { + indexResult = new Engine.IndexResult(e, index.version(), index.primaryTerm(), index.seqNo()); + } finally { + if (currentWriter != null) { + writerPool.releaseAndUnlock(currentWriter); + } + } + + if (index.origin().isFromTranslog() == false) { + final Translog.Location location; + if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) { + location = translogManager.add(new Translog.Index(index, indexResult)); + } else if (indexResult.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && indexResult.getFailure() != null + && !(indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException)) { + throw new UnsupportedOperationException("recording document failure as a no-op in translog is not " + + "supported for Data format engine"); + } else { + location = null; + } + indexResult.setTranslogLocation(location); + } + + // Track the sequence number + localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); + if (indexResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); + } + + localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); + if (indexResult.getTranslogLocation() == null && !(indexResult.getFailure() != null + && (indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException))) { + // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number + assert index.origin().isFromTranslog() || indexResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO; + localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); + } + indexResult.setTook(System.nanoTime() - index.startTime()); + indexResult.freeze(); + return indexResult; + } + + @Override + public Engine.DeleteResult delete(Engine.Delete delete) throws IOException { + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + lastWriteNanos = delete.startTime(); + + final long seqNo; + if (delete.origin() == Engine.Operation.Origin.PRIMARY) { + seqNo = localCheckpointTracker.generateSeqNo(); + } else { + seqNo = delete.seqNo(); + localCheckpointTracker.advanceMaxSeqNo(seqNo); + } + + // TODO: Implement actual delete via IndexingExecutionEngine when supported. + // For now, record the delete in the translog and track the seq no. + final Engine.DeleteResult deleteResult = new Engine.DeleteResult(1L, delete.primaryTerm(), seqNo, true); + + if (delete.origin().isFromTranslog() == false) { + final Translog.Location location = translogManager.add(new Translog.Delete(delete, deleteResult)); + deleteResult.setTranslogLocation(location); + } + + localCheckpointTracker.markSeqNoAsProcessed(seqNo); + if (deleteResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(seqNo); + } + + deleteResult.setTook(System.nanoTime() - delete.startTime()); + deleteResult.freeze(); + return deleteResult; + } catch (RuntimeException | IOException e) { + maybeFailEngine("delete id[" + delete.id() + "] origin[" + delete.origin() + "]", e); + throw e; + } + } + + @Override + public Engine.NoOpResult noOp(Engine.NoOp noOp) throws IOException { + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + final Engine.NoOpResult noOpResult = new Engine.NoOpResult(noOp.primaryTerm(), noOp.seqNo()); + + if (noOp.origin().isFromTranslog() == false) { + final Translog.Location location = translogManager.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); + noOpResult.setTranslogLocation(location); + } + + localCheckpointTracker.markSeqNoAsProcessed(noOp.seqNo()); + if (noOpResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(noOp.seqNo()); + } + + noOpResult.setTook(System.nanoTime() - noOp.startTime()); + noOpResult.freeze(); + return noOpResult; + } catch (RuntimeException | IOException e) { + maybeFailEngine("noOp seq#[" + noOp.seqNo() + "]", e); + throw e; + } + } + + @Override + public Engine.Index prepareIndex( + DocumentMapperForType docMapper, SourceToParse source, + long seqNo, long primaryTerm, long version, VersionType versionType, + Engine.Operation.Origin origin, long autoGeneratedIdTimestamp, + boolean isRetry, long ifSeqNo, long ifPrimaryTerm + ) { + long startTime = System.nanoTime(); + ParsedDocument doc = docMapper.getDocumentMapper().parse(source); + if (docMapper.getMapping() != null) { + doc.addDynamicMappingsUpdate(docMapper.getMapping()); + } + Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id())); + return new Engine.Index( + uid, doc, seqNo, primaryTerm, version, versionType, + origin, startTime, autoGeneratedIdTimestamp, isRetry, ifSeqNo, ifPrimaryTerm + ); + } + + @Override + public Engine.Delete prepareDelete( + String id, long seqNo, long primaryTerm, long version, + VersionType versionType, Engine.Operation.Origin origin, + long ifSeqNo, long ifPrimaryTerm + ) { + long startTime = System.nanoTime(); + Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); + return new Engine.Delete(id, uid, seqNo, primaryTerm, version, versionType, origin, startTime, ifSeqNo, ifPrimaryTerm); + } + + // ---- IndexerLifecycleOperations ---- + + @Override + public synchronized void refresh(String source) throws EngineException { + final long localCheckpointBeforeRefresh = localCheckpointTracker.getProcessedCheckpoint(); + boolean refreshed; + try (GatedCloseable catalogSnapshot = catalogSnapshotManager.acquireSnapshot()) { + if (store.tryIncRef()) { + try { + List> writers = writerPool.checkoutAll(); + List existingSegments = catalogSnapshot.get().getSegments(); + List newSegments = new ArrayList<>(); + List fileInfosList = new ArrayList<>(); + + for (Writer writer : writers) { + FileInfos fileInfos = writer.flush(); + Segment.Builder segmentBuilder = Segment.builder(writer.generation()); + boolean hasFiles = false; + for (Map.Entry entry : fileInfos.writerFilesMap().entrySet()) { + logger.debug( + "Writer gen={} flushed format=[{}] files={}", + writer.generation(), + entry.getKey().name(), + entry.getValue().files() + ); + segmentBuilder.addSearchableFiles(entry.getKey(), entry.getValue()); + hasFiles = true; + } + writer.close(); + if (hasFiles) { + newSegments.add(segmentBuilder.build()); + } + } + logger.debug("Produced {} new segments from flush", newSegments.size()); + + RefreshInput refreshInput = new RefreshInput(existingSegments, newSegments); + RefreshResult result = indexingExecutionEngine.refresh(refreshInput); + catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); + refreshed = true; + } finally { + store.decRef(); + } + if (refreshed) { + lastRefreshedCheckpointListener.updateRefreshedCheckpoint(localCheckpointBeforeRefresh); + } + } + } catch (AlreadyClosedException ex) { + failOnTragicEvent(ex); + throw ex; + } catch (Exception ex) { + try { + failEngine("refresh failed source[" + source + "]", ex); + } catch (Exception inner) { + ex.addSuppressed(inner); + } + throw new RefreshFailedEngineException(shardId, ex); + } + } + + @Override + public void flush(boolean force, boolean waitIfOngoing) throws EngineException { + ensureOpen(); + if (force && waitIfOngoing == false) { + throw new IllegalArgumentException( + "wait_if_ongoing must be true for a force flush: force=" + force + " wait_if_ongoing=" + waitIfOngoing + ); + } + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + if (flushLock.tryLock() == false) { + if (waitIfOngoing == false) { + return; + } + flushLock.lock(); + } + try { + // Refresh first to flush buffered data to segments + refresh("flush"); + translogManager.ensureCanFlush(); + translogManager.rollTranslogGeneration(); + translogManager.trimUnreferencedReaders(); + logger.trace("flush completed"); + } catch (AlreadyClosedException e) { + failOnTragicEvent(e); + throw e; + } catch (Exception e) { + throw new FlushFailedEngineException(shardId, e); + } finally { + flushLock.unlock(); + } + } + } + + @Override + public void flush() { + flush(false, true); + } + + @Override + public boolean shouldPeriodicallyFlush() { + ensureOpen(); + final long localCheckpointOfLastCommit = localCheckpointTracker.getPersistedCheckpoint(); + return translogManager.shouldPeriodicallyFlush( + localCheckpointOfLastCommit, + engineConfig.getIndexSettings().getFlushThresholdSize().getBytes() + ); + } + + @Override + public void writeIndexingBuffer() throws EngineException { + refresh("write indexing buffer"); + } + + @Override + public void forceMerge( + boolean flush, int maxNumSegments, boolean onlyExpungeDeletes, + boolean upgrade, boolean upgradeOnlyAncientSegments, String forceMergeUUID + ) throws EngineException, IOException { + // TODO: Delegate to IndexingExecutionEngine's Merger when merge scheduling is implemented + if (flush) { + flush(false, true); + } + } + + @Override + public long getIndexBufferRAMBytesUsed() { + return indexingExecutionEngine.getNativeBytesUsed(); + } + + @Override + public void activateThrottling() { + int count = throttleRequestCount.incrementAndGet(); + assert count >= 1 : "invalid post-increment throttleRequestCount=" + count; + if (count == 1) { + throttle.activate(); + } + } + + @Override + public void deactivateThrottling() { + int count = throttleRequestCount.decrementAndGet(); + assert count >= 0 : "invalid post-decrement throttleRequestCount=" + count; + if (count == 0) { + throttle.deactivate(); + } + } + + @Override + public boolean isThrottled() { + return throttle.isThrottled(); + } + + @Override + public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { + if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { + updateAutoIdTimestamp(Long.MAX_VALUE, true); + } + final TranslogDeletionPolicy translogDeletionPolicy = translogManager.getDeletionPolicy(); + translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); + translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); + } + + @Override + public boolean refreshNeeded() { + // A refresh is needed if there are operations since the last refresh + return true; + } + + @Override + public boolean maybeRefresh(String source) { + refresh(source); + return true; + } + + @Override + public void maybePruneDeletes() { + // No-op: data-format engines do not maintain Lucene-style delete tombstones + } + + @Override + public void verifyEngineBeforeIndexClosing() throws IllegalStateException { + final long globalCheckpoint = engineConfig.getGlobalCheckpointSupplier().getAsLong(); + final long maxSeqNo = getSeqNoStats(globalCheckpoint).getMaxSeqNo(); + if (globalCheckpoint != maxSeqNo) { + throw new IllegalStateException( + "Global checkpoint [" + globalCheckpoint + "] mismatches maximum sequence number [" + maxSeqNo + "]" + ); + } + } + + // ---- IndexerStateManager ---- + + @Override + public long getMaxSeenAutoIdTimestamp() { + return maxSeenAutoIdTimestamp.get(); + } + + @Override + public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) { + updateAutoIdTimestamp(newTimestamp, true); + } + + private void updateAutoIdTimestamp(long newTimestamp, boolean unsafe) { + assert newTimestamp >= -1 : "invalid timestamp [" + newTimestamp + "]"; + maxSeenAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); + if (unsafe) { + maxUnsafeAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); + } + } + + @Override + public long getMaxSeqNoOfUpdatesOrDeletes() { + return 0L; + } + + @Override + public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { + // no-op + } + + @Override + public long getLastWriteNanos() { + return lastWriteNanos; + } + + @Override + public long getPersistedLocalCheckpoint() { + return localCheckpointTracker.getPersistedCheckpoint(); + } + + @Override + public long getProcessedLocalCheckpoint() { + return localCheckpointTracker.getProcessedCheckpoint(); + } + + @Override + public SeqNoStats getSeqNoStats(long globalCheckpoint) { + return localCheckpointTracker.getStats(globalCheckpoint); + } + + @Override + public long getLastSyncedGlobalCheckpoint() { + return translogManager.getLastSyncedGlobalCheckpoint(); + } + + @Override + public long getMinRetainedSeqNo() { + return 0L; + } + + @Override + public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNumber) throws IOException { + ensureOpen(); + try (Translog.Snapshot snapshot = translogManager.newChangesSnapshot(fromSeqNo, toSeqNumber, false)) { + int count = 0; + while (snapshot.next() != null) { + count++; + } + return count; + } + } + + @Override + public boolean hasCompleteOperationHistory(String reason, long startingSeqNo) { + return getMinRetainedSeqNo() <= startingSeqNo; + } + + @Override + public int fillSeqNoGaps(long primaryTerm) throws IOException { + // No-op: data-format engines do not maintain Lucene-style gaps + return 0; + } + + // ---- IndexerStatistics ---- + + @Override + public CommitStats commitStats() { + // TODO: Implement commit stats from catalog snapshot metadata + return null; + } + + @Override + public DocsStats docStats() { + // TODO: Derive from catalog snapshot segment metadata + return new DocsStats(0, 0, 0); + } + + @Override + public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + return new SegmentsStats(); + } + + @Override + public CompletionStats completionStats(String... fieldNamePatterns) { + return new CompletionStats(); + } + + @Override + public PollingIngestStats pollingIngestStats() { + return null; + } + + @Override + public MergeStats getMergeStats() { + return new MergeStats(); + } + + @Override + public long getIndexThrottleTimeInMillis() { + return throttle.getThrottleTimeInMillis(); + } + + @Override + public long getWritingBytes() { + return 0; + } + + @Override + public long unreferencedFileCleanUpsPerformed() { + return 0; + } + + @Override + public long getNativeBytesUsed() { + return indexingExecutionEngine.getNativeBytesUsed(); + } + + // ---- Indexer top-level methods ---- + + @Override + public EngineConfig config() { + return engineConfig; + } + + @Override + public SafeCommitInfo getSafeCommitInfo() { + return new SafeCommitInfo(localCheckpointTracker.getProcessedCheckpoint(), 0); + } + + @Override + public TranslogManager translogManager() { + return translogManager; + } + + @Override + public Closeable acquireHistoryRetentionLock() { + return () -> {}; + } + + @Override + public Translog.Snapshot newChangesSnapshot( + String source, long fromSeqNo, long toSeqNo, + boolean requiredFullRange, boolean accurateCount + ) throws IOException { + return translogManager.newChangesSnapshot(fromSeqNo, toSeqNo, requiredFullRange); + } + + @Override + public String getHistoryUUID() { + return historyUUID; + } + + @Override + public void flushAndClose() throws IOException { + if (isClosed.get() == false) { + try (ReleasableLock lock = writeLock.acquire()) { + try { + flush(false, true); + } catch (AlreadyClosedException ex) { + logger.debug("engine already closed - skipping flushAndClose"); + } finally { + close(); + } + } + } + awaitPendingClose(); + } + + @Override + public void failEngine(String reason, @Nullable Exception failure) { + if (failEngineLock.tryLock()) { + try { + if (failedEngine.get() != null) { + logger.warn(() -> new ParameterizedMessage( + "tried to fail engine but already failed, ignoring. [{}]", reason), failure); + return; + } + failedEngine.set(failure != null ? failure : new IllegalStateException(reason)); + try { + closeNoLock("engine failed on: [" + reason + "]"); + } finally { + logger.warn(() -> new ParameterizedMessage("failed engine [{}]", reason), failure); + engineConfig.getEventListener().onFailedEngine(reason, failure); + } + } catch (Exception inner) { + if (failure != null) { + inner.addSuppressed(failure); + } + logger.warn("failEngine threw exception", inner); + } + } else { + logger.debug(() -> new ParameterizedMessage( + "tried to fail engine but could not acquire lock [{}]", reason), failure); + } + } + + @Override + public GatedCloseable acquireSnapshot() { + // TODO: Implement catalog snapshot acquisition from segment tracking + throw new UnsupportedOperationException("acquireSnapshot not yet implemented for DataFormatBasedEngine"); + } + + @Override + public GatedCloseable acquireSafeIndexCommit() throws EngineException { + // TODO: Implement when commit coordination is added + throw new UnsupportedOperationException("acquireSafeIndexCommit not yet implemented for DataFormatBasedEngine"); + } + + @Override + public GatedCloseable acquireReader() throws IOException { + // TODO: Implement when reader management is wired to EngineReaderManager + throw new UnsupportedOperationException("acquireReader not yet implemented for DataFormatBasedEngine"); + } + + @Override + public void ensureOpen() { + if (isClosed.get()) { + throw new AlreadyClosedException(shardId + " engine is closed", failedEngine.get()); + } + } + + @Override + public void close() throws IOException { + if (isClosed.get() == false) { + try (ReleasableLock lock = writeLock.acquire()) { + closeNoLock("api"); + } + } + awaitPendingClose(); + } + + private void closeNoLock(String reason) { + if (isClosed.compareAndSet(false, true)) { + assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() + : "Either the write lock must be held or the engine must be currently failing"; + try { + IOUtils.close(indexingExecutionEngine, translogManager); + } catch (Exception e) { + logger.warn("failed to close engine resources", e); + } finally { + try { + store.decRef(); + logger.debug("engine closed [{}]", reason); + } finally { + closedLatch.countDown(); + } + } + } + } + + private void awaitPendingClose() { + try { + closedLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private boolean failOnTragicEvent(AlreadyClosedException ex) { + if (translogManager.getTragicExceptionIfClosed() != null) { + failEngine("already closed by tragic event on the translog", translogManager.getTragicExceptionIfClosed()); + return true; + } else if (failedEngine.get() == null && isClosed.get() == false) { + throw new AssertionError("Unexpected AlreadyClosedException", ex); + } + return false; + } + + private boolean maybeFailEngine(String source, Exception e) { + if (e instanceof AlreadyClosedException) { + return failOnTragicEvent((AlreadyClosedException) e); + } else if (e != null && translogManager.getTragicExceptionIfClosed() == e) { + failEngine(source, e); + return true; + } + return false; + } + + /** + * Returns the underlying {@link IndexingExecutionEngine}. + * + * @return the indexing execution engine + */ + public IndexingExecutionEngine getIndexingExecutionEngine() { + return indexingExecutionEngine; + } + + /** + * Returns the current writer generation counter. + * + * @return the current writer generation + */ + public long getCurrentWriterGeneration() { + return writerGenerationCounter.get(); + } + + private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + if (origin == Engine.Operation.Origin.PRIMARY) { + assert assertPrimaryIncomingSequenceNumber(origin, seqNo); + } else { + // sequence number should be set when operation origin is not primary + assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin; + } + return true; + } + + boolean assertPrimaryIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + // sequence number should not be set when operation origin is primary + assert + seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : + "primary operations must never have an assigned sequence number but was [" + seqNo + "]"; + return true; + } + + private boolean hasBeenProcessedBefore(Engine.Operation op) { + assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "operation is not assigned seq_no"; + return localCheckpointTracker.hasProcessed(op.seqNo()); + } + + private long generateSeqNoForOperationOnPrimary(final Engine.Operation operation) { + assert operation.origin() == Engine.Operation.Origin.PRIMARY; + assert + operation.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : + "ops should not have an assigned seq no. but was: " + operation.seqNo(); + return doGenerateSeqNoForOperation(operation); + } + + private long doGenerateSeqNoForOperation(final Engine.Operation operation) { + return localCheckpointTracker.generateSeqNo(); + } + + private void markSeqNoAsSeen(long seqNo) { + localCheckpointTracker.advanceMaxSeqNo(seqNo); + } + +} diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index 003967ac2b872..8863ea4166e6e 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -345,10 +345,12 @@ protected long getMaxSeqNoFromSearcher(IndexSearcher searcher) throws IOExceptio /** * A throttling class that can be activated, causing the * {@code acquireThrottle} method to block on a lock when throttling - * is enabled + * is enabled. + * This class has been deprecated. See IndexingThrottler.java * * @opensearch.internal */ + @Deprecated protected static final class IndexThrottle { private final CounterMetric throttleTimeMillisMetric = new CounterMetric(); private volatile long startOfThrottleNS; diff --git a/server/src/main/java/org/opensearch/index/engine/EngineBackedIndexer.java b/server/src/main/java/org/opensearch/index/engine/EngineBackedIndexer.java index eb03ff54c0e11..195971ea57914 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineBackedIndexer.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineBackedIndexer.java @@ -9,6 +9,7 @@ package org.opensearch.index.engine; import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.SegmentInfos; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.unit.TimeValue; @@ -16,6 +17,7 @@ import org.opensearch.index.VersionType; import org.opensearch.index.engine.exec.Indexer; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import org.opensearch.index.engine.exec.coord.SegmentInfosCatalogSnapshot; import org.opensearch.index.mapper.DocumentMapperForType; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.merge.MergeStats; @@ -384,7 +386,9 @@ public long getNativeBytesUsed() { public GatedCloseable acquireSnapshot() { // TODO: Replace with a SegmentInfosCatalogSnapshot // For now we throw an exception as this is not yet implemented - throw new UnsupportedOperationException("acquireSnapshot is not supported in EngineBackedIndexer"); + GatedCloseable segmentInfosRef = engine.getSegmentInfosSnapshot(); + SegmentInfosCatalogSnapshot snapshot = new SegmentInfosCatalogSnapshot(segmentInfosRef.get()); + return new GatedCloseable<>(snapshot, segmentInfosRef::close); } @Override diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 93936b2048b85..5950f9c6c33a7 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -55,7 +55,10 @@ import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.commit.CommitterFactory; import org.opensearch.index.mapper.DocumentMapperForType; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.merge.MergedSegmentTransferTracker; import org.opensearch.index.seqno.RetentionLeases; @@ -117,6 +120,9 @@ public final class EngineConfig { private final Supplier documentMapperForTypeSupplier; private final ClusterApplierService clusterApplierService; private final MergedSegmentTransferTracker mergedSegmentTransferTracker; + private final DataFormatRegistry dataFormatRegistry; + private final MapperService mapperService; + private CommitterFactory committerFactory; /** * A supplier of the outstanding retention leases. This is used during merged operations to determine which operations that have been @@ -307,6 +313,9 @@ private EngineConfig(Builder builder) { this.indexReaderWarmer = builder.indexReaderWarmer; this.clusterApplierService = builder.clusterApplierService; this.mergedSegmentTransferTracker = builder.mergedSegmentTransferTracker; + this.dataFormatRegistry = builder.dataFormatRegistry; + this.mapperService = builder.mapperService; + this.committerFactory = builder.committerFactory; } /** @@ -634,6 +643,18 @@ public MergedSegmentTransferTracker getMergedSegmentTransferTracker() { return this.mergedSegmentTransferTracker; } + public DataFormatRegistry getDataFormatRegistry() { + return this.dataFormatRegistry; + } + + public MapperService getMapperService() { + return this.mapperService; + } + + public CommitterFactory getCommitterFactory() { + return this.committerFactory; + } + /** * Builder for EngineConfig class * @@ -672,6 +693,9 @@ public static class Builder { private IndexWriter.IndexReaderWarmer indexReaderWarmer; private ClusterApplierService clusterApplierService; private MergedSegmentTransferTracker mergedSegmentTransferTracker; + private DataFormatRegistry dataFormatRegistry; + private MapperService mapperService; + private CommitterFactory committerFactory; public Builder shardId(ShardId shardId) { this.shardId = shardId; @@ -828,6 +852,21 @@ public Builder mergedSegmentTransferTracker(MergedSegmentTransferTracker mergedS return this; } + public Builder dataFormatRegistry(DataFormatRegistry dataFormatRegistry) { + this.dataFormatRegistry = dataFormatRegistry; + return this; + } + + public Builder mapperService(MapperService mapperService) { + this.mapperService = mapperService; + return this; + } + + public Builder committerFactory(CommitterFactory committerFactory) { + this.committerFactory = committerFactory; + return this; + } + public EngineConfig build() { return new EngineConfig(this); } diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index 3831619d4daf4..bb06e76467e95 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -28,6 +28,8 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.commit.CommitterFactory; import org.opensearch.index.mapper.DocumentMapperForType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.merge.MergedSegmentTransferTracker; @@ -59,6 +61,7 @@ public class EngineConfigFactory { private final CodecServiceFactory codecServiceFactory; private final TranslogDeletionPolicyFactory translogDeletionPolicyFactory; private final List additionalCodecs; + private final CommitterFactory committerFactory; /** default ctor primarily used for tests without plugins */ public EngineConfigFactory(IndexSettings idxSettings) { @@ -81,6 +84,8 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti String codecServiceFactoryOverridingPlugin = null; Optional translogDeletionPolicyFactory = Optional.empty(); String translogDeletionPolicyOverridingPlugin = null; + List committerFactories = new ArrayList<>(); + for (EnginePlugin enginePlugin : enginePlugins) { // get overriding codec service from EnginePlugin if (codecService.isPresent() == false) { @@ -120,6 +125,8 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti // collect all available CodecRegistry instances enginePlugin.getAdditionalCodecs(idxSettings).ifPresent(codecRegistries::add); + + enginePlugin.getCommitterFactory().ifPresent(committerFactories::add); } if (codecService.isPresent() && codecServiceFactory.isPresent()) { @@ -131,10 +138,15 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti ); } + if (committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatenabled())) { + throw new IllegalStateException("Multiple committer factories detected: " + committerFactories); + } + final CodecService instance = codecService.orElse(null); this.codecServiceFactory = (instance != null) ? (config) -> instance : codecServiceFactory.orElse(null); this.translogDeletionPolicyFactory = translogDeletionPolicyFactory.orElse((idxs, rtls) -> null); this.additionalCodecs = Collections.unmodifiableList(codecRegistries); + this.committerFactory = committerFactories.getFirst(); } /** @@ -170,7 +182,9 @@ public EngineConfig newEngineConfig( Supplier documentMapperForTypeSupplier, IndexWriter.IndexReaderWarmer indexReaderWarmer, ClusterApplierService clusterApplierService, - MergedSegmentTransferTracker mergedSegmentTransferTracker + MergedSegmentTransferTracker mergedSegmentTransferTracker, + DataFormatRegistry dataFormatRegistry, + MapperService mapperService ) { CodecService codecServiceToUse = codecService; if (codecService == null && this.codecServiceFactory != null) { @@ -208,6 +222,9 @@ public EngineConfig newEngineConfig( .indexReaderWarmer(indexReaderWarmer) .clusterApplierService(clusterApplierService) .mergedSegmentTransferTracker(mergedSegmentTransferTracker) + .dataFormatRegistry(dataFormatRegistry) + .mapperService(mapperService) + .committerFactory(committerFactory) .build(); } diff --git a/server/src/main/java/org/opensearch/index/engine/IndexingThrottler.java b/server/src/main/java/org/opensearch/index/engine/IndexingThrottler.java new file mode 100644 index 0000000000000..4a29949515e1a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/IndexingThrottler.java @@ -0,0 +1,65 @@ +package org.opensearch.index.engine; + +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ReleasableLock; + +import java.util.concurrent.locks.ReentrantLock; + +class IndexingThrottler { + private final CounterMetric throttleTimeMillisMetric = new CounterMetric(); + private volatile long startOfThrottleNS; + private static final ReleasableLock NOOP_LOCK = new ReleasableLock(new Engine.NoOpLock()); + private final ReleasableLock lockReference = new ReleasableLock(new ReentrantLock()); + private volatile ReleasableLock lock = NOOP_LOCK; + + public Releasable acquireThrottle() { + return lock.acquire(); + } + + /** Activate throttling, which switches the lock to be a real lock */ + public void activate() { + assert lock == NOOP_LOCK : "throttling activated while already active"; + startOfThrottleNS = System.nanoTime(); + lock = lockReference; + } + + /** Deactivate throttling, which switches the lock to be an always-acquirable NoOpLock */ + public void deactivate() { + assert lock != NOOP_LOCK : "throttling deactivated but not active"; + lock = NOOP_LOCK; + + assert startOfThrottleNS > 0 : "Bad state of startOfThrottleNS"; + long throttleTimeNS = System.nanoTime() - startOfThrottleNS; + if (throttleTimeNS >= 0) { + // Paranoia (System.nanoTime() is supposed to be monotonic): time slip may have occurred but never want + // to add a negative number + throttleTimeMillisMetric.inc(TimeValue.nsecToMSec(throttleTimeNS)); + } + } + + long getThrottleTimeInMillis() { + long currentThrottleNS = 0; + if (isThrottled() && startOfThrottleNS != 0) { + currentThrottleNS += System.nanoTime() - startOfThrottleNS; + if (currentThrottleNS < 0) { + // Paranoia (System.nanoTime() is supposed to be monotonic): time slip must have happened, have to ignore this value + currentThrottleNS = 0; + } + } + + return throttleTimeMillisMetric.count() + TimeValue.nsecToMSec(currentThrottleNS); + } + + boolean isThrottled() { + return lock != NOOP_LOCK; + } + + boolean throttleLockIsHeldByCurrentThread() { // to be used in assertions and tests only + if (isThrottled()) { + return lock.isHeldByCurrentThread(); + } + return false; + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index a9b672f20af77..c4dd262aa2137 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -175,7 +175,7 @@ public class InternalEngine extends Engine { private volatile SegmentInfos lastCommittedSegmentInfos; - private final IndexThrottle throttle; + private final IndexingThrottler throttle; private final CombinedDeletionPolicy combinedDeletionPolicy; @@ -257,7 +257,7 @@ public TranslogManager translogManager() { engineConfig.getIndexSettings(), getMergedSegmentTransferTracker() ); - throttle = new IndexThrottle(); + throttle = new IndexingThrottler(); try { store.trimUnsafeCommits(engineConfig.getTranslogConfig().getTranslogPath()); final Map userData = store.readLastCommittedSegmentsInfo().getUserData(); @@ -330,7 +330,7 @@ public void onFailure(String reason, Exception ex) { // Set the Refresh checkpoint first and then sync child with parent to ensure parent Checkpoint is grater than Refresh // checkpoint. - this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker.getProcessedCheckpoint()); + this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker); this.internalReaderManager.addListener(lastRefreshedCheckpointListener); internalReaderManager.addListener(documentIndexWriter); maxSeqNoOfUpdatesOrDeletes = new AtomicLong( @@ -2505,14 +2505,14 @@ protected static Map commitDataAsMap(final DocumentIndexWriter i * Returned the last local checkpoint value has been refreshed internally. */ public final long lastRefreshedCheckpoint() { - return lastRefreshedCheckpointListener.refreshedCheckpoint.get(); + return lastRefreshedCheckpointListener.lastRefreshedCheckpoint(); } /** * Returns the current local checkpoint getting refreshed internally. */ public final long currentOngoingRefreshCheckpoint() { - return lastRefreshedCheckpointListener.pendingCheckpoint.get(); + return lastRefreshedCheckpointListener.pendingCheckpoint(); } private final Object refreshIfNeededMutex = new Object(); @@ -2530,38 +2530,6 @@ protected final void refreshIfNeeded(String source, long requestingSeqNo) { } } - private final class LastRefreshedCheckpointListener implements ReferenceManager.RefreshListener { - final AtomicLong refreshedCheckpoint; - volatile AtomicLong pendingCheckpoint; - - LastRefreshedCheckpointListener(long initialLocalCheckpoint) { - this.refreshedCheckpoint = new AtomicLong(initialLocalCheckpoint); - this.pendingCheckpoint = new AtomicLong(initialLocalCheckpoint); - } - - @Override - public void beforeRefresh() { - // all changes until this point should be visible after refresh - pendingCheckpoint.updateAndGet(curr -> Math.max(curr, localCheckpointTracker.getProcessedCheckpoint())); - } - - @Override - public void afterRefresh(boolean didRefresh) { - if (didRefresh) { - updateRefreshedCheckpoint(pendingCheckpoint.get()); - } - } - - void updateRefreshedCheckpoint(long checkpoint) { - refreshedCheckpoint.updateAndGet(curr -> Math.max(curr, checkpoint)); - assert refreshedCheckpoint.get() >= checkpoint : refreshedCheckpoint.get() + " < " + checkpoint; - // This shouldn't be required ideally, but we're also invoking this method from refresh as of now. - // This change is added as safety check to ensure that our checkpoint values are consistent at all times. - pendingCheckpoint.updateAndGet(curr -> Math.max(curr, checkpoint)); - - } - } - @Override public final long getMaxSeenAutoIdTimestamp() { return maxSeenAutoIdTimestamp.get(); diff --git a/server/src/main/java/org/opensearch/index/engine/LastRefreshedCheckpointListener.java b/server/src/main/java/org/opensearch/index/engine/LastRefreshedCheckpointListener.java new file mode 100644 index 0000000000000..d696748fe4aec --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/LastRefreshedCheckpointListener.java @@ -0,0 +1,48 @@ +package org.opensearch.index.engine; + +import org.apache.lucene.search.ReferenceManager; +import org.opensearch.index.seqno.LocalCheckpointTracker; + +import java.util.concurrent.atomic.AtomicLong; + +public class LastRefreshedCheckpointListener implements ReferenceManager.RefreshListener { + private final AtomicLong refreshedCheckpoint; + private final AtomicLong pendingCheckpoint; + private final LocalCheckpointTracker localCheckpointTracker; + + LastRefreshedCheckpointListener(LocalCheckpointTracker localCheckpointTracker) { + this.refreshedCheckpoint = new AtomicLong(localCheckpointTracker.getProcessedCheckpoint()); + this.pendingCheckpoint = new AtomicLong(localCheckpointTracker.getProcessedCheckpoint()); + this.localCheckpointTracker = localCheckpointTracker; + } + + @Override + public void beforeRefresh() { + // all changes until this point should be visible after refresh + pendingCheckpoint.updateAndGet(curr -> Math.max(curr, localCheckpointTracker.getProcessedCheckpoint())); + } + + @Override + public void afterRefresh(boolean didRefresh) { + if (didRefresh) { + updateRefreshedCheckpoint(pendingCheckpoint.get()); + } + } + + void updateRefreshedCheckpoint(long checkpoint) { + refreshedCheckpoint.updateAndGet(curr -> Math.max(curr, checkpoint)); + assert refreshedCheckpoint.get() >= checkpoint : refreshedCheckpoint.get() + " < " + checkpoint; + // This shouldn't be required ideally, but we're also invoking this method from refresh as of now. + // This change is added as safety check to ensure that our checkpoint values are consistent at all times. + pendingCheckpoint.updateAndGet(curr -> Math.max(curr, checkpoint)); + + } + + long lastRefreshedCheckpoint() { + return refreshedCheckpoint.get(); + } + + long pendingCheckpoint() { + return pendingCheckpoint.get(); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingConfig.java b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingConfig.java new file mode 100644 index 0000000000000..d62ded68ce7ff --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingConfig.java @@ -0,0 +1,12 @@ +package org.opensearch.index.engine.dataformat; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.dataformat.commit.Committer; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.store.Store; + +public record IndexingConfig(Store store, + IndexSettings indexSettings, + MapperService mapperService, + Committer committer) { +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java index d517649f35e87..f6df2c25b39bc 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/IndexingExecutionEngine.java @@ -10,6 +10,7 @@ import org.opensearch.common.annotation.ExperimentalApi; +import java.io.Closeable; import java.io.IOException; import java.util.Collection; import java.util.Map; @@ -24,7 +25,7 @@ * @opensearch.experimental */ @ExperimentalApi -public interface IndexingExecutionEngine> { +public interface IndexingExecutionEngine> extends Closeable { /** * Creates a new writer for the given writer generation. * diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java b/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java index c2df17af8a153..7892b6ab746ce 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java @@ -22,7 +22,7 @@ * @opensearch.experimental */ @ExperimentalApi -public record RefreshInput(List existingSegments, List writerFiles) { +public record RefreshInput(List existingSegments, List writerFiles) { public RefreshInput { existingSegments = List.copyOf(existingSegments); @@ -44,7 +44,7 @@ public static Builder builder() { @ExperimentalApi public static class Builder { private List existingSegments = new ArrayList<>(); - private List writerFiles = new ArrayList<>(); + private List segments = new ArrayList<>(); private Builder() {} @@ -65,8 +65,8 @@ public Builder existingSegments(List existingSegments) { * @param writerFileSet the writer file set to add * @return this builder */ - public Builder addWriterFileSet(WriterFileSet writerFileSet) { - this.writerFiles.add(writerFileSet); + public Builder addSegment(Segment segment) { + this.segments.add(segment); return this; } @@ -76,7 +76,7 @@ public Builder addWriterFileSet(WriterFileSet writerFileSet) { * @return the constructed RefreshInput */ public RefreshInput build() { - return new RefreshInput(existingSegments, writerFiles); + return new RefreshInput(existingSegments, segments); } } } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java b/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java index f3dbd3374d5b8..25e4894f77b54 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/Writer.java @@ -9,6 +9,7 @@ package org.opensearch.index.engine.dataformat; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.queue.Lockable; import java.io.Closeable; import java.io.IOException; @@ -21,7 +22,7 @@ * @opensearch.experimental */ @ExperimentalApi -public interface Writer

> extends Closeable { +public interface Writer

> extends Closeable, Lockable { /** * Adds a document to the writer. @@ -46,4 +47,10 @@ public interface Writer

> extends Closeable { * @throws IOException if an I/O error occurs */ void sync() throws IOException; + + /** + * The generation number associated with this writer + * @return the generation number + */ + long generation(); } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java new file mode 100644 index 0000000000000..ec913ea3e4f4f --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java @@ -0,0 +1,10 @@ +package org.opensearch.index.engine.dataformat.commit; + +import java.util.Map; + +public interface Committer { + + Map readLastCommittedUserData(); + + long getLastCommittedGeneration(); +} diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java new file mode 100644 index 0000000000000..d4a126ef0d092 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java @@ -0,0 +1,7 @@ +package org.opensearch.index.engine.dataformat.commit; + +import org.opensearch.index.store.Store; + +public interface CommitterFactory { + Committer getCommitter(Store store); +} diff --git a/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareEngineFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareEngineFactory.java index f32e6aa4410fd..1cf0d3df96fae 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareEngineFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareEngineFactory.java @@ -23,54 +23,10 @@ import java.util.Map; /** - * Factory that discovers {@link SearchBackEndPlugin}s via - * {@link PluginsService} and builds the per-format reader managers consumed by {@link DataFormatAwareEngine}. - *

- * This keeps DataformatAwareEngine decoupled from the plugin layer. + * Keeping this around this since this is needed for bwc IndexModule method * * @opensearch.experimental */ @ExperimentalApi public class DataFormatAwareEngineFactory { - - private final Map> readerManagers = new HashMap<>(); - private final IndexFileDeleter indexFileDeleter; - - @SuppressWarnings("rawtypes") - public DataFormatAwareEngineFactory( - PluginsService pluginsService, - ShardPath shardPath, - MapperService mapperService, - IndexSettings indexSettings - ) throws IOException { - for (SearchBackEndPlugin plugin : pluginsService.filterPlugins(SearchBackEndPlugin.class)) { - List formats = plugin.getSupportedFormats(); - if (formats == null) { - continue; - } - for (DataFormat format : formats) { - // TODO: use mapperService and indexSettings to filter formats relevant to this index - readerManagers.put(format, plugin.createReaderManager(format, shardPath)); - } - } - this.indexFileDeleter = new IndexFileDeleter(null, shardPath); - } - - /** - * Creates a new {@link DataFormatAwareEngine} populated with the discovered - * reader managers and memoizing suppliers. - */ - public DataFormatAwareEngine create() { - return new DataFormatAwareEngine(readerManagers); - } - - /** - * Creates a {@link CatalogSnapshotLifecycleListener} that routes events - * through the {@link IndexFileDeleter} and fans out to the given reader managers. - * - * @param readerManagers the per-format reader managers that receive notifications - */ - public CatalogSnapshotLifecycleListener createCatalogSnapshotListener(Map> readerManagers) { - return new DataFormatEngineCatalogSnapshotListener(readerManagers, indexFileDeleter); - } } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java new file mode 100644 index 0000000000000..6733eb5b65e50 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java @@ -0,0 +1,12 @@ +package org.opensearch.index.engine.exec; + +import org.opensearch.index.engine.DataFormatBasedEngine; +import org.opensearch.index.engine.EngineConfig; + +public class DataFormatAwareIndexerFactory implements IndexerFactory { + + @Override + public Indexer createIndexer(EngineConfig config) { + return new DataFormatBasedEngine(config); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/exec/EngineBackedIndexerFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/EngineBackedIndexerFactory.java new file mode 100644 index 0000000000000..87a8451d4ed95 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/exec/EngineBackedIndexerFactory.java @@ -0,0 +1,19 @@ +package org.opensearch.index.engine.exec; + +import org.opensearch.index.engine.EngineBackedIndexer; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.engine.EngineFactory; + +public class EngineBackedIndexerFactory implements IndexerFactory { + + private final EngineFactory engineFactory; + + public EngineBackedIndexerFactory(EngineFactory engineFactory) { + this.engineFactory = engineFactory; + } + + @Override + public Indexer createIndexer(EngineConfig engineConfig) { + return new EngineBackedIndexer(engineFactory.newReadWriteEngine(engineConfig)); + } +} diff --git a/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java new file mode 100644 index 0000000000000..1ffac94d25cdc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java @@ -0,0 +1,12 @@ +package org.opensearch.index.engine.exec; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.engine.EngineConfig; + +import java.util.function.Supplier; + +@ExperimentalApi +public interface IndexerFactory { + + Indexer createIndexer(EngineConfig config); +} diff --git a/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java b/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java index 9426bbeaad47b..b6316104ae369 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/coord/DataformatAwareCatalogSnapshot.java @@ -99,7 +99,7 @@ public long getId() { @Override public List getSegments() { - return segments; + return List.copyOf(segments); } @Override diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8080b3ef22e05..61fe489119faa 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -136,7 +136,6 @@ import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineException; -import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.engine.IngestionEngine; import org.opensearch.index.engine.MergedSegmentWarmerFactory; import org.opensearch.index.engine.NRTReplicationEngine; @@ -147,6 +146,7 @@ import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.exec.Indexer; +import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.fielddata.FieldDataStats; import org.opensearch.index.fielddata.ShardFieldData; import org.opensearch.index.flush.FlushStats; @@ -320,7 +320,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl private final Object engineMutex = new Object(); // lock ordering: engineMutex -> mutex private final AtomicReference currentEngineReference = new AtomicReference<>(); private final AtomicReference currentCompositeEngineReference = new AtomicReference<>(); - final EngineFactory engineFactory; + final IndexerFactory indexerFactory; final EngineConfigFactory engineConfigFactory; private final IndexingOperationListener indexingOperationListeners; @@ -422,7 +422,7 @@ public IndexShard( final IndexCache indexCache, final MapperService mapperService, final SimilarityService similarityService, - final EngineFactory engineFactory, + final IndexerFactory indexerFactory, final EngineConfigFactory engineConfigFactory, final IndexEventListener indexEventListener, final CheckedFunction indexReaderWrapper, @@ -461,7 +461,7 @@ public IndexShard( this.warmer = warmer; this.similarityService = similarityService; Objects.requireNonNull(store, "Store must be provided to the index shard"); - this.engineFactory = Objects.requireNonNull(engineFactory); + this.indexerFactory = Objects.requireNonNull(indexerFactory); this.engineConfigFactory = Objects.requireNonNull(engineConfigFactory); this.codecService = engineConfigFactory.newDefaultCodecService(indexSettings, mapperService, logger); this.store = store; @@ -3018,7 +3018,7 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier, b } // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata). // TODO: For composite engine, this would be replaced by a separate factory. - final Indexer newEngine = new EngineBackedIndexer(engineFactory.newReadWriteEngine(config)); + final Indexer newEngine = indexerFactory.createIndexer(config); onNewEngine(newEngine); currentEngineReference.set(newEngine); @@ -4474,7 +4474,9 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro () -> docMapper(), mergedSegmentWarmerFactory.get(this), clusterApplierService, - mergedSegmentTransferTracker + mergedSegmentTransferTracker, + dataFormatRegistry, + mapperService ); } @@ -5051,8 +5053,8 @@ public ShardFailure(ShardRouting routing, String reason, @Nullable Exception cau } } - EngineFactory getEngineFactory() { - return engineFactory; + IndexerFactory getIndexerFactory() { + return indexerFactory; } EngineConfigFactory getEngineConfigFactory() { @@ -5394,7 +5396,7 @@ public void close() throws IOException { if ((indexSettings.isRemoteTranslogStoreEnabled() || this.isRemoteSeeded()) && shardRouting.primary()) { syncRemoteTranslogAndUpdateGlobalCheckpoint(); } - newEngineReference.set(new EngineBackedIndexer(engineFactory.newReadWriteEngine(newEngineConfig(replicationTracker)))); + newEngineReference.set(indexerFactory.createIndexer(newEngineConfig(replicationTracker))); onNewEngine(newEngineReference.get()); } final TranslogRecoveryRunner translogRunner = (snapshot) -> { diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 28593ca80dc83..fd82d3edecc35 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -124,6 +124,9 @@ import org.opensearch.index.engine.NoOpEngine; import org.opensearch.index.engine.ReadOnlyEngine; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.exec.DataFormatAwareIndexerFactory; +import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; +import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.flush.FlushStats; import org.opensearch.index.get.GetStats; @@ -1094,7 +1097,7 @@ private synchronized IndexService createIndexService( final IndexModule indexModule = new IndexModule( idxSettings, analysisRegistry, - getEngineFactory(idxSettings), + getIndexerFactory(idxSettings), getEngineConfigFactory(idxSettings), directoryFactories, compositeDirectoryFactories, @@ -1164,6 +1167,14 @@ private IngestionConsumerFactory getIngestionConsumerFactory(final IndexSettings return null; } + private IndexerFactory getIndexerFactory(final IndexSettings idxSettings) { + if (idxSettings.isPluggableDataFormatenabled()) { + return new DataFormatAwareIndexerFactory(); + } else { + return new EngineBackedIndexerFactory(getEngineFactory(idxSettings)); + } + } + private EngineFactory getEngineFactory(final IndexSettings idxSettings) { final IndexMetadata indexMetadata = idxSettings.getIndexMetadata(); if (indexMetadata != null && indexMetadata.getState() == IndexMetadata.State.CLOSE) { @@ -1217,7 +1228,7 @@ public synchronized MapperService createIndexMapperService(IndexMetadata indexMe final IndexModule indexModule = new IndexModule( idxSettings, analysisRegistry, - getEngineFactory(idxSettings), + getIndexerFactory(idxSettings), getEngineConfigFactory(idxSettings), directoryFactories, compositeDirectoryFactories, diff --git a/server/src/main/java/org/opensearch/plugins/EnginePlugin.java b/server/src/main/java/org/opensearch/plugins/EnginePlugin.java index 16556014c4793..c06174f12ba1d 100644 --- a/server/src/main/java/org/opensearch/plugins/EnginePlugin.java +++ b/server/src/main/java/org/opensearch/plugins/EnginePlugin.java @@ -37,6 +37,8 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; +import org.opensearch.index.engine.dataformat.commit.Committer; +import org.opensearch.index.engine.dataformat.commit.CommitterFactory; import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.translog.TranslogDeletionPolicy; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; @@ -116,4 +118,8 @@ default Optional getAdditionalCodecs(IndexSettings indexSettin default Optional getCustomTranslogDeletionPolicyFactory() { return Optional.empty(); } + + default Optional getCommitterFactory() { + return Optional.empty(); + } } diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java index bcab71432dd1d..36a5cf0585095 100644 --- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java @@ -86,6 +86,7 @@ import org.opensearch.index.engine.InternalEngineFactory; import org.opensearch.index.engine.InternalEngineTests; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; import org.opensearch.index.fielddata.IndexFieldDataCache; import org.opensearch.index.mapper.ParsedDocument; import org.opensearch.index.mapper.Uid; @@ -122,6 +123,7 @@ import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.engine.MockEngineFactory; +import org.opensearch.test.engine.MockIndexerFactory; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transport; @@ -288,7 +290,7 @@ private IndexService newIndexService(IndexModule module) throws IOException { } public void testWrapperIsBound() throws IOException { - final MockEngineFactory engineFactory = new MockEngineFactory(AssertingDirectoryReader.class); + final MockIndexerFactory engineFactory = new MockIndexerFactory(AssertingDirectoryReader.class); IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, @@ -303,7 +305,7 @@ public void testWrapperIsBound() throws IOException { IndexService indexService = newIndexService(module); assertTrue(indexService.getReaderWrapper() instanceof Wrapper); - assertSame(indexService.getEngineFactory(), module.getEngineFactory()); + assertSame(indexService.getIndexerFactory(), module.getIndexerFactory()); indexService.close("simon says", false); } @@ -318,7 +320,7 @@ public void testRegisterIndexStore() throws IOException { final IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), indexStoreFactories, () -> true, @@ -646,7 +648,7 @@ public void testRegisterCustomRecoveryStateFactory() throws IOException { final IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), Collections.emptyMap(), () -> true, @@ -677,7 +679,7 @@ public void testStoreFactory() throws IOException { final IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), Collections.emptyMap(), Collections.emptyMap(), @@ -709,7 +711,7 @@ public void testStoreFactoryWithEmptySetting() throws IOException { final IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), Collections.emptyMap(), Collections.emptyMap(), @@ -739,7 +741,7 @@ public void testUnknownStoreFactory() { final IndexModule module = new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), Collections.emptyMap(), Collections.emptyMap(), @@ -770,7 +772,7 @@ private static IndexModule createIndexModule(IndexSettings indexSettings, Analys return new IndexModule( indexSettings, emptyAnalysisRegistry, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), new EngineConfigFactory(indexSettings), Collections.emptyMap(), () -> true, diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java index 8e2f57023cac0..abf086ea782d9 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java @@ -40,6 +40,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.InternalEngineFactory; +import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; import org.opensearch.index.seqno.ReplicationTracker; import org.opensearch.index.seqno.RetentionLease; import org.opensearch.index.seqno.RetentionLeaseStats; @@ -165,7 +166,7 @@ private void runExpirationTest(final boolean primary) throws IOException { .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), TimeValue.timeValueMillis(retentionLeaseMillis)) .build(); // current time is mocked through the thread pool - final IndexShard indexShard = newStartedShard(primary, settings, new InternalEngineFactory()); + final IndexShard indexShard = newStartedShard(primary, settings, new EngineBackedIndexerFactory(new InternalEngineFactory())); final long primaryTerm = indexShard.getOperationPrimaryTerm(); try { final long[] retainingSequenceNumbers = new long[1]; @@ -242,7 +243,7 @@ public void testPersistence() throws IOException { .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), Long.MAX_VALUE, TimeUnit.NANOSECONDS) .build(); - final IndexShard indexShard = newStartedShard(true, settings, new InternalEngineFactory()); + final IndexShard indexShard = newStartedShard(true, settings, new EngineBackedIndexerFactory(new InternalEngineFactory())); try { final int length = randomIntBetween(0, 8); final long[] minimumRetainingSequenceNumbers = new long[length]; diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index e06a40847c60e..5e2a9235a1b3a 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2701,7 +2701,7 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { newShardIndexMetadata, null, null, - shard.getEngineFactory(), + shard.getIndexerFactory(), shard.getEngineConfigFactory(), shard.getGlobalCheckpointSyncer(), shard.getRetentionLeaseSyncer(), diff --git a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java index 2a293132d4459..1b33e0e15a018 100644 --- a/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java +++ b/server/src/test/java/org/opensearch/index/shard/ReplicaRecoveryWithRemoteTranslogOnPrimaryTests.java @@ -77,7 +77,7 @@ public void testStartSequenceForReplicaRecovery() throws Exception { newIndexMetadata, null, null, - replica.getEngineFactory(), + replica.getIndexerFactory(), replica.getEngineConfigFactory(), replica.getGlobalCheckpointSyncer(), replica.getRetentionLeaseSyncer(), diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java index 9c717c796daae..ac3cde3b472b3 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java @@ -596,9 +596,9 @@ public void testGetEngineFactory() throws IOException { .build(); final IndexService indexService = indicesService.createIndex(indexMetadata, Collections.emptyList(), false); if (value != null && value) { - assertThat(indexService.getEngineFactory(), instanceOf(FooEnginePlugin.FooEngineFactory.class)); + assertThat(indexService.getIndexerFactory(), instanceOf(FooEnginePlugin.FooEngineFactory.class)); } else { - assertThat(indexService.getEngineFactory(), instanceOf(InternalEngineFactory.class)); + assertThat(indexService.getIndexerFactory(), instanceOf(InternalEngineFactory.class)); } } } diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index a761769b52adf..822f8d0538fde 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -102,7 +102,9 @@ import org.opensearch.index.engine.MergedSegmentWarmerFactory; import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; import org.opensearch.index.engine.exec.Indexer; +import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory; @@ -331,7 +333,7 @@ protected IndexShard newShard(boolean primary) throws IOException { * another shard) */ protected IndexShard newShard(final boolean primary, final Settings settings) throws IOException { - return newShard(primary, settings, new InternalEngineFactory()); + return newShard(primary, settings, new EngineBackedIndexerFactory(new InternalEngineFactory())); } /** @@ -340,9 +342,9 @@ protected IndexShard newShard(final boolean primary, final Settings settings) th * @param primary indicates whether to a primary shard (ready to recover from an empty store) or a replica (ready to recover from * another shard) * @param settings the settings to use for this shard - * @param engineFactory the engine factory to use for this shard + * @param indexerFactory the indexer factory to use for this shard */ - protected IndexShard newShard(boolean primary, Settings settings, EngineFactory engineFactory) throws IOException { + protected IndexShard newShard(boolean primary, Settings settings, IndexerFactory indexerFactory) throws IOException { final RecoverySource recoverySource = primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE; @@ -353,7 +355,7 @@ protected IndexShard newShard(boolean primary, Settings settings, EngineFactory ShardRoutingState.INITIALIZING, recoverySource ); - return newShard(shardRouting, settings, engineFactory); + return newShard(shardRouting, settings, indexerFactory); } protected IndexShard newShard(ShardRouting shardRouting, final IndexingOperationListener... listeners) throws IOException { @@ -362,7 +364,7 @@ protected IndexShard newShard(ShardRouting shardRouting, final IndexingOperation protected IndexShard newShard(ShardRouting shardRouting, final Settings settings, final IndexingOperationListener... listeners) throws IOException { - return newShard(shardRouting, settings, new InternalEngineFactory(), listeners); + return newShard(shardRouting, settings, new EngineBackedIndexerFactory(new InternalEngineFactory()), listeners); } /** @@ -370,13 +372,13 @@ protected IndexShard newShard(ShardRouting shardRouting, final Settings settings * * @param shardRouting the {@link ShardRouting} to use for this shard * @param settings the settings to use for this shard - * @param engineFactory the engine factory to use for this shard + * @param indexerFactory the indexer factory to use for this shard * @param listeners an optional set of listeners to add to the shard */ protected IndexShard newShard( final ShardRouting shardRouting, final Settings settings, - final EngineFactory engineFactory, + final IndexerFactory indexerFactory, final IndexingOperationListener... listeners ) throws IOException { assert shardRouting.initializing() : shardRouting; @@ -391,7 +393,7 @@ protected IndexShard newShard( .settings(indexSettings) .primaryTerm(0, primaryTerm) .putMapping("{ \"properties\": {} }"); - return newShard(shardRouting, metadata.build(), null, engineFactory, () -> {}, RetentionLeaseSyncer.EMPTY, null, listeners); + return newShard(shardRouting, metadata.build(), null, indexerFactory, () -> {}, RetentionLeaseSyncer.EMPTY, null, listeners); } /** @@ -410,7 +412,7 @@ protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperatio ShardRoutingState.INITIALIZING, primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE ); - return newShard(shardRouting, Settings.EMPTY, new InternalEngineFactory(), listeners); + return newShard(shardRouting, Settings.EMPTY, new EngineBackedIndexerFactory(new InternalEngineFactory()), listeners); } /** @@ -458,7 +460,7 @@ protected IndexShard newShard( shardRouting, indexMetadata, readerWrapper, - new InternalEngineFactory(), + new EngineBackedIndexerFactory(new InternalEngineFactory()), globalCheckpointSyncer, RetentionLeaseSyncer.EMPTY, null @@ -477,10 +479,10 @@ protected IndexShard newShard( ShardRouting routing, IndexMetadata indexMetadata, @Nullable CheckedFunction indexReaderWrapper, - EngineFactory engineFactory, + IndexerFactory indexerFactory, IndexingOperationListener... listeners ) throws IOException { - return newShard(routing, indexMetadata, indexReaderWrapper, engineFactory, () -> {}, RetentionLeaseSyncer.EMPTY, null, listeners); + return newShard(routing, indexMetadata, indexReaderWrapper, indexerFactory, () -> {}, RetentionLeaseSyncer.EMPTY, null, listeners); } /** @@ -496,7 +498,7 @@ protected IndexShard newShard( ShardRouting routing, IndexMetadata indexMetadata, @Nullable CheckedFunction indexReaderWrapper, - @Nullable EngineFactory engineFactory, + @Nullable IndexerFactory indexerFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, Path path, @@ -506,7 +508,7 @@ protected IndexShard newShard( routing, indexMetadata, indexReaderWrapper, - engineFactory, + indexerFactory, globalCheckpointSyncer, retentionLeaseSyncer, DefaultRecoverySettings.INSTANCE, @@ -532,7 +534,7 @@ protected IndexShard newShard( ShardRouting routing, IndexMetadata indexMetadata, @Nullable CheckedFunction indexReaderWrapper, - @Nullable EngineFactory engineFactory, + @Nullable IndexerFactory indexerFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, RecoverySettings recoverySettings, @@ -550,7 +552,7 @@ protected IndexShard newShard( indexMetadata, null, indexReaderWrapper, - engineFactory, + indexerFactory, new EngineConfigFactory(new IndexSettings(indexMetadata, indexMetadata.getSettings())), globalCheckpointSyncer, retentionLeaseSyncer, @@ -581,7 +583,7 @@ protected IndexShard newShard( IndexMetadata indexMetadata, @Nullable CheckedFunction storeProvider, @Nullable CheckedFunction indexReaderWrapper, - @Nullable EngineFactory engineFactory, + @Nullable IndexerFactory indexerFactory, @Nullable EngineConfigFactory engineConfigFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, @@ -595,7 +597,7 @@ protected IndexShard newShard( indexMetadata, storeProvider, indexReaderWrapper, - engineFactory, + indexerFactory, engineConfigFactory, globalCheckpointSyncer, retentionLeaseSyncer, @@ -650,7 +652,7 @@ protected IndexShard newShard(boolean primary, SegmentReplicationCheckpointPubli metadata, null, null, - new NRTReplicationEngineFactory(), + new EngineBackedIndexerFactory(new NRTReplicationEngineFactory()), new EngineConfigFactory(new IndexSettings(metadata, metadata.getSettings())), () -> {}, RetentionLeaseSyncer.EMPTY, @@ -683,7 +685,7 @@ protected IndexShard newShard( IndexMetadata indexMetadata, @Nullable CheckedFunction storeProvider, @Nullable CheckedFunction indexReaderWrapper, - @Nullable EngineFactory engineFactory, + @Nullable IndexerFactory indexerFactory, @Nullable EngineConfigFactory engineConfigFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, @@ -776,7 +778,7 @@ protected IndexShard newShard( indexCache, mapperService, similarityService, - engineFactory, + indexerFactory, engineConfigFactory, indexEventListener, indexReaderWrapper, @@ -953,7 +955,7 @@ protected IndexShard reinitShard(IndexShard current, ShardRouting routing, Path current, routing, current.indexSettings.getIndexMetadata(), - current.engineFactory, + current.indexerFactory, current.engineConfigFactory, remotePath, listeners @@ -966,13 +968,13 @@ protected IndexShard reinitShard(IndexShard current, ShardRouting routing, Path * @param routing the shard routing to use for the newly created shard. * @param listeners new listerns to use for the newly created shard * @param indexMetadata the index metadata to use for the newly created shard - * @param engineFactory the engine factory for the new shard + * @param indexerFactory the indexer factory for the new shard */ protected IndexShard reinitShard( IndexShard current, ShardRouting routing, IndexMetadata indexMetadata, - EngineFactory engineFactory, + IndexerFactory indexerFactory, EngineConfigFactory engineConfigFactory, Path remotePath, IndexingOperationListener... listeners @@ -984,7 +986,7 @@ protected IndexShard reinitShard( indexMetadata, null, null, - engineFactory, + indexerFactory, engineConfigFactory, current.getGlobalCheckpointSyncer(), current.getRetentionLeaseSyncer(), @@ -1006,7 +1008,7 @@ protected IndexShard newStartedShard() throws IOException { * @param settings the settings to use for this shard */ protected IndexShard newStartedShard(Settings settings) throws IOException { - return newStartedShard(randomBoolean(), settings, new InternalEngineFactory()); + return newStartedShard(randomBoolean(), settings, new EngineBackedIndexerFactory(new InternalEngineFactory())); } /** @@ -1015,7 +1017,7 @@ protected IndexShard newStartedShard(Settings settings) throws IOException { * @param primary controls whether the shard will be a primary or a replica. */ protected IndexShard newStartedShard(final boolean primary) throws IOException { - return newStartedShard(primary, Settings.EMPTY, new InternalEngineFactory()); + return newStartedShard(primary, Settings.EMPTY, new EngineBackedIndexerFactory(new InternalEngineFactory())); } /** @@ -1025,7 +1027,7 @@ protected IndexShard newStartedShard(final boolean primary) throws IOException { * @param settings the settings to use for this shard */ protected IndexShard newStartedShard(final boolean primary, Settings settings) throws IOException { - return newStartedShard(primary, settings, new InternalEngineFactory()); + return newStartedShard(primary, settings, new EngineBackedIndexerFactory(new InternalEngineFactory())); } /** @@ -1033,11 +1035,11 @@ protected IndexShard newStartedShard(final boolean primary, Settings settings) t * * @param primary controls whether the shard will be a primary or a replica. * @param settings the settings to use for this shard - * @param engineFactory the engine factory to use for this shard + * @param indexerFactory the indexer factory to use for this shard */ - protected IndexShard newStartedShard(final boolean primary, final Settings settings, final EngineFactory engineFactory) + protected IndexShard newStartedShard(final boolean primary, final Settings settings, final IndexerFactory indexerFactory) throws IOException { - return newStartedShard(p -> newShard(p, settings, engineFactory), primary); + return newStartedShard(p -> newShard(p, settings, indexerFactory), primary); } /** diff --git a/test/framework/src/main/java/org/opensearch/test/engine/MockIndexerFactory.java b/test/framework/src/main/java/org/opensearch/test/engine/MockIndexerFactory.java new file mode 100644 index 0000000000000..b33f1dfb59c32 --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/test/engine/MockIndexerFactory.java @@ -0,0 +1,21 @@ +package org.opensearch.test.engine; + +import org.apache.lucene.index.FilterDirectoryReader; +import org.opensearch.index.engine.EngineBackedIndexer; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.engine.exec.Indexer; +import org.opensearch.index.engine.exec.IndexerFactory; + +public class MockIndexerFactory implements IndexerFactory { + + private final Class wrapper; + + public MockIndexerFactory(Class wrapper) { + this.wrapper = wrapper; + } + + @Override + public Indexer createIndexer(EngineConfig config) { + return new EngineBackedIndexer(new MockInternalEngine(config, wrapper)); + } +} From ed4f0391e61f244c3b1b4c945fd7913af5097532 Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Fri, 10 Apr 2026 10:19:23 +0530 Subject: [PATCH 2/4] Wire reader Signed-off-by: Mohit Godwani --- .../be/datafusion/DataFusionPlugin.java | 19 +- .../analytics/exec/DefaultPlanExecutor.java | 8 +- .../exec/DefaultPlanExecutorTests.java | 12 +- .../composite/CompositeEnginePlugin.java | 29 +- .../CompositeIndexingExecutionEngine.java | 29 +- .../plugins/parquet-data-format/build.gradle | 6 + .../parquet/ParquetDataFormatPlugin.java | 3 +- .../parquet/engine/ParquetIndexingEngine.java | 17 +- .../index/engine/DataFormatAwareEngine.java | 1091 ++++++++++++++++- .../index/engine/DataFormatBasedEngine.java | 1068 ---------------- .../index/engine/EngineConfigFactory.java | 34 +- .../engine/dataformat/DataFormatPlugin.java | 2 +- .../engine/dataformat/DataFormatRegistry.java | 12 +- .../index/engine/dataformat/RefreshInput.java | 3 +- .../engine/dataformat/commit/Committer.java | 3 + .../dataformat/commit/CommitterFactory.java | 2 + .../exec/DataFormatAwareIndexerFactory.java | 5 +- .../engine/exec/EngineReaderManager.java | 3 +- .../index/engine/exec/IndexFileDeleter.java | 4 +- .../index/engine/exec/IndexerFactory.java | 1 + .../opensearch/index/shard/IndexShard.java | 27 +- .../org/opensearch/index/shard/ShardPath.java | 4 + .../dataformat/DataFormatPluginTests.java | 1 - 23 files changed, 1173 insertions(+), 1210 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index c406856e7fa47..3799e5b541846 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -23,6 +23,7 @@ import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.engine.dataformat.DataFormat; +import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.engine.exec.EngineReaderManager; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.Plugin; @@ -37,6 +38,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.function.Supplier; /** @@ -118,7 +120,22 @@ public EngineReaderManager createReaderManager(DataFormat form * Data formats this plugin can handle. Used by CompositeEngine to route queries. */ public List getSupportedFormats() { - return null; // TODO : List.of("parquet"); + return List.of(new DataFormat() { + @Override + public String name() { + return "parquet"; + } + + @Override + public long priority() { + return 0; + } + + @Override + public Set supportedFields() { + return Set.of(); + } + }); } @Override diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java index a8fb9982352b8..c22cc17c698a6 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java @@ -20,7 +20,7 @@ import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; import org.opensearch.cluster.service.ClusterService; import org.opensearch.index.IndexService; -import org.opensearch.index.engine.DataFormatAwareEngine; +import org.opensearch.index.engine.exec.IndexReaderProvider; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; @@ -69,14 +69,14 @@ public Iterable execute(RelNode logicalFragment, Object context) { } IndexShard shard = resolveShard(tableName); - DataFormatAwareEngine dataFormatAwareEngine = shard.getCompositeEngine(); - if (dataFormatAwareEngine == null) { + IndexReaderProvider indexReaderProvider = shard.getReaderProvider(); + if (indexReaderProvider == null) { throw new IllegalStateException("No CompositeEngine on shard [" + shard.shardId() + "]"); } SearchShardTask task = null; // TODO: init task List rows = new ArrayList<>(); - try (var dataFormatAwareReader = dataFormatAwareEngine.acquireReader()) { + try (var dataFormatAwareReader = indexReaderProvider.acquireReader()) { ExecutionContext ctx = new ExecutionContext(tableName, task, dataFormatAwareReader.get()); try (SearchExecEngine engine = provider.createSearchExecEngine(ctx)) { logger.info("[DefaultPlanExecutor] Executing via [{}]", provider.name()); diff --git a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java index 14d9dd2a17ed6..5b413d021c5a0 100644 --- a/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java +++ b/sandbox/plugins/analytics-engine/src/test/java/org/opensearch/analytics/exec/DefaultPlanExecutorTests.java @@ -34,6 +34,9 @@ import org.opensearch.core.index.Index; import org.opensearch.index.IndexService; import org.opensearch.index.engine.DataFormatAwareEngine; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.engine.EngineConfigFactory; +import org.opensearch.index.engine.EngineTestCase; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.engine.exec.EngineReaderManager; @@ -114,11 +117,11 @@ public void testEndToEndExecuteWithMockBackend() throws IOException { readerManager.afterRefresh(true, ref.get()); } - DataFormatAwareEngine engine = new DataFormatAwareEngine(Map.of(format, readerManager), snapshotManager); + DataFormatAwareEngine engine = new DataFormatAwareEngine(null); // Mock shard + cluster wiring IndexShard shard = mock(IndexShard.class); - when(shard.getCompositeEngine()).thenReturn(engine); + when(shard.getReaderProvider()).thenReturn(engine); Index index = new Index("my_index", "uuid"); IndexMetadata indexMetadata = mock(IndexMetadata.class); @@ -239,6 +242,11 @@ public void onFilesDeleted(Collection files) {} @Override public void onFilesAdded(Collection files) {} + + @Override + public void close() throws IOException { + readers.clear(); + } } static class MockCatalogSnapshot extends CatalogSnapshot { diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeEnginePlugin.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeEnginePlugin.java index 182171c499ff0..8a9dc03640e4b 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeEnginePlugin.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeEnginePlugin.java @@ -10,21 +10,31 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatPlugin; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardPath; import org.opensearch.plugins.ExtensiblePlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; +import org.opensearch.watcher.ResourceWatcherService; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.function.Supplier; /** * Sandbox plugin that provides a {@link CompositeIndexingExecutionEngine} for @@ -61,6 +71,11 @@ public class CompositeEnginePlugin extends Plugin implements ExtensiblePlugin, D Setting.Property.Final ); + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier) { + return super.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptService, xContentRegistry, environment, nodeEnvironment, namedWriteableRegistry, indexNameExpressionResolver, repositoriesServiceSupplier); + } + /** * Index setting that lists the secondary data formats for an index. * Secondary formats receive writes alongside the primary but are not used @@ -133,12 +148,12 @@ public List> getSettings() { @Override public DataFormat getDataFormat() { // TODO: Dataformat for Composite is per index, while this one talks about cluster level. Switching it off for now - return null; + return new CompositeDataFormat(); } @Override - public IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings) { - return new CompositeIndexingExecutionEngine(dataFormatPlugins, indexSettings, mapperService, shardPath); + public IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { + return new CompositeIndexingExecutionEngine(indexSettings, mapperService, dataFormatRegistry, shardPath); } /** diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index b35234760a56d..b68d39ba28327 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -15,15 +15,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.dataformat.DataFormat; -import org.opensearch.index.engine.dataformat.DataFormatPlugin; -import org.opensearch.index.engine.dataformat.DocumentInput; -import org.opensearch.index.engine.dataformat.FileInfos; -import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; -import org.opensearch.index.engine.dataformat.Merger; -import org.opensearch.index.engine.dataformat.RefreshInput; -import org.opensearch.index.engine.dataformat.RefreshResult; -import org.opensearch.index.engine.dataformat.Writer; +import org.opensearch.index.engine.dataformat.*; import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.mapper.MapperService; @@ -75,19 +67,17 @@ public class CompositeIndexingExecutionEngine implements IndexingExecutionEngine * The writer pool is created internally and initialized with a writer supplier * that creates {@link CompositeWriter} instances bound to this engine. * - * @param dataFormatPlugins the discovered data format plugins keyed by format name * @param indexSettings the index settings containing composite configuration * @param mapperService the mapper service for field mapping resolution * @param shardPath the shard path for file storage * @throws IllegalArgumentException if any configured format is not registered */ public CompositeIndexingExecutionEngine( - Map dataFormatPlugins, IndexSettings indexSettings, MapperService mapperService, + DataFormatRegistry dataFormatRegistry, ShardPath shardPath ) { - Objects.requireNonNull(dataFormatPlugins, "dataFormatPlugins must not be null"); Objects.requireNonNull(indexSettings, "indexSettings must not be null"); Settings settings = indexSettings.getSettings(); @@ -95,18 +85,15 @@ public CompositeIndexingExecutionEngine( String primaryFormatName = CompositeEnginePlugin.PRIMARY_DATA_FORMAT.get(settings); List secondaryFormatNames = CompositeEnginePlugin.SECONDARY_DATA_FORMATS.get(settings); - validateFormatsRegistered(dataFormatPlugins, primaryFormatName, secondaryFormatNames); - List allFormats = new ArrayList<>(); - DataFormatPlugin primaryPlugin = dataFormatPlugins.get(primaryFormatName); - this.primaryEngine = primaryPlugin.indexingEngine(mapperService, shardPath, indexSettings); - allFormats.add(primaryPlugin.getDataFormat()); + this.primaryEngine = dataFormatRegistry.getIndexingEngine(dataFormatRegistry.format(primaryFormatName), mapperService, shardPath, indexSettings); + allFormats.add(dataFormatRegistry.format(primaryFormatName)); List> secondaries = new ArrayList<>(); for (String secondaryName : secondaryFormatNames) { - DataFormatPlugin secondaryPlugin = dataFormatPlugins.get(secondaryName); - secondaries.add(secondaryPlugin.indexingEngine(mapperService, shardPath, indexSettings)); - allFormats.add(secondaryPlugin.getDataFormat()); + DataFormat secondaryFormat = dataFormatRegistry.format(secondaryName); + secondaries.add(dataFormatRegistry.getIndexingEngine(secondaryFormat, mapperService, shardPath, indexSettings)); + allFormats.add(secondaryFormat); } this.secondaryEngines = Set.copyOf(secondaries); @@ -174,7 +161,7 @@ public RefreshResult refresh(RefreshInput refreshInput) throws IOException { for (IndexingExecutionEngine engine : secondaryEngines) { secResults.add(engine.refresh(refreshInput)); } - return null; + return primary; } @Override diff --git a/sandbox/plugins/parquet-data-format/build.gradle b/sandbox/plugins/parquet-data-format/build.gradle index f00bc5c04af09..2ce75dc9d7d30 100644 --- a/sandbox/plugins/parquet-data-format/build.gradle +++ b/sandbox/plugins/parquet-data-format/build.gradle @@ -30,6 +30,12 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson_annotations}" + + // SLF4J logging implementation (required by Apache Arrow) + implementation 'org.slf4j:slf4j-api:2.0.17' + + // Bridge for slf4j<-->log4j compatibility + implementation "org.apache.logging.log4j:log4j-slf4j2-impl:2.23.1" } tasks.named("dependencyLicenses").configure { diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java index 504722af216d1..b8bc1d5318f78 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/ParquetDataFormatPlugin.java @@ -20,6 +20,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatPlugin; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.IndexingExecutionEngine; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardPath; @@ -88,7 +89,7 @@ public DataFormat getDataFormat() { } @Override - public IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings) { + public IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry) { return new ParquetIndexingEngine( settings, dataFormat, diff --git a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java index 60713a2d05b5e..4f4c51dec4dde 100644 --- a/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java +++ b/sandbox/plugins/parquet-data-format/src/main/java/org/opensearch/parquet/engine/ParquetIndexingEngine.java @@ -19,7 +19,6 @@ import org.opensearch.index.engine.dataformat.RefreshResult; import org.opensearch.index.engine.dataformat.Writer; import org.opensearch.index.engine.exec.Segment; -import org.opensearch.index.engine.exec.WriterFileSet; import org.opensearch.index.shard.ShardPath; import org.opensearch.parquet.bridge.RustBridge; import org.opensearch.parquet.memory.ArrowBufferPool; @@ -27,7 +26,6 @@ import org.opensearch.parquet.writer.ParquetWriter; import org.opensearch.threadpool.ThreadPool; -import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -92,6 +90,11 @@ public ParquetIndexingEngine( this.bufferPool = new ArrowBufferPool(settings); this.settings = settings; this.threadPool = threadPool; + try { + Files.createDirectory(shardPath.resolve("parquet")); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override @@ -119,12 +122,10 @@ public RefreshResult refresh(RefreshInput refreshInput) throws IOException { if (refreshInput == null) { return new RefreshResult(List.of()); } - List segments = new ArrayList<>(refreshInput.existingSegments()); - long gen = segments.stream().mapToLong(Segment::generation).max().orElse(-1) + 1; - for (WriterFileSet wfs : refreshInput.writerFiles()) { - segments.add(Segment.builder(gen++).addSearchableFiles(dataFormat, wfs).build()); - } - return new RefreshResult(segments); + List segments = new ArrayList<>(); + segments.addAll(refreshInput.existingSegments()); + segments.addAll(refreshInput.writerFiles()); + return new RefreshResult(List.copyOf(segments)); } @Override diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java index 8ba14b5fcacf4..ecf28ac054893 100644 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/DataFormatAwareEngine.java @@ -8,59 +8,934 @@ package org.opensearch.index.engine; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.AlreadyClosedException; +import org.opensearch.common.Nullable; +import org.opensearch.common.SetOnce; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.concurrent.GatedCloseable; -import org.opensearch.index.engine.dataformat.DataFormat; -import org.opensearch.index.engine.exec.DataFormatAwareEngineFactory; -import org.opensearch.index.engine.exec.EngineReaderManager; -import org.opensearch.index.engine.exec.IndexReaderProvider; +import org.opensearch.common.lease.Releasable; +import org.opensearch.common.logging.Loggers; +import org.opensearch.common.queue.LockablePool; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ReleasableLock; +import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.index.AppendOnlyIndexOperationRetryException; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.index.VersionType; +import org.opensearch.index.engine.dataformat.*; +import org.opensearch.index.engine.dataformat.commit.Committer; +import org.opensearch.index.engine.exec.*; +import org.opensearch.index.engine.exec.Segment; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.index.engine.exec.coord.CatalogSnapshotManager; +import org.opensearch.index.mapper.*; +import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.seqno.LocalCheckpointTracker; +import org.opensearch.index.seqno.SeqNoStats; +import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.shard.DocsStats; +import org.opensearch.index.store.Store; +import org.opensearch.index.translog.DefaultTranslogDeletionPolicy; +import org.opensearch.index.translog.InternalTranslogManager; +import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.TranslogCorruptedException; +import org.opensearch.index.translog.TranslogDeletionPolicy; +import org.opensearch.index.translog.TranslogException; +import org.opensearch.index.translog.TranslogManager; +import org.opensearch.index.translog.TranslogOperationHelper; +import org.opensearch.index.translog.listener.TranslogEventListener; +import org.opensearch.indices.pollingingest.PollingIngestStats; +import org.opensearch.search.suggest.completion.CompletionStats; import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiFunction; + +import org.apache.lucene.index.Term; + +import static org.opensearch.index.engine.Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID; /** - * Owns all reader managers, lazily creates search engines, index filter providers - * and source providers per data format. + * An {@link Indexer} implementation that delegates to an {@link IndexingExecutionEngine}. + * This engine manages the full lifecycle of indexing, + * deleting, and searching documents on a single shard by coordinating between the + * underlying data format engine and the translog. *

- * Instances are created by {@link DataFormatAwareEngineFactory}. + * Mirrors the responsibilities of {@link InternalEngine} but is decoupled from Lucene: + *

    + *
  • Sequence number generation and local checkpoint tracking
    • + *
  • Translog management for durability and recovery
    • + *
  • Refresh to make documents searchable via catalog snapshots
    • + *
  • Flush to commit catalog snapshots and trim the translog
    • + *
  • Throttling when merges fall behind or indexing buffer is full
    • + *
  • Soft-deletes policy for peer-recovery retention
    • + *
* * @opensearch.experimental */ @ExperimentalApi -public class DataFormatAwareEngine implements IndexReaderProvider, Closeable { +public class DataFormatAwareEngine implements Indexer { + + private final Logger logger; + private final EngineConfig engineConfig; + private final ShardId shardId; + private final Store store; + + private final IndexingExecutionEngine indexingExecutionEngine; + private final IndexingStrategyPlanner indexingStrategyPlanner; + private final LockablePool> writerPool; + private final AtomicLong writerGenerationCounter; private final Map> readerManagers; - private volatile CatalogSnapshotManager catalogSnapshotManager; + private final CatalogSnapshotManager catalogSnapshotManager; + private final Committer committer; + + // Translog for durability and recovery + private final TranslogManager translogManager; + + // Sequence number tracking + private final LocalCheckpointTracker localCheckpointTracker; + + // Throttling + private final IndexingThrottler throttle; + private final AtomicInteger throttleRequestCount = new AtomicInteger(); + + // Timestamps and seq-no markers + private final AtomicLong maxUnsafeAutoIdTimestamp = new AtomicLong(-1); + private final AtomicLong maxSeenAutoIdTimestamp = new AtomicLong(-1); + private volatile long lastWriteNanos = System.nanoTime(); + + // Lifecycle + private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final SetOnce failedEngine = new SetOnce<>(); + private final CountDownLatch closedLatch = new CountDownLatch(1); + + // Concurrency locks + private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); + private final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); + private final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); + private final Lock flushLock = new ReentrantLock(); + private final ReentrantLock failEngineLock = new ReentrantLock(); + + + // Refresh tracker + private final LastRefreshedCheckpointListener lastRefreshedCheckpointListener; + + @Nullable + private final String historyUUID; /** - * Constructs a new DataFormatAwareEngine. - * Prefer using {@link DataFormatAwareEngineFactory#create()}. + * Constructs a DataFormatBasedEngine. + * + * @param engineConfig the engine configuration */ - public DataFormatAwareEngine(Map> readerManagers, CatalogSnapshotManager catalogSnapshotManager) { - this.readerManagers = readerManagers; - this.catalogSnapshotManager = catalogSnapshotManager; + public DataFormatAwareEngine(EngineConfig engineConfig) { + this.logger = Loggers.getLogger(DataFormatAwareEngine.class, engineConfig.getShardId()); + this.engineConfig = engineConfig; + this.shardId = engineConfig.getShardId(); + this.store = engineConfig.getStore(); + this.throttle = new IndexingThrottler(); + + if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { + updateAutoIdTimestamp(Long.MAX_VALUE, true); + } + + boolean success = false; + TranslogManager translogManagerRef = null; + + try { + store.incRef(); + + this.committer = engineConfig.getCommitterFactory().getCommitter(store); + this.catalogSnapshotManager = new CatalogSnapshotManager(0, 0, 0, List.of(), -1, Map.of()); + + // Read history UUID and translog UUID from last commit + final Map userData = committer.readLastCommittedUserData(); + String translogUUID = Objects.requireNonNull(userData.get(Translog.TRANSLOG_UUID_KEY)); + + // Initialize translog + // TODO: Once file deleter is merged, we will add relevant listeners + final TranslogEventListener translogEventListener = createInternalTranslogEventListener(); + translogManagerRef = createTranslogManager(translogUUID, translogEventListener); + this.translogManager = translogManagerRef; + + // Initialize local checkpoint tracker from last committed segment infos + this.localCheckpointTracker = createLocalCheckpointTracker(LocalCheckpointTracker::new); + + // Initialize from commit data + this.historyUUID = userData.get(Engine.HISTORY_UUID_KEY); + updateAutoIdTimestamp(Long.parseLong(userData.get(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID)), true); + + + // Move to data format aware writers and readers. + DataFormatRegistry registry = engineConfig.getDataFormatRegistry(); + // Create indexing engine + // Pass committer here as well. + this.indexingExecutionEngine = registry.getIndexingEngine(registry.format(engineConfig.getIndexSettings().pluggableDataFormat()), + engineConfig.getMapperService(), store.shardPath(), engineConfig.getIndexSettings()); + this.writerGenerationCounter = new AtomicLong(committer.getLastCommittedGeneration()); + this.writerPool = new LockablePool<>( + () -> indexingExecutionEngine.createWriter(writerGenerationCounter.getAndIncrement()), + LinkedList::new, + Runtime.getRuntime().availableProcessors() + ); + // Create Reader managers + // We will pass IndexViewProvider to this, which would contain store + // and any index specific attributes useful for reads. + this.readerManagers = registry.getReaderManagers(engineConfig.getMapperService(), + engineConfig.getIndexSettings(), + store.shardPath()); + + this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker); + this.indexingStrategyPlanner = new IndexingStrategyPlanner( + engineConfig.getIndexSettings(), + engineConfig.getShardId(), + new LiveVersionMap(), + maxUnsafeAutoIdTimestamp::get, + () -> 0L, + localCheckpointTracker::getProcessedCheckpoint, + this::hasBeenProcessedBefore, + op -> OpVsEngineDocStatus.OP_NEWER, + (a, b) -> null, + this::updateAutoIdTimestamp, + (a, b) -> null + ); + success = true; + logger.trace("created new DataFormatBasedEngine"); + } catch (IOException | TranslogCorruptedException e) { + throw new EngineCreationFailureException(shardId, "failed to create engine", e); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(translogManagerRef); + if (isClosed.get() == false) { + store.decRef(); + } + } + } } - /** - * Constructs a new DataFormatAwareEngine without a snapshot manager. - * The manager must be set via {@link #setCatalogSnapshotManager} before acquiring readers. - */ - public DataFormatAwareEngine(Map> readerManagers) { - this.readerManagers = readerManagers; + private LocalCheckpointTracker createLocalCheckpointTracker( + BiFunction supplier + ) throws IOException { + final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit( + store.readLastCommittedSegmentsInfo().getUserData().entrySet() + ); + logger.trace("recovered max_seq_no [{}] and local_checkpoint [{}]", seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); + return supplier.apply(seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); + } + + private TranslogEventListener createInternalTranslogEventListener() { + return new TranslogEventListener() { + @Override + public void onAfterTranslogSync() { + try { + // TODO: Handle file deletion policy + translogManager.trimUnreferencedReaders(); + } catch (IOException ex) { + throw new TranslogException(shardId, "failed to trim translog on sync", ex); + } + } + + @Override + public void onAfterTranslogRecovery() { + flush(false, true); + translogManager.trimUnreferencedTranslogFiles(); + } + + @Override + public void onFailure(String reason, Exception ex) { + if (ex instanceof AlreadyClosedException) { + failOnTragicEvent((AlreadyClosedException) ex); + } else { + failEngine(reason, ex); + } + } + }; + } + + private TranslogManager createTranslogManager( + String translogUUID, + TranslogEventListener translogEventListener + ) throws IOException { + TranslogDeletionPolicy deletionPolicy = getTranslogDeletionPolicy(); + return new InternalTranslogManager( + engineConfig.getTranslogConfig(), + engineConfig.getPrimaryTermSupplier(), + engineConfig.getGlobalCheckpointSupplier(), + deletionPolicy, + shardId, + readLock, + () -> localCheckpointTracker, + translogUUID, + translogEventListener, + this::ensureOpen, + engineConfig.getTranslogFactory(), + engineConfig.getStartedPrimarySupplier(), + TranslogOperationHelper.create(engineConfig) + ); + } + + private TranslogDeletionPolicy getTranslogDeletionPolicy() { + TranslogDeletionPolicy custom = null; + if (engineConfig.getCustomTranslogDeletionPolicyFactory() != null) { + custom = engineConfig.getCustomTranslogDeletionPolicyFactory() + .create(engineConfig.getIndexSettings(), engineConfig.retentionLeasesSupplier()); + } + return Objects.requireNonNullElseGet(custom, () -> new DefaultTranslogDeletionPolicy( + engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(), + engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis(), + engineConfig.getIndexSettings().getTranslogRetentionTotalFiles() + )); + } + + @Override + public Engine.IndexResult index(Engine.Index index) throws IOException { + assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field(); + final boolean doThrottle = index.origin().isRecovery() == false; + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + try (Releasable indexThrottle = doThrottle ? throttle.acquireThrottle() : () -> {}) { + lastWriteNanos = index.startTime(); + final IndexingStrategy plan = indexingStrategyPlanner.planOperationAsPrimary(index); + final Engine.IndexResult indexResult; + if (plan.earlyResultOnPreFlightError.isPresent()) { + assert index.origin() == Engine.Operation.Origin.PRIMARY : index.origin(); + indexResult = (Engine.IndexResult) plan.earlyResultOnPreFlightError.get(); + assert indexResult.getResultType() == Engine.Result.Type.FAILURE : indexResult.getResultType(); + } else { + if (index.origin() == Engine.Operation.Origin.PRIMARY) { + index = new Engine.Index( + index.uid(), + index.parsedDoc(), + generateSeqNoForOperationOnPrimary(index), + index.primaryTerm(), + index.version(), + index.versionType(), + index.origin(), + index.startTime(), + index.getAutoGeneratedIdTimestamp(), + index.isRetry(), + index.getIfSeqNo(), + index.getIfPrimaryTerm() + ); + } else { + markSeqNoAsSeen(index.seqNo()); + } + + assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); + + if (plan.executeOpOnEngine) { + logger.debug("Indexing doc id=[{}] seqNo=[{}] primaryTerm=[{}] — writing to engine", + index.id(), index.seqNo(), index.primaryTerm()); + indexResult = indexIntoEngine(index); + } else { + indexResult = + new Engine.IndexResult(plan.version, index.primaryTerm(), index.seqNo(), plan.currentNotFoundOrDeleted); + } + } + return indexResult; + } + } catch (RuntimeException | IOException e) { + maybeFailEngine("index id[" + index.id() + "] origin[" + index.origin() + "]", e); + throw e; + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOException { + Engine.IndexResult indexResult; + if (index.origin() == Engine.Operation.Origin.PRIMARY) { + // Assign a sequence number on the primary + final long seqNo = localCheckpointTracker.generateSeqNo(); + index = new Engine.Index( + index.uid(), index.parsedDoc(), seqNo, index.primaryTerm(), + index.version(), index.versionType(), index.origin(), index.startTime(), + index.getAutoGeneratedIdTimestamp(), index.isRetry(), + index.getIfSeqNo(), index.getIfPrimaryTerm() + ); + } else { + // On replica, advance max seq no + localCheckpointTracker.advanceMaxSeqNo(index.seqNo()); + } + + assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); + + // Convert ParsedDocument to DocumentInput and write via the execution engine's writer + Writer currentWriter = null; + try { + currentWriter = writerPool.getAndLock(); + + //Replace with index#documentInput + DocumentInput docInput = indexingExecutionEngine.newDocumentInput(); + + WriteResult result = currentWriter.addDoc(docInput); + + if (result instanceof WriteResult.Success(long version, long term, long seqNo)) { + indexResult = new Engine.IndexResult(version, term, seqNo, true); + } else { + WriteResult.Failure f = (WriteResult.Failure) result; + indexResult = new Engine.IndexResult(f.cause(), index.version(), index.primaryTerm(), index.seqNo()); + } + } catch (Exception e) { + indexResult = new Engine.IndexResult(e, index.version(), index.primaryTerm(), index.seqNo()); + } finally { + if (currentWriter != null) { + writerPool.releaseAndUnlock(currentWriter); + } + } + + logger.info(indexResult.getFailure()); + if (index.origin().isFromTranslog() == false) { + final Translog.Location location; + if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) { + location = translogManager.add(new Translog.Index(index, indexResult)); + } else if (indexResult.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && indexResult.getFailure() != null + && !(indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException)) { + throw new UnsupportedOperationException("recording document failure as a no-op in translog is not " + + "supported for Data format engine"); + } else { + location = null; + } + indexResult.setTranslogLocation(location); + } + + // Track the sequence number + localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); + if (indexResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); + } + + localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); + if (indexResult.getTranslogLocation() == null && !(indexResult.getFailure() != null + && (indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException))) { + // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number + assert index.origin().isFromTranslog() || indexResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO; + localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); + } + indexResult.setTook(System.nanoTime() - index.startTime()); + indexResult.freeze(); + return indexResult; + } + + @Override + public Engine.DeleteResult delete(Engine.Delete delete) throws IOException { + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + lastWriteNanos = delete.startTime(); + + final long seqNo; + if (delete.origin() == Engine.Operation.Origin.PRIMARY) { + seqNo = localCheckpointTracker.generateSeqNo(); + } else { + seqNo = delete.seqNo(); + localCheckpointTracker.advanceMaxSeqNo(seqNo); + } + + // TODO: Implement actual delete via IndexingExecutionEngine when supported. + // For now, record the delete in the translog and track the seq no. + final Engine.DeleteResult deleteResult = new Engine.DeleteResult(1L, delete.primaryTerm(), seqNo, true); + + if (delete.origin().isFromTranslog() == false) { + final Translog.Location location = translogManager.add(new Translog.Delete(delete, deleteResult)); + deleteResult.setTranslogLocation(location); + } + + localCheckpointTracker.markSeqNoAsProcessed(seqNo); + if (deleteResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(seqNo); + } + + deleteResult.setTook(System.nanoTime() - delete.startTime()); + deleteResult.freeze(); + return deleteResult; + } catch (RuntimeException | IOException e) { + maybeFailEngine("delete id[" + delete.id() + "] origin[" + delete.origin() + "]", e); + throw e; + } + } + + @Override + public Engine.NoOpResult noOp(Engine.NoOp noOp) throws IOException { + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + final Engine.NoOpResult noOpResult = new Engine.NoOpResult(noOp.primaryTerm(), noOp.seqNo()); + + if (noOp.origin().isFromTranslog() == false) { + final Translog.Location location = translogManager.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); + noOpResult.setTranslogLocation(location); + } + + localCheckpointTracker.markSeqNoAsProcessed(noOp.seqNo()); + if (noOpResult.getTranslogLocation() == null) { + localCheckpointTracker.markSeqNoAsPersisted(noOp.seqNo()); + } + + noOpResult.setTook(System.nanoTime() - noOp.startTime()); + noOpResult.freeze(); + return noOpResult; + } catch (RuntimeException | IOException e) { + maybeFailEngine("noOp seq#[" + noOp.seqNo() + "]", e); + throw e; + } + } + + @Override + public Engine.Index prepareIndex( + DocumentMapperForType docMapper, SourceToParse source, + long seqNo, long primaryTerm, long version, VersionType versionType, + Engine.Operation.Origin origin, long autoGeneratedIdTimestamp, + boolean isRetry, long ifSeqNo, long ifPrimaryTerm + ) { + long startTime = System.nanoTime(); + ParsedDocument doc = docMapper.getDocumentMapper().parse(source); + if (docMapper.getMapping() != null) { + doc.addDynamicMappingsUpdate(docMapper.getMapping()); + } + Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id())); + return new Engine.Index( + uid, doc, seqNo, primaryTerm, version, versionType, + origin, startTime, autoGeneratedIdTimestamp, isRetry, ifSeqNo, ifPrimaryTerm + ); + } + + @Override + public Engine.Delete prepareDelete( + String id, long seqNo, long primaryTerm, long version, + VersionType versionType, Engine.Operation.Origin origin, + long ifSeqNo, long ifPrimaryTerm + ) { + long startTime = System.nanoTime(); + Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); + return new Engine.Delete(id, uid, seqNo, primaryTerm, version, versionType, origin, startTime, ifSeqNo, ifPrimaryTerm); + } + + @Override + public void refresh(String source) throws EngineException { + final long localCheckpointBeforeRefresh = localCheckpointTracker.getProcessedCheckpoint(); + boolean refreshed = false; + try (GatedCloseable catalogSnapshot = catalogSnapshotManager.acquireSnapshot()) { + if (store.tryIncRef()) { + try { + List> writers = writerPool.checkoutAll(); + List existingSegments = catalogSnapshot.get().getSegments(); + List newSegments = new ArrayList<>(); + List fileInfosList = new ArrayList<>(); + + for (Writer writer : writers) { + FileInfos fileInfos = writer.flush(); + Segment.Builder segmentBuilder = Segment.builder(writer.generation()); + boolean hasFiles = false; + for (Map.Entry entry : fileInfos.writerFilesMap().entrySet()) { + logger.debug( + "Writer gen={} flushed format=[{}] files={}", + writer.generation(), + entry.getKey().name(), + entry.getValue().files() + ); + segmentBuilder.addSearchableFiles(entry.getKey(), entry.getValue()); + hasFiles = true; + } + writer.close(); + if (hasFiles) { + newSegments.add(segmentBuilder.build()); + } + refreshed |= hasFiles; + } + logger.debug("Produced {} new segments from flush", newSegments.size()); + + RefreshInput refreshInput = new RefreshInput(existingSegments, newSegments); + RefreshResult result = indexingExecutionEngine.refresh(refreshInput); + catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); + refreshed = true; + } finally { + store.decRef(); + } + if (refreshed) { + lastRefreshedCheckpointListener.updateRefreshedCheckpoint(localCheckpointBeforeRefresh); + } + } + } catch (AlreadyClosedException ex) { + failOnTragicEvent(ex); + throw ex; + } catch (Exception ex) { + try { + failEngine("refresh failed source[" + source + "]", ex); + } catch (Exception inner) { + ex.addSuppressed(inner); + } + throw new RefreshFailedEngineException(shardId, ex); + } + } + + @Override + public void flush(boolean force, boolean waitIfOngoing) throws EngineException { + ensureOpen(); + if (force && waitIfOngoing == false) { + throw new IllegalArgumentException( + "wait_if_ongoing must be true for a force flush: force=" + force + " wait_if_ongoing=" + waitIfOngoing + ); + } + try (ReleasableLock ignored = readLock.acquire()) { + ensureOpen(); + if (flushLock.tryLock() == false) { + if (waitIfOngoing == false) { + return; + } + flushLock.lock(); + } + try { + // Refresh first to flush buffered data to segments + refresh("flush"); + // committer.commit(); + translogManager.ensureCanFlush(); + translogManager.rollTranslogGeneration(); + translogManager.trimUnreferencedReaders(); + logger.trace("flush completed"); + } catch (AlreadyClosedException e) { + failOnTragicEvent(e); + throw e; + } catch (Exception e) { + throw new FlushFailedEngineException(shardId, e); + } finally { + flushLock.unlock(); + } + } + } + + @Override + public void flush() { + flush(false, true); + } + + @Override + public boolean shouldPeriodicallyFlush() { + ensureOpen(); + final long localCheckpointOfLastCommit = localCheckpointTracker.getPersistedCheckpoint(); + return translogManager.shouldPeriodicallyFlush( + localCheckpointOfLastCommit, + engineConfig.getIndexSettings().getFlushThresholdSize().getBytes() + ); + } + + @Override + public void writeIndexingBuffer() throws EngineException { + refresh("write indexing buffer"); + } + + @Override + public void forceMerge( + boolean flush, int maxNumSegments, boolean onlyExpungeDeletes, + boolean upgrade, boolean upgradeOnlyAncientSegments, String forceMergeUUID + ) throws EngineException, IOException { + // TODO: Delegate to IndexingExecutionEngine's Merger when merge scheduling is implemented + if (flush) { + flush(false, true); + } + } + + @Override + public long getIndexBufferRAMBytesUsed() { + return indexingExecutionEngine.getNativeBytesUsed(); + } + + @Override + public void activateThrottling() { + int count = throttleRequestCount.incrementAndGet(); + assert count >= 1 : "invalid post-increment throttleRequestCount=" + count; + if (count == 1) { + throttle.activate(); + } + } + + @Override + public void deactivateThrottling() { + int count = throttleRequestCount.decrementAndGet(); + assert count >= 0 : "invalid post-decrement throttleRequestCount=" + count; + if (count == 0) { + throttle.deactivate(); + } + } + + @Override + public boolean isThrottled() { + return throttle.isThrottled(); + } + + @Override + public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { + if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { + updateAutoIdTimestamp(Long.MAX_VALUE, true); + } + final TranslogDeletionPolicy translogDeletionPolicy = translogManager.getDeletionPolicy(); + translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); + translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); + } + + @Override + public boolean refreshNeeded() { + // A refresh is needed if there are operations since the last refresh + return true; + } + + @Override + public boolean maybeRefresh(String source) { + refresh(source); + return true; + } + + @Override + public void maybePruneDeletes() { + // No-op: data-format engines do not maintain Lucene-style delete tombstones + } + + @Override + public void verifyEngineBeforeIndexClosing() throws IllegalStateException { + final long globalCheckpoint = engineConfig.getGlobalCheckpointSupplier().getAsLong(); + final long maxSeqNo = getSeqNoStats(globalCheckpoint).getMaxSeqNo(); + if (globalCheckpoint != maxSeqNo) { + throw new IllegalStateException( + "Global checkpoint [" + globalCheckpoint + "] mismatches maximum sequence number [" + maxSeqNo + "]" + ); + } + } + + // ---- IndexerStateManager ---- + + @Override + public long getMaxSeenAutoIdTimestamp() { + return maxSeenAutoIdTimestamp.get(); + } + + @Override + public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) { + updateAutoIdTimestamp(newTimestamp, true); + } + + private void updateAutoIdTimestamp(long newTimestamp, boolean unsafe) { + assert newTimestamp >= -1 : "invalid timestamp [" + newTimestamp + "]"; + maxSeenAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); + if (unsafe) { + maxUnsafeAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); + } + } + + @Override + public long getMaxSeqNoOfUpdatesOrDeletes() { + return 0L; + } + + @Override + public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { + // no-op + } + + @Override + public long getLastWriteNanos() { + return lastWriteNanos; + } + + @Override + public long getPersistedLocalCheckpoint() { + return localCheckpointTracker.getPersistedCheckpoint(); + } + + @Override + public long getProcessedLocalCheckpoint() { + return localCheckpointTracker.getProcessedCheckpoint(); + } + + @Override + public SeqNoStats getSeqNoStats(long globalCheckpoint) { + return localCheckpointTracker.getStats(globalCheckpoint); + } + + @Override + public long getLastSyncedGlobalCheckpoint() { + return translogManager.getLastSyncedGlobalCheckpoint(); + } + + @Override + public long getMinRetainedSeqNo() { + return 0L; } - public void setCatalogSnapshotManager(CatalogSnapshotManager catalogSnapshotManager) { - this.catalogSnapshotManager = catalogSnapshotManager; + @Override + public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNumber) throws IOException { + ensureOpen(); + try (Translog.Snapshot snapshot = translogManager.newChangesSnapshot(fromSeqNo, toSeqNumber, false)) { + int count = 0; + while (snapshot.next() != null) { + count++; + } + return count; + } + } + + @Override + public boolean hasCompleteOperationHistory(String reason, long startingSeqNo) { + return getMinRetainedSeqNo() <= startingSeqNo; } - public EngineReaderManager getReaderManager(DataFormat format) { - return readerManagers.get(format); + @Override + public int fillSeqNoGaps(long primaryTerm) throws IOException { + // No-op: data-format engines do not maintain Lucene-style gaps + return 0; + } + + // ---- IndexerStatistics ---- + + @Override + public CommitStats commitStats() { + // TODO: Implement commit stats from catalog snapshot metadata + return null; + } + + @Override + public DocsStats docStats() { + // TODO: Derive from catalog snapshot segment metadata + return new DocsStats(0, 0, 0); + } + + @Override + public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + return new SegmentsStats(); + } + + @Override + public CompletionStats completionStats(String... fieldNamePatterns) { + return new CompletionStats(); + } + + @Override + public PollingIngestStats pollingIngestStats() { + return null; + } + + @Override + public MergeStats getMergeStats() { + return new MergeStats(); + } + + @Override + public long getIndexThrottleTimeInMillis() { + return throttle.getThrottleTimeInMillis(); + } + + @Override + public long getWritingBytes() { + return 0; + } + + @Override + public long unreferencedFileCleanUpsPerformed() { + return 0; + } + + @Override + public long getNativeBytesUsed() { + return indexingExecutionEngine.getNativeBytesUsed(); + } + + // ---- Indexer top-level methods ---- + + @Override + public EngineConfig config() { + return engineConfig; + } + + @Override + public SafeCommitInfo getSafeCommitInfo() { + return new SafeCommitInfo(localCheckpointTracker.getProcessedCheckpoint(), 0); + } + + @Override + public TranslogManager translogManager() { + return translogManager; + } + + @Override + public Closeable acquireHistoryRetentionLock() { + return () -> {}; + } + + @Override + public Translog.Snapshot newChangesSnapshot( + String source, long fromSeqNo, long toSeqNo, + boolean requiredFullRange, boolean accurateCount + ) throws IOException { + return translogManager.newChangesSnapshot(fromSeqNo, toSeqNo, requiredFullRange); + } + + @Override + public String getHistoryUUID() { + return historyUUID; + } + + @Override + public void flushAndClose() throws IOException { + if (isClosed.get() == false) { + try (ReleasableLock lock = writeLock.acquire()) { + try { + flush(false, true); + } catch (AlreadyClosedException ex) { + logger.debug("engine already closed - skipping flushAndClose"); + } finally { + close(); + } + } + } + awaitPendingClose(); + } + + @Override + public void failEngine(String reason, @Nullable Exception failure) { + if (failEngineLock.tryLock()) { + try { + if (failedEngine.get() != null) { + logger.warn(() -> new ParameterizedMessage( + "tried to fail engine but already failed, ignoring. [{}]", reason), failure); + return; + } + failedEngine.set(failure != null ? failure : new IllegalStateException(reason)); + try { + closeNoLock("engine failed on: [" + reason + "]"); + } finally { + logger.warn(() -> new ParameterizedMessage("failed engine [{}]", reason), failure); + engineConfig.getEventListener().onFailedEngine(reason, failure); + } + } catch (Exception inner) { + if (failure != null) { + inner.addSuppressed(failure); + } + logger.warn("failEngine threw exception", inner); + } + } else { + logger.debug(() -> new ParameterizedMessage( + "tried to fail engine but could not acquire lock [{}]", reason), failure); + } + } + + @Override + public GatedCloseable acquireSnapshot() { + return catalogSnapshotManager.acquireSnapshot(); + } + + @Override + public GatedCloseable acquireSafeIndexCommit() throws EngineException { + // TODO: Implement when commit coordination is added + throw new UnsupportedOperationException("acquireSafeIndexCommit not yet implemented for DataFormatBasedEngine"); } /** @@ -90,6 +965,146 @@ public GatedCloseable acquireReader() throws IOException { } } + @Override + public void ensureOpen() { + if (isClosed.get()) { + throw new AlreadyClosedException(shardId + " engine is closed", failedEngine.get()); + } + } + + @Override + public void close() throws IOException { + if (isClosed.get() == false) { + try (ReleasableLock lock = writeLock.acquire()) { + closeNoLock("api"); + } + } + awaitPendingClose(); + } + + private void closeNoLock(String reason) { + if (isClosed.compareAndSet(false, true)) { + assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() + : "Either the write lock must be held or the engine must be currently failing"; + try { + IOUtils.close(indexingExecutionEngine, translogManager); + closeReaders(); + } catch (Exception e) { + logger.warn("failed to close engine resources", e); + } finally { + try { + store.decRef(); + logger.debug("engine closed [{}]", reason); + } finally { + closedLatch.countDown(); + } + } + } + } + + private void closeReaders() throws IOException { + List exceptions = new ArrayList<>(); + for (EngineReaderManager rm : readerManagers.values()) { + try { + rm.close(); + } catch (Exception e) { + exceptions.add(e); + } + } + if (exceptions.isEmpty() == false) { + IOException ioException = new IOException("Failed to close DataFormatAwareEngine resources"); + for (Exception e : exceptions) { + ioException.addSuppressed(e); + } + throw ioException; + } + } + + private void awaitPendingClose() { + try { + closedLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private boolean failOnTragicEvent(AlreadyClosedException ex) { + if (translogManager.getTragicExceptionIfClosed() != null) { + failEngine("already closed by tragic event on the translog", translogManager.getTragicExceptionIfClosed()); + return true; + } else if (failedEngine.get() == null && isClosed.get() == false) { + throw new AssertionError("Unexpected AlreadyClosedException", ex); + } + return false; + } + + private boolean maybeFailEngine(String source, Exception e) { + if (e instanceof AlreadyClosedException) { + return failOnTragicEvent((AlreadyClosedException) e); + } else if (e != null && translogManager.getTragicExceptionIfClosed() == e) { + failEngine(source, e); + return true; + } + return false; + } + + /** + * Returns the underlying {@link IndexingExecutionEngine}. + * + * @return the indexing execution engine + */ + public IndexingExecutionEngine getIndexingExecutionEngine() { + return indexingExecutionEngine; + } + + /** + * Returns the current writer generation counter. + * + * @return the current writer generation + */ + public long getCurrentWriterGeneration() { + return writerGenerationCounter.get(); + } + + private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + if (origin == Engine.Operation.Origin.PRIMARY) { + assert assertPrimaryIncomingSequenceNumber(origin, seqNo); + } else { + // sequence number should be set when operation origin is not primary + assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin; + } + return true; + } + + boolean assertPrimaryIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { + // sequence number should not be set when operation origin is primary + assert + seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : + "primary operations must never have an assigned sequence number but was [" + seqNo + "]"; + return true; + } + + private boolean hasBeenProcessedBefore(Engine.Operation op) { + assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "operation is not assigned seq_no"; + return localCheckpointTracker.hasProcessed(op.seqNo()); + } + + private long generateSeqNoForOperationOnPrimary(final Engine.Operation operation) { + assert operation.origin() == Engine.Operation.Origin.PRIMARY; + assert + operation.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : + "ops should not have an assigned seq no. but was: " + operation.seqNo(); + return doGenerateSeqNoForOperation(operation); + } + + private long doGenerateSeqNoForOperation(final Engine.Operation operation) { + return localCheckpointTracker.generateSeqNo(); + } + + private void markSeqNoAsSeen(long seqNo) { + localCheckpointTracker.advanceMaxSeqNo(seqNo); + } + /** * A catalog-snapshot-backed data-format aware reader providing per-format reader access. * Closing this reader releases the catalog snapshot reference. @@ -153,24 +1168,4 @@ public void close() { } } - @Override - public void close() throws IOException { - List exceptions = new ArrayList<>(); - for (EngineReaderManager rm : readerManagers.values()) { - if (rm instanceof Closeable) { - try { - ((Closeable) rm).close(); - } catch (Exception e) { - exceptions.add(e); - } - } - } - if (exceptions.isEmpty() == false) { - IOException ioException = new IOException("Failed to close DataFormatAwareEngine resources"); - for (Exception e : exceptions) { - ioException.addSuppressed(e); - } - throw ioException; - } - } } diff --git a/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java b/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java deleted file mode 100644 index cd2945578a207..0000000000000 --- a/server/src/main/java/org/opensearch/index/engine/DataFormatBasedEngine.java +++ /dev/null @@ -1,1068 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.engine; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; -import org.apache.lucene.index.IndexCommit; -import org.apache.lucene.store.AlreadyClosedException; -import org.opensearch.common.Nullable; -import org.opensearch.common.SetOnce; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.common.concurrent.GatedCloseable; -import org.opensearch.common.lease.Releasable; -import org.opensearch.common.logging.Loggers; -import org.opensearch.common.queue.LockablePool; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.ReleasableLock; -import org.opensearch.common.util.io.IOUtils; -import org.opensearch.core.common.unit.ByteSizeValue; -import org.opensearch.core.index.AppendOnlyIndexOperationRetryException; -import org.opensearch.core.index.shard.ShardId; -import org.opensearch.index.VersionType; -import org.opensearch.index.engine.dataformat.*; -import org.opensearch.index.engine.dataformat.commit.Committer; -import org.opensearch.index.engine.exec.*; -import org.opensearch.index.engine.exec.Segment; -import org.opensearch.index.engine.exec.coord.CatalogSnapshot; -import org.opensearch.index.engine.exec.coord.CatalogSnapshotManager; -import org.opensearch.index.mapper.*; -import org.opensearch.index.merge.MergeStats; -import org.opensearch.index.seqno.LocalCheckpointTracker; -import org.opensearch.index.seqno.SeqNoStats; -import org.opensearch.index.seqno.SequenceNumbers; -import org.opensearch.index.shard.DocsStats; -import org.opensearch.index.store.Store; -import org.opensearch.index.translog.DefaultTranslogDeletionPolicy; -import org.opensearch.index.translog.InternalTranslogManager; -import org.opensearch.index.translog.Translog; -import org.opensearch.index.translog.TranslogCorruptedException; -import org.opensearch.index.translog.TranslogDeletionPolicy; -import org.opensearch.index.translog.TranslogException; -import org.opensearch.index.translog.TranslogManager; -import org.opensearch.index.translog.TranslogOperationHelper; -import org.opensearch.index.translog.listener.TranslogEventListener; -import org.opensearch.indices.pollingingest.PollingIngestStats; -import org.opensearch.search.suggest.completion.CompletionStats; - -import java.io.Closeable; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.BiFunction; - -import org.apache.lucene.index.Term; - -import static org.opensearch.index.engine.Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID; - -/** - * An {@link Indexer} implementation that delegates to an {@link IndexingExecutionEngine}. - * This engine manages the full lifecycle of indexing, - * deleting, and searching documents on a single shard by coordinating between the - * underlying data format engine and the translog. - *

- * Mirrors the responsibilities of {@link InternalEngine} but is decoupled from Lucene: - *

    - *
  • Sequence number generation and local checkpoint tracking
    • - *
  • Translog management for durability and recovery
    • - *
  • Refresh to make documents searchable via catalog snapshots
    • - *
  • Flush to commit catalog snapshots and trim the translog
    • - *
  • Throttling when merges fall behind or indexing buffer is full
    • - *
  • Soft-deletes policy for peer-recovery retention
    • - *
- * - * @opensearch.experimental - */ -@ExperimentalApi -public class DataFormatBasedEngine implements Indexer { - - private final Logger logger; - private final EngineConfig engineConfig; - private final ShardId shardId; - private final Store store; - - private final IndexingExecutionEngine indexingExecutionEngine; - private final IndexingStrategyPlanner indexingStrategyPlanner; - private final LockablePool> writerPool; - private final AtomicLong writerGenerationCounter; - - private final Map> readerManagers; - private volatile CatalogSnapshotManager catalogSnapshotManager; - private Committer committer; - - // Translog for durability and recovery - private final TranslogManager translogManager; - - // Sequence number tracking - private final LocalCheckpointTracker localCheckpointTracker; - - // Throttling - private final IndexingThrottler throttle; - private final AtomicInteger throttleRequestCount = new AtomicInteger(); - - // Timestamps and seq-no markers - private final AtomicLong maxUnsafeAutoIdTimestamp = new AtomicLong(-1); - private final AtomicLong maxSeenAutoIdTimestamp = new AtomicLong(-1); - private volatile long lastWriteNanos = System.nanoTime(); - - // Lifecycle - private final AtomicBoolean isClosed = new AtomicBoolean(false); - private final SetOnce failedEngine = new SetOnce<>(); - private final CountDownLatch closedLatch = new CountDownLatch(1); - - // Concurrency locks - private final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock(); - private final ReleasableLock readLock = new ReleasableLock(rwl.readLock()); - private final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock()); - private final Lock flushLock = new ReentrantLock(); - private final ReentrantLock failEngineLock = new ReentrantLock(); - - - // Refresh tracker - private final LastRefreshedCheckpointListener lastRefreshedCheckpointListener; - - @Nullable - private final String historyUUID; - - /** - * Constructs a DataFormatBasedEngine. - * - * @param engineConfig the engine configuration - */ - public DataFormatBasedEngine(EngineConfig engineConfig) { - this.logger = Loggers.getLogger(DataFormatBasedEngine.class, engineConfig.getShardId()); - this.engineConfig = engineConfig; - this.shardId = engineConfig.getShardId(); - this.store = engineConfig.getStore(); - this.throttle = new IndexingThrottler(); - - if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { - updateAutoIdTimestamp(Long.MAX_VALUE, true); - } - - boolean success = false; - TranslogManager translogManagerRef = null; - - try { - store.incRef(); - - // init committer with store - Committer constructingCommitter = engineConfig.getCommitterFactory().getCommitter(store); - - // Read history UUID and translog UUID from last commit - final Map userData = constructingCommitter.readLastCommittedUserData(); - String translogUUID = Objects.requireNonNull(userData.get(Translog.TRANSLOG_UUID_KEY)); - - // Initialize translog - // TODO: Once file deleter is merged, we will add relevant listeners - final TranslogEventListener translogEventListener = createInternalTranslogEventListener(); - translogManagerRef = createTranslogManager(translogUUID, translogEventListener); - this.translogManager = translogManagerRef; - - // Initialize local checkpoint tracker from last committed segment infos - this.localCheckpointTracker = createLocalCheckpointTracker(LocalCheckpointTracker::new); - - // Initialize from commit data - this.historyUUID = userData.get(Engine.HISTORY_UUID_KEY); - updateAutoIdTimestamp(Long.parseLong(userData.get(MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID)), true); - - - // Move to data format aware writers and readers. - DataFormatRegistry registry = engineConfig.getDataFormatRegistry(); - // Create indexing engine - // Pass committer here as well. - this.indexingExecutionEngine = registry.getIndexingEngine(registry.format(engineConfig.getIndexSettings().pluggableDataFormat()), - engineConfig.getMapperService(), store.shardPath(), engineConfig.getIndexSettings()); - this.writerGenerationCounter = new AtomicLong(committer.getLastCommittedGeneration()); - this.writerPool = new LockablePool<>( - () -> indexingExecutionEngine.createWriter(writerGenerationCounter.getAndIncrement()), - LinkedList::new, - Runtime.getRuntime().availableProcessors() - ); - // Create Reader managers - // We will pass IndexViewProvider to this, which would contain store - // and any index specific attributes useful for reads. - this.readerManagers = registry.getReaderManagers(engineConfig.getMapperService(), - engineConfig.getIndexSettings(), - store.shardPath()); - - this.lastRefreshedCheckpointListener = new LastRefreshedCheckpointListener(localCheckpointTracker); - this.indexingStrategyPlanner = new IndexingStrategyPlanner( - engineConfig.getIndexSettings(), - engineConfig.getShardId(), - new LiveVersionMap(), - maxUnsafeAutoIdTimestamp::get, - () -> 0L, - localCheckpointTracker::getProcessedCheckpoint, - this::hasBeenProcessedBefore, - op -> OpVsEngineDocStatus.OP_NEWER, - (a, b) -> null, - this::updateAutoIdTimestamp, - (a, b) -> null - ); - success = true; - logger.trace("created new DataFormatBasedEngine"); - } catch (IOException | TranslogCorruptedException e) { - throw new EngineCreationFailureException(shardId, "failed to create engine", e); - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(translogManagerRef); - if (isClosed.get() == false) { - store.decRef(); - } - } - } - } - - private LocalCheckpointTracker createLocalCheckpointTracker( - BiFunction supplier - ) throws IOException { - final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit( - store.readLastCommittedSegmentsInfo().getUserData().entrySet() - ); - logger.trace("recovered max_seq_no [{}] and local_checkpoint [{}]", seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); - return supplier.apply(seqNoStats.maxSeqNo, seqNoStats.localCheckpoint); - } - - private TranslogEventListener createInternalTranslogEventListener() { - return new TranslogEventListener() { - @Override - public void onAfterTranslogSync() { - try { - // TODO: Handle file deletion policy - translogManager.trimUnreferencedReaders(); - } catch (IOException ex) { - throw new TranslogException(shardId, "failed to trim translog on sync", ex); - } - } - - @Override - public void onAfterTranslogRecovery() { - flush(false, true); - translogManager.trimUnreferencedTranslogFiles(); - } - - @Override - public void onFailure(String reason, Exception ex) { - if (ex instanceof AlreadyClosedException) { - failOnTragicEvent((AlreadyClosedException) ex); - } else { - failEngine(reason, ex); - } - } - }; - } - - private TranslogManager createTranslogManager( - String translogUUID, - TranslogEventListener translogEventListener - ) throws IOException { - TranslogDeletionPolicy deletionPolicy = getTranslogDeletionPolicy(); - return new InternalTranslogManager( - engineConfig.getTranslogConfig(), - engineConfig.getPrimaryTermSupplier(), - engineConfig.getGlobalCheckpointSupplier(), - deletionPolicy, - shardId, - readLock, - () -> localCheckpointTracker, - translogUUID, - translogEventListener, - this::ensureOpen, - engineConfig.getTranslogFactory(), - engineConfig.getStartedPrimarySupplier(), - TranslogOperationHelper.create(engineConfig) - ); - } - - private TranslogDeletionPolicy getTranslogDeletionPolicy() { - TranslogDeletionPolicy custom = null; - if (engineConfig.getCustomTranslogDeletionPolicyFactory() != null) { - custom = engineConfig.getCustomTranslogDeletionPolicyFactory() - .create(engineConfig.getIndexSettings(), engineConfig.retentionLeasesSupplier()); - } - return Objects.requireNonNullElseGet(custom, () -> new DefaultTranslogDeletionPolicy( - engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(), - engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis(), - engineConfig.getIndexSettings().getTranslogRetentionTotalFiles() - )); - } - - @Override - public Engine.IndexResult index(Engine.Index index) throws IOException { - assert Objects.equals(index.uid().field(), IdFieldMapper.NAME) : index.uid().field(); - final boolean doThrottle = index.origin().isRecovery() == false; - try (ReleasableLock ignored = readLock.acquire()) { - ensureOpen(); - try (Releasable indexThrottle = doThrottle ? throttle.acquireThrottle() : () -> {}) { - lastWriteNanos = index.startTime(); - final IndexingStrategy plan = indexingStrategyPlanner.planOperationAsPrimary(index); - final Engine.IndexResult indexResult; - if (plan.earlyResultOnPreFlightError.isPresent()) { - assert index.origin() == Engine.Operation.Origin.PRIMARY : index.origin(); - indexResult = (Engine.IndexResult) plan.earlyResultOnPreFlightError.get(); - assert indexResult.getResultType() == Engine.Result.Type.FAILURE : indexResult.getResultType(); - } else { - if (index.origin() == Engine.Operation.Origin.PRIMARY) { - index = new Engine.Index( - index.uid(), - index.parsedDoc(), - generateSeqNoForOperationOnPrimary(index), - index.primaryTerm(), - index.version(), - index.versionType(), - index.origin(), - index.startTime(), - index.getAutoGeneratedIdTimestamp(), - index.isRetry(), - index.getIfSeqNo(), - index.getIfPrimaryTerm() - ); - } else { - markSeqNoAsSeen(index.seqNo()); - } - - assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); - - if (plan.executeOpOnEngine) { - logger.debug("Indexing doc id=[{}] seqNo=[{}] primaryTerm=[{}] — writing to engine", - index.id(), index.seqNo(), index.primaryTerm()); - indexResult = indexIntoEngine(index); - } else { - indexResult = - new Engine.IndexResult(plan.version, index.primaryTerm(), index.seqNo(), plan.currentNotFoundOrDeleted); - } - } - return indexResult; - } - } catch (RuntimeException | IOException e) { - maybeFailEngine("index id[" + index.id() + "] origin[" + index.origin() + "]", e); - throw e; - } - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private Engine.IndexResult indexIntoEngine(Engine.Index index) throws IOException { - Engine.IndexResult indexResult; - if (index.origin() == Engine.Operation.Origin.PRIMARY) { - // Assign a sequence number on the primary - final long seqNo = localCheckpointTracker.generateSeqNo(); - index = new Engine.Index( - index.uid(), index.parsedDoc(), seqNo, index.primaryTerm(), - index.version(), index.versionType(), index.origin(), index.startTime(), - index.getAutoGeneratedIdTimestamp(), index.isRetry(), - index.getIfSeqNo(), index.getIfPrimaryTerm() - ); - } else { - // On replica, advance max seq no - localCheckpointTracker.advanceMaxSeqNo(index.seqNo()); - } - - assert index.seqNo() >= 0 : "ops should have an assigned seq no.; origin: " + index.origin(); - - // Convert ParsedDocument to DocumentInput and write via the execution engine's writer - Writer currentWriter = null; - try { - currentWriter = writerPool.getAndLock(); - - //Replace with index#documentInput - DocumentInput docInput = indexingExecutionEngine.newDocumentInput(); - - WriteResult result = currentWriter.addDoc(docInput); - - if (result instanceof WriteResult.Success(long version, long term, long seqNo)) { - indexResult = new Engine.IndexResult(version, term, seqNo, true); - } else { - WriteResult.Failure f = (WriteResult.Failure) result; - indexResult = new Engine.IndexResult(f.cause(), index.version(), index.primaryTerm(), index.seqNo()); - } - } catch (Exception e) { - indexResult = new Engine.IndexResult(e, index.version(), index.primaryTerm(), index.seqNo()); - } finally { - if (currentWriter != null) { - writerPool.releaseAndUnlock(currentWriter); - } - } - - if (index.origin().isFromTranslog() == false) { - final Translog.Location location; - if (indexResult.getResultType() == Engine.Result.Type.SUCCESS) { - location = translogManager.add(new Translog.Index(index, indexResult)); - } else if (indexResult.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && indexResult.getFailure() != null - && !(indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException)) { - throw new UnsupportedOperationException("recording document failure as a no-op in translog is not " + - "supported for Data format engine"); - } else { - location = null; - } - indexResult.setTranslogLocation(location); - } - - // Track the sequence number - localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); - if (indexResult.getTranslogLocation() == null) { - localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); - } - - localCheckpointTracker.markSeqNoAsProcessed(indexResult.getSeqNo()); - if (indexResult.getTranslogLocation() == null && !(indexResult.getFailure() != null - && (indexResult.getFailure() instanceof AppendOnlyIndexOperationRetryException))) { - // the op is coming from the translog (and is hence persisted already) or it does not have a sequence number - assert index.origin().isFromTranslog() || indexResult.getSeqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO; - localCheckpointTracker.markSeqNoAsPersisted(indexResult.getSeqNo()); - } - indexResult.setTook(System.nanoTime() - index.startTime()); - indexResult.freeze(); - return indexResult; - } - - @Override - public Engine.DeleteResult delete(Engine.Delete delete) throws IOException { - try (ReleasableLock ignored = readLock.acquire()) { - ensureOpen(); - lastWriteNanos = delete.startTime(); - - final long seqNo; - if (delete.origin() == Engine.Operation.Origin.PRIMARY) { - seqNo = localCheckpointTracker.generateSeqNo(); - } else { - seqNo = delete.seqNo(); - localCheckpointTracker.advanceMaxSeqNo(seqNo); - } - - // TODO: Implement actual delete via IndexingExecutionEngine when supported. - // For now, record the delete in the translog and track the seq no. - final Engine.DeleteResult deleteResult = new Engine.DeleteResult(1L, delete.primaryTerm(), seqNo, true); - - if (delete.origin().isFromTranslog() == false) { - final Translog.Location location = translogManager.add(new Translog.Delete(delete, deleteResult)); - deleteResult.setTranslogLocation(location); - } - - localCheckpointTracker.markSeqNoAsProcessed(seqNo); - if (deleteResult.getTranslogLocation() == null) { - localCheckpointTracker.markSeqNoAsPersisted(seqNo); - } - - deleteResult.setTook(System.nanoTime() - delete.startTime()); - deleteResult.freeze(); - return deleteResult; - } catch (RuntimeException | IOException e) { - maybeFailEngine("delete id[" + delete.id() + "] origin[" + delete.origin() + "]", e); - throw e; - } - } - - @Override - public Engine.NoOpResult noOp(Engine.NoOp noOp) throws IOException { - try (ReleasableLock ignored = readLock.acquire()) { - ensureOpen(); - final Engine.NoOpResult noOpResult = new Engine.NoOpResult(noOp.primaryTerm(), noOp.seqNo()); - - if (noOp.origin().isFromTranslog() == false) { - final Translog.Location location = translogManager.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); - noOpResult.setTranslogLocation(location); - } - - localCheckpointTracker.markSeqNoAsProcessed(noOp.seqNo()); - if (noOpResult.getTranslogLocation() == null) { - localCheckpointTracker.markSeqNoAsPersisted(noOp.seqNo()); - } - - noOpResult.setTook(System.nanoTime() - noOp.startTime()); - noOpResult.freeze(); - return noOpResult; - } catch (RuntimeException | IOException e) { - maybeFailEngine("noOp seq#[" + noOp.seqNo() + "]", e); - throw e; - } - } - - @Override - public Engine.Index prepareIndex( - DocumentMapperForType docMapper, SourceToParse source, - long seqNo, long primaryTerm, long version, VersionType versionType, - Engine.Operation.Origin origin, long autoGeneratedIdTimestamp, - boolean isRetry, long ifSeqNo, long ifPrimaryTerm - ) { - long startTime = System.nanoTime(); - ParsedDocument doc = docMapper.getDocumentMapper().parse(source); - if (docMapper.getMapping() != null) { - doc.addDynamicMappingsUpdate(docMapper.getMapping()); - } - Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id())); - return new Engine.Index( - uid, doc, seqNo, primaryTerm, version, versionType, - origin, startTime, autoGeneratedIdTimestamp, isRetry, ifSeqNo, ifPrimaryTerm - ); - } - - @Override - public Engine.Delete prepareDelete( - String id, long seqNo, long primaryTerm, long version, - VersionType versionType, Engine.Operation.Origin origin, - long ifSeqNo, long ifPrimaryTerm - ) { - long startTime = System.nanoTime(); - Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(id)); - return new Engine.Delete(id, uid, seqNo, primaryTerm, version, versionType, origin, startTime, ifSeqNo, ifPrimaryTerm); - } - - // ---- IndexerLifecycleOperations ---- - - @Override - public synchronized void refresh(String source) throws EngineException { - final long localCheckpointBeforeRefresh = localCheckpointTracker.getProcessedCheckpoint(); - boolean refreshed; - try (GatedCloseable catalogSnapshot = catalogSnapshotManager.acquireSnapshot()) { - if (store.tryIncRef()) { - try { - List> writers = writerPool.checkoutAll(); - List existingSegments = catalogSnapshot.get().getSegments(); - List newSegments = new ArrayList<>(); - List fileInfosList = new ArrayList<>(); - - for (Writer writer : writers) { - FileInfos fileInfos = writer.flush(); - Segment.Builder segmentBuilder = Segment.builder(writer.generation()); - boolean hasFiles = false; - for (Map.Entry entry : fileInfos.writerFilesMap().entrySet()) { - logger.debug( - "Writer gen={} flushed format=[{}] files={}", - writer.generation(), - entry.getKey().name(), - entry.getValue().files() - ); - segmentBuilder.addSearchableFiles(entry.getKey(), entry.getValue()); - hasFiles = true; - } - writer.close(); - if (hasFiles) { - newSegments.add(segmentBuilder.build()); - } - } - logger.debug("Produced {} new segments from flush", newSegments.size()); - - RefreshInput refreshInput = new RefreshInput(existingSegments, newSegments); - RefreshResult result = indexingExecutionEngine.refresh(refreshInput); - catalogSnapshotManager.commitNewSnapshot(result.refreshedSegments()); - refreshed = true; - } finally { - store.decRef(); - } - if (refreshed) { - lastRefreshedCheckpointListener.updateRefreshedCheckpoint(localCheckpointBeforeRefresh); - } - } - } catch (AlreadyClosedException ex) { - failOnTragicEvent(ex); - throw ex; - } catch (Exception ex) { - try { - failEngine("refresh failed source[" + source + "]", ex); - } catch (Exception inner) { - ex.addSuppressed(inner); - } - throw new RefreshFailedEngineException(shardId, ex); - } - } - - @Override - public void flush(boolean force, boolean waitIfOngoing) throws EngineException { - ensureOpen(); - if (force && waitIfOngoing == false) { - throw new IllegalArgumentException( - "wait_if_ongoing must be true for a force flush: force=" + force + " wait_if_ongoing=" + waitIfOngoing - ); - } - try (ReleasableLock ignored = readLock.acquire()) { - ensureOpen(); - if (flushLock.tryLock() == false) { - if (waitIfOngoing == false) { - return; - } - flushLock.lock(); - } - try { - // Refresh first to flush buffered data to segments - refresh("flush"); - translogManager.ensureCanFlush(); - translogManager.rollTranslogGeneration(); - translogManager.trimUnreferencedReaders(); - logger.trace("flush completed"); - } catch (AlreadyClosedException e) { - failOnTragicEvent(e); - throw e; - } catch (Exception e) { - throw new FlushFailedEngineException(shardId, e); - } finally { - flushLock.unlock(); - } - } - } - - @Override - public void flush() { - flush(false, true); - } - - @Override - public boolean shouldPeriodicallyFlush() { - ensureOpen(); - final long localCheckpointOfLastCommit = localCheckpointTracker.getPersistedCheckpoint(); - return translogManager.shouldPeriodicallyFlush( - localCheckpointOfLastCommit, - engineConfig.getIndexSettings().getFlushThresholdSize().getBytes() - ); - } - - @Override - public void writeIndexingBuffer() throws EngineException { - refresh("write indexing buffer"); - } - - @Override - public void forceMerge( - boolean flush, int maxNumSegments, boolean onlyExpungeDeletes, - boolean upgrade, boolean upgradeOnlyAncientSegments, String forceMergeUUID - ) throws EngineException, IOException { - // TODO: Delegate to IndexingExecutionEngine's Merger when merge scheduling is implemented - if (flush) { - flush(false, true); - } - } - - @Override - public long getIndexBufferRAMBytesUsed() { - return indexingExecutionEngine.getNativeBytesUsed(); - } - - @Override - public void activateThrottling() { - int count = throttleRequestCount.incrementAndGet(); - assert count >= 1 : "invalid post-increment throttleRequestCount=" + count; - if (count == 1) { - throttle.activate(); - } - } - - @Override - public void deactivateThrottling() { - int count = throttleRequestCount.decrementAndGet(); - assert count >= 0 : "invalid post-decrement throttleRequestCount=" + count; - if (count == 0) { - throttle.deactivate(); - } - } - - @Override - public boolean isThrottled() { - return throttle.isThrottled(); - } - - @Override - public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { - if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { - updateAutoIdTimestamp(Long.MAX_VALUE, true); - } - final TranslogDeletionPolicy translogDeletionPolicy = translogManager.getDeletionPolicy(); - translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); - translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); - } - - @Override - public boolean refreshNeeded() { - // A refresh is needed if there are operations since the last refresh - return true; - } - - @Override - public boolean maybeRefresh(String source) { - refresh(source); - return true; - } - - @Override - public void maybePruneDeletes() { - // No-op: data-format engines do not maintain Lucene-style delete tombstones - } - - @Override - public void verifyEngineBeforeIndexClosing() throws IllegalStateException { - final long globalCheckpoint = engineConfig.getGlobalCheckpointSupplier().getAsLong(); - final long maxSeqNo = getSeqNoStats(globalCheckpoint).getMaxSeqNo(); - if (globalCheckpoint != maxSeqNo) { - throw new IllegalStateException( - "Global checkpoint [" + globalCheckpoint + "] mismatches maximum sequence number [" + maxSeqNo + "]" - ); - } - } - - // ---- IndexerStateManager ---- - - @Override - public long getMaxSeenAutoIdTimestamp() { - return maxSeenAutoIdTimestamp.get(); - } - - @Override - public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) { - updateAutoIdTimestamp(newTimestamp, true); - } - - private void updateAutoIdTimestamp(long newTimestamp, boolean unsafe) { - assert newTimestamp >= -1 : "invalid timestamp [" + newTimestamp + "]"; - maxSeenAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); - if (unsafe) { - maxUnsafeAutoIdTimestamp.updateAndGet(curr -> Math.max(curr, newTimestamp)); - } - } - - @Override - public long getMaxSeqNoOfUpdatesOrDeletes() { - return 0L; - } - - @Override - public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { - // no-op - } - - @Override - public long getLastWriteNanos() { - return lastWriteNanos; - } - - @Override - public long getPersistedLocalCheckpoint() { - return localCheckpointTracker.getPersistedCheckpoint(); - } - - @Override - public long getProcessedLocalCheckpoint() { - return localCheckpointTracker.getProcessedCheckpoint(); - } - - @Override - public SeqNoStats getSeqNoStats(long globalCheckpoint) { - return localCheckpointTracker.getStats(globalCheckpoint); - } - - @Override - public long getLastSyncedGlobalCheckpoint() { - return translogManager.getLastSyncedGlobalCheckpoint(); - } - - @Override - public long getMinRetainedSeqNo() { - return 0L; - } - - @Override - public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNumber) throws IOException { - ensureOpen(); - try (Translog.Snapshot snapshot = translogManager.newChangesSnapshot(fromSeqNo, toSeqNumber, false)) { - int count = 0; - while (snapshot.next() != null) { - count++; - } - return count; - } - } - - @Override - public boolean hasCompleteOperationHistory(String reason, long startingSeqNo) { - return getMinRetainedSeqNo() <= startingSeqNo; - } - - @Override - public int fillSeqNoGaps(long primaryTerm) throws IOException { - // No-op: data-format engines do not maintain Lucene-style gaps - return 0; - } - - // ---- IndexerStatistics ---- - - @Override - public CommitStats commitStats() { - // TODO: Implement commit stats from catalog snapshot metadata - return null; - } - - @Override - public DocsStats docStats() { - // TODO: Derive from catalog snapshot segment metadata - return new DocsStats(0, 0, 0); - } - - @Override - public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { - return new SegmentsStats(); - } - - @Override - public CompletionStats completionStats(String... fieldNamePatterns) { - return new CompletionStats(); - } - - @Override - public PollingIngestStats pollingIngestStats() { - return null; - } - - @Override - public MergeStats getMergeStats() { - return new MergeStats(); - } - - @Override - public long getIndexThrottleTimeInMillis() { - return throttle.getThrottleTimeInMillis(); - } - - @Override - public long getWritingBytes() { - return 0; - } - - @Override - public long unreferencedFileCleanUpsPerformed() { - return 0; - } - - @Override - public long getNativeBytesUsed() { - return indexingExecutionEngine.getNativeBytesUsed(); - } - - // ---- Indexer top-level methods ---- - - @Override - public EngineConfig config() { - return engineConfig; - } - - @Override - public SafeCommitInfo getSafeCommitInfo() { - return new SafeCommitInfo(localCheckpointTracker.getProcessedCheckpoint(), 0); - } - - @Override - public TranslogManager translogManager() { - return translogManager; - } - - @Override - public Closeable acquireHistoryRetentionLock() { - return () -> {}; - } - - @Override - public Translog.Snapshot newChangesSnapshot( - String source, long fromSeqNo, long toSeqNo, - boolean requiredFullRange, boolean accurateCount - ) throws IOException { - return translogManager.newChangesSnapshot(fromSeqNo, toSeqNo, requiredFullRange); - } - - @Override - public String getHistoryUUID() { - return historyUUID; - } - - @Override - public void flushAndClose() throws IOException { - if (isClosed.get() == false) { - try (ReleasableLock lock = writeLock.acquire()) { - try { - flush(false, true); - } catch (AlreadyClosedException ex) { - logger.debug("engine already closed - skipping flushAndClose"); - } finally { - close(); - } - } - } - awaitPendingClose(); - } - - @Override - public void failEngine(String reason, @Nullable Exception failure) { - if (failEngineLock.tryLock()) { - try { - if (failedEngine.get() != null) { - logger.warn(() -> new ParameterizedMessage( - "tried to fail engine but already failed, ignoring. [{}]", reason), failure); - return; - } - failedEngine.set(failure != null ? failure : new IllegalStateException(reason)); - try { - closeNoLock("engine failed on: [" + reason + "]"); - } finally { - logger.warn(() -> new ParameterizedMessage("failed engine [{}]", reason), failure); - engineConfig.getEventListener().onFailedEngine(reason, failure); - } - } catch (Exception inner) { - if (failure != null) { - inner.addSuppressed(failure); - } - logger.warn("failEngine threw exception", inner); - } - } else { - logger.debug(() -> new ParameterizedMessage( - "tried to fail engine but could not acquire lock [{}]", reason), failure); - } - } - - @Override - public GatedCloseable acquireSnapshot() { - // TODO: Implement catalog snapshot acquisition from segment tracking - throw new UnsupportedOperationException("acquireSnapshot not yet implemented for DataFormatBasedEngine"); - } - - @Override - public GatedCloseable acquireSafeIndexCommit() throws EngineException { - // TODO: Implement when commit coordination is added - throw new UnsupportedOperationException("acquireSafeIndexCommit not yet implemented for DataFormatBasedEngine"); - } - - @Override - public GatedCloseable acquireReader() throws IOException { - // TODO: Implement when reader management is wired to EngineReaderManager - throw new UnsupportedOperationException("acquireReader not yet implemented for DataFormatBasedEngine"); - } - - @Override - public void ensureOpen() { - if (isClosed.get()) { - throw new AlreadyClosedException(shardId + " engine is closed", failedEngine.get()); - } - } - - @Override - public void close() throws IOException { - if (isClosed.get() == false) { - try (ReleasableLock lock = writeLock.acquire()) { - closeNoLock("api"); - } - } - awaitPendingClose(); - } - - private void closeNoLock(String reason) { - if (isClosed.compareAndSet(false, true)) { - assert rwl.isWriteLockedByCurrentThread() || failEngineLock.isHeldByCurrentThread() - : "Either the write lock must be held or the engine must be currently failing"; - try { - IOUtils.close(indexingExecutionEngine, translogManager); - } catch (Exception e) { - logger.warn("failed to close engine resources", e); - } finally { - try { - store.decRef(); - logger.debug("engine closed [{}]", reason); - } finally { - closedLatch.countDown(); - } - } - } - } - - private void awaitPendingClose() { - try { - closedLatch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private boolean failOnTragicEvent(AlreadyClosedException ex) { - if (translogManager.getTragicExceptionIfClosed() != null) { - failEngine("already closed by tragic event on the translog", translogManager.getTragicExceptionIfClosed()); - return true; - } else if (failedEngine.get() == null && isClosed.get() == false) { - throw new AssertionError("Unexpected AlreadyClosedException", ex); - } - return false; - } - - private boolean maybeFailEngine(String source, Exception e) { - if (e instanceof AlreadyClosedException) { - return failOnTragicEvent((AlreadyClosedException) e); - } else if (e != null && translogManager.getTragicExceptionIfClosed() == e) { - failEngine(source, e); - return true; - } - return false; - } - - /** - * Returns the underlying {@link IndexingExecutionEngine}. - * - * @return the indexing execution engine - */ - public IndexingExecutionEngine getIndexingExecutionEngine() { - return indexingExecutionEngine; - } - - /** - * Returns the current writer generation counter. - * - * @return the current writer generation - */ - public long getCurrentWriterGeneration() { - return writerGenerationCounter.get(); - } - - private boolean assertIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { - if (origin == Engine.Operation.Origin.PRIMARY) { - assert assertPrimaryIncomingSequenceNumber(origin, seqNo); - } else { - // sequence number should be set when operation origin is not primary - assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no.; origin: " + origin; - } - return true; - } - - boolean assertPrimaryIncomingSequenceNumber(final Engine.Operation.Origin origin, final long seqNo) { - // sequence number should not be set when operation origin is primary - assert - seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO : - "primary operations must never have an assigned sequence number but was [" + seqNo + "]"; - return true; - } - - private boolean hasBeenProcessedBefore(Engine.Operation op) { - assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "operation is not assigned seq_no"; - return localCheckpointTracker.hasProcessed(op.seqNo()); - } - - private long generateSeqNoForOperationOnPrimary(final Engine.Operation operation) { - assert operation.origin() == Engine.Operation.Origin.PRIMARY; - assert - operation.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : - "ops should not have an assigned seq no. but was: " + operation.seqNo(); - return doGenerateSeqNoForOperation(operation); - } - - private long doGenerateSeqNoForOperation(final Engine.Operation operation) { - return localCheckpointTracker.generateSeqNo(); - } - - private void markSeqNoAsSeen(long seqNo) { - localCheckpointTracker.advanceMaxSeqNo(seqNo); - } - -} diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index bb06e76467e95..e74ed90363aad 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -29,6 +29,7 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.dataformat.commit.Committer; import org.opensearch.index.engine.dataformat.commit.CommitterFactory; import org.opensearch.index.mapper.DocumentMapperForType; import org.opensearch.index.mapper.MapperService; @@ -42,16 +43,15 @@ import org.opensearch.plugins.PluginsService; import org.opensearch.threadpool.ThreadPool; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; +import java.io.IOException; +import java.util.*; import java.util.function.BooleanSupplier; import java.util.function.LongSupplier; import java.util.function.Supplier; +import static org.opensearch.index.engine.Engine.HISTORY_UUID_KEY; +import static org.opensearch.index.translog.Translog.TRANSLOG_UUID_KEY; + /** * A factory to create an EngineConfig based on custom plugin overrides * @@ -139,9 +139,29 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti } if (committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatenabled())) { - throw new IllegalStateException("Multiple committer factories detected: " + committerFactories); + committerFactories.add(new CommitterFactory() { + @Override + public Committer getCommitter(Store store) { + return new Committer() { + @Override + public Map readLastCommittedUserData() { + try { + return store.readLastCommittedSegmentsInfo().getUserData(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public long getLastCommittedGeneration() { + return 0; + } + }; + } + }); } + final CodecService instance = codecService.orElse(null); this.codecServiceFactory = (instance != null) ? (config) -> instance : codecServiceFactory.orElse(null); this.translogDeletionPolicyFactory = translogDeletionPolicyFactory.orElse((idxs, rtls) -> null); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java index 9c28cf4cb825b..db492e40898c9 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatPlugin.java @@ -45,5 +45,5 @@ public interface DataFormatPlugin { * @param indexSettings the index settings * @return the indexing execution engine instance */ - IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings); + IndexingExecutionEngine indexingEngine(MapperService mapperService, ShardPath shardPath, IndexSettings indexSettings, DataFormatRegistry dataFormatRegistry); } diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java index 3df6560769fdb..e0790b09ce0ea 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/DataFormatRegistry.java @@ -71,16 +71,6 @@ public DataFormatRegistry(PluginsService pluginsService) { } } - if (!readerManagerBuilders.keySet().equals(dataFormatPlugiRegistry.keySet())) { - throw new IllegalStateException( - "Cannot build registry as data formats have missing indexing engine/reader managers" - + " - formats with reader managers: " - + readerManagerBuilders.keySet() - + ", formats with plugins: " - + dataFormatPlugiRegistry.keySet() - ); - } - this.dataFormatPluginRegistry = Map.copyOf(dataFormatPlugiRegistry); this.dataFormats = Map.copyOf(dataFormats); this.readerManagerBuilders = Map.copyOf(readerManagerBuilders); @@ -106,7 +96,7 @@ public DataFormatRegistry(PluginsService pluginsService) { if (plugin == null) { throw new IllegalArgumentException("No plugin registered for DataFormat [" + format.name() + "]"); } - return plugin.indexingEngine(mapperService, shardPath, indexSettings); + return plugin.indexingEngine(mapperService, shardPath, indexSettings, this); } public DataFormat format(String name) { diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java b/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java index 7892b6ab746ce..833701aa937c1 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/RefreshInput.java @@ -10,7 +10,6 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.exec.Segment; -import org.opensearch.index.engine.exec.WriterFileSet; import java.util.ArrayList; import java.util.List; @@ -62,7 +61,7 @@ public Builder existingSegments(List existingSegments) { /** * Adds a writer file set. * - * @param writerFileSet the writer file set to add + * @param segment the segment set to add * @return this builder */ public Builder addSegment(Segment segment) { diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java index ec913ea3e4f4f..dc53c69abf80d 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/Committer.java @@ -1,7 +1,10 @@ package org.opensearch.index.engine.dataformat.commit; +import org.opensearch.common.annotation.ExperimentalApi; + import java.util.Map; +@ExperimentalApi public interface Committer { Map readLastCommittedUserData(); diff --git a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java index d4a126ef0d092..4c6e2fe044e84 100644 --- a/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/dataformat/commit/CommitterFactory.java @@ -1,7 +1,9 @@ package org.opensearch.index.engine.dataformat.commit; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.store.Store; +@ExperimentalApi public interface CommitterFactory { Committer getCommitter(Store store); } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java index 6733eb5b65e50..946a01585a7a2 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/DataFormatAwareIndexerFactory.java @@ -1,12 +1,13 @@ package org.opensearch.index.engine.exec; -import org.opensearch.index.engine.DataFormatBasedEngine; +import org.opensearch.index.engine.DataFormatAwareEngine; import org.opensearch.index.engine.EngineConfig; + public class DataFormatAwareIndexerFactory implements IndexerFactory { @Override public Indexer createIndexer(EngineConfig config) { - return new DataFormatBasedEngine(config); + return new DataFormatAwareEngine(config); } } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/EngineReaderManager.java b/server/src/main/java/org/opensearch/index/engine/exec/EngineReaderManager.java index b7d412850bd3d..682925c33e835 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/EngineReaderManager.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/EngineReaderManager.java @@ -11,6 +11,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; +import java.io.Closeable; import java.io.IOException; /** @@ -23,6 +24,6 @@ * @opensearch.experimental */ @ExperimentalApi -public interface EngineReaderManager extends CatalogSnapshotLifecycleListener, FilesListener { +public interface EngineReaderManager extends CatalogSnapshotLifecycleListener, FilesListener, Closeable { T getReader(CatalogSnapshot catalogSnapshot) throws IOException; } diff --git a/server/src/main/java/org/opensearch/index/engine/exec/IndexFileDeleter.java b/server/src/main/java/org/opensearch/index/engine/exec/IndexFileDeleter.java index 214402e4064b5..50fe6a62f0595 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/IndexFileDeleter.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/IndexFileDeleter.java @@ -9,7 +9,6 @@ package org.opensearch.index.engine.exec; import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.index.engine.DataFormatAwareEngine; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.exec.coord.CatalogSnapshot; import org.opensearch.index.shard.ShardPath; @@ -28,8 +27,7 @@ * added or fully dereferenced after catalog snapshot changes. *

* This class does not notify reader managers itself — it returns the - * computed change sets so the caller ({@link DataFormatAwareEngine}) - * can route notifications to the appropriate reader managers. + * computed change sets so the caller can route notifications to the appropriate reader managers. * * @opensearch.experimental */ diff --git a/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java b/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java index 1ffac94d25cdc..657cd94bd66f7 100644 --- a/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/exec/IndexerFactory.java @@ -6,6 +6,7 @@ import java.util.function.Supplier; @ExperimentalApi +@FunctionalInterface public interface IndexerFactory { Indexer createIndexer(EngineConfig config); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 61fe489119faa..532e7cf847857 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -129,7 +129,6 @@ import org.opensearch.index.cache.request.ShardRequestCache; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.CommitStats; -import org.opensearch.index.engine.DataFormatAwareEngine; import org.opensearch.index.engine.Engine; import org.opensearch.index.engine.Engine.GetResult; import org.opensearch.index.engine.EngineBackedIndexer; @@ -145,6 +144,7 @@ import org.opensearch.index.engine.Segment; import org.opensearch.index.engine.SegmentsStats; import org.opensearch.index.engine.dataformat.DataFormatRegistry; +import org.opensearch.index.engine.exec.IndexReaderProvider; import org.opensearch.index.engine.exec.Indexer; import org.opensearch.index.engine.exec.IndexerFactory; import org.opensearch.index.fielddata.FieldDataStats; @@ -319,7 +319,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl private volatile long pendingPrimaryTerm; // see JavaDocs for getPendingPrimaryTerm private final Object engineMutex = new Object(); // lock ordering: engineMutex -> mutex private final AtomicReference currentEngineReference = new AtomicReference<>(); - private final AtomicReference currentCompositeEngineReference = new AtomicReference<>(); final IndexerFactory indexerFactory; final EngineConfigFactory engineConfigFactory; @@ -607,12 +606,6 @@ public boolean shouldCache(Query query) { } } this.dataFormatRegistry = dataFormatRegistry; - if (dataFormatRegistry != null) { - // TODO: This should go away and we should use indexer directly. - this.currentCompositeEngineReference.set( - new DataFormatAwareEngine(dataFormatRegistry.getReaderManagers(mapperService, indexSettings, path)) - ); - } } /** @@ -2248,20 +2241,6 @@ public Engine.Searcher acquireSearcher(String source) { return acquireSearcher(source, Engine.SearcherScope.EXTERNAL); } - /** - * Returns the current CompositeEngine, or null if no optimized index is active. - */ - public DataFormatAwareEngine getCompositeEngine() { - return currentCompositeEngineReference.get(); - } - - /** - * Sets the CompositeEngine for this shard (called during shard initialization for optimized indexes). - */ - public void setCompositeEngine(DataFormatAwareEngine dataFormatAwareEngine) { - currentCompositeEngineReference.set(dataFormatAwareEngine); - } - private void markSearcherAccessed() { lastSearcherAccess.lazySet(threadPool.relativeTimeInMillis()); } @@ -5965,6 +5944,10 @@ public ShardIngestionState getIngestionState() { return ingestionEngine.getIngestionState(); } + public IndexReaderProvider getReaderProvider() { + return getIndexer(); + } + /** * Async shard refresh task for running refreshes at shard level independently */ diff --git a/server/src/main/java/org/opensearch/index/shard/ShardPath.java b/server/src/main/java/org/opensearch/index/shard/ShardPath.java index 911bfec94e190..e2cf2257e813a 100644 --- a/server/src/main/java/org/opensearch/index/shard/ShardPath.java +++ b/server/src/main/java/org/opensearch/index/shard/ShardPath.java @@ -93,6 +93,10 @@ public Path resolveIndex() { return path.resolve(INDEX_FOLDER_NAME); } + public Path resolve(String subDirectory) { + return path.resolve(subDirectory); + } + public Path getDataPath() { return path; } diff --git a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java index 4c5191ac1af2c..4221e7b3ee886 100644 --- a/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java +++ b/server/src/test/java/org/opensearch/index/engine/dataformat/DataFormatPluginTests.java @@ -14,7 +14,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.index.shard.ShardId; import org.opensearch.index.IndexSettings; -import org.opensearch.index.engine.DataFormatAwareEngine; import org.opensearch.index.engine.dataformat.stub.MockCatalogSnapshot; import org.opensearch.index.engine.dataformat.stub.MockDataFormat; import org.opensearch.index.engine.dataformat.stub.MockDataFormatPlugin; From a04c8a96142176e87450f3ad35c33c6c81752f13 Mon Sep 17 00:00:00 2001 From: Mohit Godwani Date: Fri, 10 Apr 2026 11:11:51 +0530 Subject: [PATCH 3/4] Fix post rebase Signed-off-by: Mohit Godwani --- .../gradle/testclusters/RunTask.java | 1 + .../nativebridge/spi/NativeLibraryLoader.java | 1 + .../index/engine/EngineConfigFactory.java | 29 ++++++++----------- .../opensearch/indices/IndicesService.java | 2 +- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java index c5035f3b082fe..f13340b18ef8a 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/RunTask.java @@ -192,6 +192,7 @@ public void beforeStart() { node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=" + debugPort); debugPort += 1; } + node.jvmArgs("--enable-native-access=ALL-UNNAMED"); if (keystorePassword.length() > 0) { node.keystorePassword(keystorePassword); } diff --git a/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeLibraryLoader.java b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeLibraryLoader.java index 16e22da13ab1a..84418d292383c 100644 --- a/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeLibraryLoader.java +++ b/sandbox/libs/dataformat-native/src/main/java/org/opensearch/nativebridge/spi/NativeLibraryLoader.java @@ -168,3 +168,4 @@ private static SymbolLookup loadLibrary() { throw new RuntimeException("Failed to load native library '" + LIBRARY_NAME + "'"); } } + diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index e74ed90363aad..87a10c38ea32a 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -138,25 +138,20 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti ); } - if (committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatenabled())) { - committerFactories.add(new CommitterFactory() { + if (committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatEnabled())) { + committerFactories.add(store -> new Committer() { @Override - public Committer getCommitter(Store store) { - return new Committer() { - @Override - public Map readLastCommittedUserData() { - try { - return store.readLastCommittedSegmentsInfo().getUserData(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + public Map readLastCommittedUserData() { + try { + return store.readLastCommittedSegmentsInfo().getUserData(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - @Override - public long getLastCommittedGeneration() { - return 0; - } - }; + @Override + public long getLastCommittedGeneration() { + return 0; } }); } diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index fd82d3edecc35..ea838a57cfd14 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -1168,7 +1168,7 @@ private IngestionConsumerFactory getIngestionConsumerFactory(final IndexSettings } private IndexerFactory getIndexerFactory(final IndexSettings idxSettings) { - if (idxSettings.isPluggableDataFormatenabled()) { + if (idxSettings.isPluggableDataFormatEnabled()) { return new DataFormatAwareIndexerFactory(); } else { return new EngineBackedIndexerFactory(getEngineFactory(idxSettings)); From 1e7fb4ba1363905ba0090e8a2527f13e2cc9207b Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Fri, 10 Apr 2026 22:27:26 +0530 Subject: [PATCH 4/4] Introduce basic IT and fix up tests Signed-off-by: Bukhtawar Khan --- sandbox/plugins/composite-engine/build.gradle | 22 ++++++ .../composite/CompositeParquetIndexIT.java | 72 +++++++++++++++++++ .../CompositeIndexingExecutionEngine.java | 14 ++++ .../composite/CompositeEnginePluginTests.java | 3 +- ...CompositeIndexingExecutionEngineTests.java | 23 +----- .../composite/CompositeTestHelper.java | 61 ++++++++++------ .../index/engine/EngineConfigFactory.java | 2 +- ...enSearchIndexLevelReplicationTestCase.java | 9 +-- .../index/shard/IndexShardTestCase.java | 9 ++- 9 files changed, 164 insertions(+), 51 deletions(-) create mode 100644 sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java diff --git a/sandbox/plugins/composite-engine/build.gradle b/sandbox/plugins/composite-engine/build.gradle index 1547c0c33f445..9f622c53f9a5f 100644 --- a/sandbox/plugins/composite-engine/build.gradle +++ b/sandbox/plugins/composite-engine/build.gradle @@ -11,9 +11,31 @@ opensearchplugin { classname = 'org.opensearch.composite.CompositeEnginePlugin' } +apply plugin: 'opensearch.internal-cluster-test' + +// Test compilation needs JDK 25 to consume parquet-data-format (which targets JDK 25) +tasks.named('compileTestJava').configure { + sourceCompatibility = JavaVersion.toVersion(25) + targetCompatibility = JavaVersion.toVersion(25) +} + +tasks.named('compileInternalClusterTestJava').configure { + sourceCompatibility = JavaVersion.toVersion(25) + targetCompatibility = JavaVersion.toVersion(25) +} + +tasks.named('internalClusterTest').configure { + jvmArgs += [ + '--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED', + '--enable-native-access=ALL-UNNAMED' + ] +} + dependencies { api project(':libs:opensearch-concurrent-queue') api project(':sandbox:libs:composite-common') compileOnly project(':server') testImplementation project(':test:framework') + testImplementation project(':sandbox:plugins:parquet-data-format') + internalClusterTestImplementation project(':sandbox:plugins:parquet-data-format') } diff --git a/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java new file mode 100644 index 0000000000000..86a6652545e1b --- /dev/null +++ b/sandbox/plugins/composite-engine/src/internalClusterTest/java/org/opensearch/composite/CompositeParquetIndexIT.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.composite; + +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.parquet.ParquetDataFormatPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Integration test that validates a composite index with parquet as the primary data format + * can be created and its settings are correctly persisted. + * + * Requires JDK 25 and sandbox enabled. Run with: + * ./gradlew :sandbox:plugins:composite-engine:test \ + * --tests "*.CompositeParquetIndexIT" \ + * -Dsandbox.enabled=true + */ +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 1) +public class CompositeParquetIndexIT extends OpenSearchIntegTestCase { + + private static final String INDEX_NAME = "test-composite-parquet"; + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ParquetDataFormatPlugin.class, CompositeEnginePlugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(FeatureFlags.PLUGGABLE_DATAFORMAT_EXPERIMENTAL_FLAG, true) + .build(); + } + + public void testCreateCompositeParquetIndex() { + Settings indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.pluggable.dataformat.enabled", true) + .put("index.composite.primary_data_format", "parquet") + .putList("index.composite.secondary_data_formats") + .build(); + + CreateIndexResponse response = client().admin().indices().prepareCreate(INDEX_NAME).setSettings(indexSettings).get(); + assertTrue("Index creation should be acknowledged", response.isAcknowledged()); + + GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings(INDEX_NAME).get(); + Settings actual = settingsResponse.getIndexToSettings().get(INDEX_NAME); + + assertEquals("1", actual.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)); + assertEquals("0", actual.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); + assertEquals("true", actual.get("index.pluggable.dataformat.enabled")); + assertEquals("parquet", actual.get("index.composite.primary_data_format")); + + ensureGreen(INDEX_NAME); + } +} diff --git a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java index b68d39ba28327..c968b18ef4f01 100644 --- a/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java +++ b/sandbox/plugins/composite-engine/src/main/java/org/opensearch/composite/CompositeIndexingExecutionEngine.java @@ -100,6 +100,20 @@ public CompositeIndexingExecutionEngine( this.compositeDataFormat = new CompositeDataFormat(allFormats); } + /** + * Package-private constructor for testing. Allows direct injection of engines + * without requiring a DataFormatRegistry. + */ + CompositeIndexingExecutionEngine( + IndexingExecutionEngine primaryEngine, + Set> secondaryEngines, + CompositeDataFormat compositeDataFormat + ) { + this.primaryEngine = primaryEngine; + this.secondaryEngines = secondaryEngines; + this.compositeDataFormat = compositeDataFormat; + } + /** * Validates that the primary and all secondary data format plugins are registered. * diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeEnginePluginTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeEnginePluginTests.java index c43fc891e3279..7e9b85b5a4f25 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeEnginePluginTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeEnginePluginTests.java @@ -104,7 +104,8 @@ public DataFormat getDataFormat() { public org.opensearch.index.engine.dataformat.IndexingExecutionEngine indexingEngine( org.opensearch.index.mapper.MapperService mapperService, org.opensearch.index.shard.ShardPath shardPath, - IndexSettings indexSettings + IndexSettings indexSettings, + org.opensearch.index.engine.dataformat.DataFormatRegistry dataFormatRegistry ) { return null; } diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java index fb6bc8f08c0f2..4d47384ca9843 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeIndexingExecutionEngineTests.java @@ -48,10 +48,9 @@ public void testConstructorThrowsWhenPrimaryFormatNotRegistered() { Map plugins = new HashMap<>(); plugins.put("lucene", CompositeTestHelper.stubPlugin("lucene", 1)); - IndexSettings indexSettings = createIndexSettings("parquet"); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new CompositeIndexingExecutionEngine(plugins, indexSettings, null, null) + () -> CompositeIndexingExecutionEngine.validateFormatsRegistered(plugins, "parquet", List.of()) ); assertTrue(ex.getMessage().contains("parquet")); } @@ -60,31 +59,15 @@ public void testConstructorThrowsWhenSecondaryFormatNotRegistered() { Map plugins = new HashMap<>(); plugins.put("lucene", CompositeTestHelper.stubPlugin("lucene", 1)); - Settings settings = Settings.builder() - .put("index.composite.primary_data_format", "lucene") - .putList("index.composite.secondary_data_formats", "parquet") - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build(); - IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> new CompositeIndexingExecutionEngine(plugins, indexSettings, null, null) + () -> CompositeIndexingExecutionEngine.validateFormatsRegistered(plugins, "lucene", List.of("parquet")) ); assertTrue(ex.getMessage().contains("parquet")); } - public void testConstructorRejectsNullDataFormatPlugins() { - IndexSettings indexSettings = createIndexSettings("lucene"); - expectThrows(NullPointerException.class, () -> new CompositeIndexingExecutionEngine(null, indexSettings, null, null)); - } - public void testConstructorRejectsNullIndexSettings() { - Map plugins = Map.of("lucene", CompositeTestHelper.stubPlugin("lucene", 1)); - expectThrows(NullPointerException.class, () -> new CompositeIndexingExecutionEngine(plugins, null, null, null)); + expectThrows(NullPointerException.class, () -> new CompositeIndexingExecutionEngine(null, null, null, null)); } public void testValidateFormatsRegisteredAcceptsValidConfig() { diff --git a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java index af83555e7b7b3..423bb74a0c481 100644 --- a/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java +++ b/sandbox/plugins/composite-engine/src/test/java/org/opensearch/composite/CompositeTestHelper.java @@ -8,12 +8,10 @@ package org.opensearch.composite; -import org.opensearch.Version; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.dataformat.DataFormat; import org.opensearch.index.engine.dataformat.DataFormatPlugin; +import org.opensearch.index.engine.dataformat.DataFormatRegistry; import org.opensearch.index.engine.dataformat.DocumentInput; import org.opensearch.index.engine.dataformat.FieldTypeCapabilities; import org.opensearch.index.engine.dataformat.FileInfos; @@ -26,9 +24,11 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.shard.ShardPath; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -44,27 +44,21 @@ private CompositeTestHelper() {} * Creates a CompositeIndexingExecutionEngine with stub engines for testing. */ static CompositeIndexingExecutionEngine createStubEngine(String primaryName, String... secondaryNames) { - Map plugins = new HashMap<>(); - plugins.put(primaryName, stubPlugin(primaryName, 1)); - for (String name : secondaryNames) { - plugins.put(name, stubPlugin(name, 2)); - } + DataFormat primaryFormat = stubFormat(primaryName, 1, Set.of()); + IndexingExecutionEngine primaryEngine = new StubIndexingExecutionEngine(primaryFormat); - Settings.Builder settingsBuilder = Settings.builder() - .put("index.composite.primary_data_format", primaryName) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + List allFormats = new ArrayList<>(); + allFormats.add(primaryFormat); - if (secondaryNames.length > 0) { - settingsBuilder.putList("index.composite.secondary_data_formats", secondaryNames); + Set> secondaryEngines = new LinkedHashSet<>(); + for (String name : secondaryNames) { + DataFormat secondaryFormat = stubFormat(name, 2, Set.of()); + secondaryEngines.add(new StubIndexingExecutionEngine(secondaryFormat)); + allFormats.add(secondaryFormat); } - Settings settings = settingsBuilder.build(); - IndexMetadata indexMetadata = IndexMetadata.builder("test-index").settings(settings).build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); - - return new CompositeIndexingExecutionEngine(plugins, indexSettings, null, null); + CompositeDataFormat compositeDataFormat = new CompositeDataFormat(allFormats); + return new CompositeIndexingExecutionEngine(primaryEngine, secondaryEngines, compositeDataFormat); } static DataFormatPlugin stubPlugin(String formatName, long priority) { @@ -79,7 +73,8 @@ public DataFormat getDataFormat() { public IndexingExecutionEngine indexingEngine( MapperService mapperService, ShardPath shardPath, - IndexSettings indexSettings + IndexSettings indexSettings, + DataFormatRegistry dataFormatRegistry ) { return new StubIndexingExecutionEngine(format); } @@ -98,7 +93,8 @@ public DataFormat getDataFormat() { public IndexingExecutionEngine indexingEngine( MapperService mapperService, ShardPath shardPath, - IndexSettings indexSettings + IndexSettings indexSettings, + DataFormatRegistry dataFormatRegistry ) { return new StubIndexingExecutionEngine(format); } @@ -173,6 +169,9 @@ public long getNextWriterGeneration() { public DocumentInput newDocumentInput() { return new StubDocumentInput(); } + + @Override + public void close() {} } /** @@ -206,6 +205,22 @@ public void sync() {} @Override public void close() {} + + @Override + public long generation() { + return 0L; + } + + @Override + public void lock() {} + + @Override + public boolean tryLock() { + return true; + } + + @Override + public void unlock() {} } /** diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index 87a10c38ea32a..a05fefa9eed0e 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -138,7 +138,7 @@ public EngineConfigFactory(PluginsService pluginsService, IndexSettings idxSetti ); } - if (committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatEnabled())) { + if (committerFactories.isEmpty() || committerFactories.size() > 1 || (committerFactories.size() != 1 && idxSettings.isPluggableDataFormatEnabled())) { committerFactories.add(store -> new Committer() { @Override public Map readLastCommittedUserData() { diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index 5897171b383e8..35a6a71b582b5 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -89,6 +89,7 @@ import org.opensearch.index.engine.EngineConfigFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.index.engine.InternalEngineFactory; +import org.opensearch.index.engine.exec.EngineBackedIndexerFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.seqno.GlobalCheckpointSyncAction; import org.opensearch.index.seqno.RetentionLease; @@ -301,7 +302,7 @@ protected ReplicationGroup( primaryRouting, indexMetadata, null, - getEngineFactory(primaryRouting), + new EngineBackedIndexerFactory(getEngineFactory(primaryRouting)), () -> {}, retentionLeaseSyncer, recoverySettings, @@ -460,7 +461,7 @@ public IndexShard addReplica(Path remotePath) throws IOException { replicaRouting, indexMetadata, null, - getEngineFactory(replicaRouting), + new EngineBackedIndexerFactory(getEngineFactory(replicaRouting)), () -> {}, retentionLeaseSyncer, remotePath @@ -475,7 +476,7 @@ public IndexShard addReplica(Path remotePath, RecoverySettings recoverySettings) replicaRouting, indexMetadata, null, - getEngineFactory(replicaRouting), + new EngineBackedIndexerFactory(getEngineFactory(replicaRouting)), () -> {}, retentionLeaseSyncer, recoverySettings, @@ -517,7 +518,7 @@ public synchronized IndexShard addReplicaWithExistingPath(final ShardPath shardP indexMetadata, null, null, - getEngineFactory(shardRouting), + new EngineBackedIndexerFactory(getEngineFactory(shardRouting)), getEngineConfigFactory(new IndexSettings(indexMetadata, indexMetadata.getSettings())), () -> {}, retentionLeaseSyncer, diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 822f8d0538fde..dc05426767005 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -1558,10 +1558,15 @@ protected String snapshotShard(final IndexShard shard, final Snapshot snapshot, } /** - * Helper method to access (package-protected) engine from tests + * Helper method to access (package-protected) engine from tests. + * Returns null if the indexer is not engine-backed (e.g., DataFormatAwareEngine). */ public static Engine getEngine(IndexShard indexShard) { - return ((EngineBackedIndexer) indexShard.getIndexer()).getEngine(); + Indexer indexer = indexShard.getIndexer(); + if (indexer instanceof EngineBackedIndexer engineBackedIndexer) { + return engineBackedIndexer.getEngine(); + } + return null; } public static Indexer getIndexer(IndexShard indexShard) {