Test active load file group isolation

Author: Caideyipi

integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java

@@ -20,8 +20,11 @@
 package org.apache.iotdb.db.it;
 
 import org.apache.iotdb.commons.auth.entity.PrivilegeType;
+import org.apache.iotdb.commons.path.MeasurementPath;
 import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
 import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
@@ -985,6 +988,62 @@ public void testLoadWithEmptyTsFile() throws Exception {
     }
   }
 
+  @Test
+  public void testAsyncLoadKeepsSameNamedTsFilesWithModsIsolated() throws Exception {
+    registerSchema();
+
+    long expectedPointCount = 0;
+    // Before each file group had its own transfer directory, these same-named TsFiles and mods
+    // were renamed independently in one shared directory and could be paired with the wrong file.
+    for (int i = 0; i < 2; i++) {
+      final File sourceDir = new File(tmpDir, "source-" + i);
+      Assert.assertTrue(sourceDir.mkdirs());
+      try (final TsFileGenerator generator =
+          new TsFileGenerator(new File(sourceDir, "1-0-0-0.tsfile"))) {
+        generator.resetRandom(i);
+        generator.registerTimeseries(
+            SchemaConfig.DEVICE_0, Collections.singletonList(SchemaConfig.MEASUREMENT_00));
+        generator.generateData(SchemaConfig.DEVICE_0, 20, 1, false, TimeUnit.SECONDS.toMillis(i));
+        // Each group contributes 20 points and its own mods deletes exactly one. Losing either
+        // TsFile-to-mods pairing therefore leaves 39 points instead of the expected 38.
+        expectedPointCount += generator.getTotalNumber() - 1;
+      }
+      try (final ModificationFile modificationFile =
+          new ModificationFile(
+              new File(sourceDir, "1-0-0-0.tsfile" + ModificationFile.FILE_SUFFIX), false)) {
+        modificationFile.write(
+            new TreeDeletionEntry(
+                new MeasurementPath(
+                    SchemaConfig.DEVICE_0, SchemaConfig.MEASUREMENT_00.getMeasurementName()),
+                TimeUnit.SECONDS.toMillis(i) + 1,
+                TimeUnit.SECONDS.toMillis(i) + 1));
+      }
+    }
+
+    try (final Connection connection = EnvFactory.getEnv().getConnection();
+        final Statement statement = connection.createStatement()) {
+      statement.execute(
+          String.format(
+              "load \"%s\" with ('async'='true','database-level'='2','on-success'='delete')",
+              tmpDir.getAbsolutePath()));
+    }
+
+    TestUtils.assertDataEventuallyOnEnv(
+        EnvFactory.getEnv(),
+        "select count("
+            + SchemaConfig.MEASUREMENT_00.getMeasurementName()
+            + ") from "
+            + SchemaConfig.DEVICE_0,
+        Collections.singletonMap(
+            "count("
+                + SchemaConfig.DEVICE_0
+                + "."
+                + SchemaConfig.MEASUREMENT_00.getMeasurementName()
+                + ")",
+            Long.toString(expectedPointCount)),
+        30);
+  }
+
   @Test
   public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
     try (final TsFileGenerator generator =

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/load/LoadTsFileNodeTest.java

@@ -97,6 +97,8 @@ public void testLoadTsFilePieceNode() {
   public void testCleanContinuesAfterOneFileCannotBeDeleted() throws Exception {
     final File tempDir = Files.createTempDirectory("load-node-clean").toFile();
     try {
+      // A non-empty directory at the TsFile path makes that deletion fail deterministically. The
+      // companion cleanup must still continue instead of sharing the same try-catch block.
       final File tsFile = new File(tempDir, "1-0-0-0.tsfile");
       Assert.assertTrue(tsFile.mkdirs());
       Assert.assertTrue(new File(tsFile, "non-empty").createNewFile());

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileSchedulerTest.java

@@ -142,6 +142,8 @@ public void testTsFileDataManagerClearReleasesCachedMemory() throws Exception {
         dataManagerConstructor.newInstance(
             mock(LoadTsFileScheduler.class), mock(LoadSingleTsFileNode.class), memoryBlock);
 
+    // Simulate data buffered before split or routing aborts. clear() is the last chance to return
+    // this accounting to the shared LOAD memory block.
     final long cachedMemorySize = 128L;
     memoryBlock.addMemoryUsage(cachedMemorySize);
     final Field dataSizeField = dataManagerClass.getDeclaredField("dataSize");

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadDirScannerTest.java

@@ -80,6 +80,8 @@ public void tearDown() {
 
   @Test
   public void testScanDeduplicatesTsFileAndCompanionFiles() throws Exception {
+    // The recursive scanner sees all four paths, but every companion maps back to the same TsFile.
+    // Enqueuing each path separately used to consume queue capacity and schedule duplicate loads.
     final File tsFile = createCompletedTsFile(pendingDir, "1-0-0-0.tsfile");
     Assert.assertTrue(
         new File(tsFile.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile());
@@ -106,6 +108,8 @@ public void testScanDeduplicatesTsFileAndCompanionFiles() throws Exception {
 
   @Test
   public void testAttributeAndTransferDirectoriesDoNotImplyTableModel() throws Exception {
+    // Async tree loads add attribute and per-handoff transfer directories below pending. These are
+    // internal directories, not table database names inferred from a user-created subdirectory.
     final Map<String, String> attributes =
         ActiveLoadPathHelper.buildAttributes(null, 2, false, false, null, false, "test-user");
     final File attributeDir = ActiveLoadPathHelper.resolveTargetDir(pendingDir, attributes);

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoaderTest.java

@@ -119,6 +119,8 @@ public void testStopClearsPendingFilesForRestart() throws Exception {
     Assert.assertTrue(pendingQueue.enqueue(tsFilePath, tempDir.getAbsolutePath(), false, false));
     Assert.assertTrue(loader.isFilePendingOrLoading(new File(tsFilePath)));
 
+    // A loader restart reuses this queue. Stale pending membership otherwise makes the scanner
+    // believe the on-disk TsFile is already scheduled and it will never enqueue it again.
     loader.stop();
 
     Assert.assertFalse(loader.isFilePendingOrLoading(new File(tsFilePath)));

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/util/LoadUtilTest.java

@@ -59,6 +59,8 @@ public void tearDown() {
   public void testTransferFilesKeepsCompanionsTogetherAndDeletesSourcesAfterHandoff()
       throws Exception {
     final List<File> sourceFiles = createTsFileAndCompanions();
+    // Existing same-named files represent a previous handoff. A new group must be isolated rather
+    // than independently renaming the TsFile and companions in this shared target directory.
     for (final File sourceFile : sourceFiles) {
       Files.write(
           new File(targetDir, sourceFile.getName()).toPath(),
@@ -86,6 +88,8 @@ public void testTransferFilesKeepsCompanionsTogetherAndDeletesSourcesAfterHandof
   @Test
   public void testTransferFailureDoesNotDeleteSources() throws Exception {
     final List<File> sourceFiles = createTsFileAndCompanions();
+    // A regular file cannot contain the temporary transfer directory, forcing handoff to fail
+    // before ownership of any source file can be released.
     final File invalidTargetDir = new File(tempDir, "target-file");
     Assert.assertTrue(invalidTargetDir.createNewFile());