From 7efbdb2bca3e113c9fb8485ee4bbfd49cd86aee0 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 31 Mar 2022 18:02:42 +0800 Subject: [PATCH 01/22] reconstruct compaction package --- .../CompactionExceptionHandler.java | 3 +-- .../{task => }/CompactionRecoverManager.java | 9 ++++---- .../compaction/CompactionTaskManager.java | 1 + .../task/RewriteCrossSpaceCompactionTask.java | 8 +++---- .../SizeTieredCompactionSelector.java | 9 ++++---- .../sizetiered/SizeTieredCompactionTask.java | 4 ++-- .../log/CompactionLogAnalyzer.java | 21 +++++++++---------- .../{utils => }/log/CompactionLogger.java | 3 +-- .../{ => log}/TsFileIdentifier.java | 2 +- .../task/CompactionRecoverTask.java | 6 +++--- .../{ => task}/CompactionTaskComparator.java | 3 +-- .../VirtualStorageGroupProcessor.java | 2 +- .../CompactionTaskComparatorTest.java | 1 + .../engine/compaction/TsFileIdentifierUT.java | 3 ++- .../CrossSpaceCompactionExceptionTest.java | 8 +++---- ...ewriteCrossSpaceCompactionRecoverTest.java | 6 +++--- .../inner/InnerCompactionLogTest.java | 2 +- .../InnerSpaceCompactionExceptionTest.java | 4 ++-- .../InnerSpaceCompactionUtilsOldTest.java | 2 +- .../SizeTieredCompactionRecoverTest.java | 8 +++---- ...sSpaceCompactionRecoverCompatibleTest.java | 2 +- ...TieredCompactionRecoverCompatibleTest.java | 2 +- .../SizeTieredCompactionRecoverTest.java | 6 +++--- .../utils/CompactionClearUtils.java | 2 +- 24 files changed, 59 insertions(+), 58 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{task => }/CompactionExceptionHandler.java (98%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{task => }/CompactionRecoverManager.java (91%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{utils => }/log/CompactionLogAnalyzer.java (84%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{utils => }/log/CompactionLogger.java (96%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{ => log}/TsFileIdentifier.java (99%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{ => task}/CompactionTaskComparator.java (97%) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionExceptionHandler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionExceptionHandler.java similarity index 98% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionExceptionHandler.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionExceptionHandler.java index 91cca7b6216fe..3523394333eee 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionExceptionHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionExceptionHandler.java @@ -17,11 +17,10 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.task; +package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionRecoverManager.java similarity index 91% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverManager.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionRecoverManager.java index 9d14c0482e8c2..26a583879cdb1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionRecoverManager.java @@ -16,11 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.task; +package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.directories.DirectoryManager; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; @@ -31,8 +32,8 @@ import java.util.List; import java.util.regex.Pattern; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.CROSS_COMPACTION_LOG_NAME_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD; /** * CompactionRecoverManager searches compaction log and call {@link CompactionRecoverTask} to diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java index b0ecbe53f7dde..591948175dd17 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionTaskStatus; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.utils.datastructure.FixedPriorityBlockingQueue; import com.google.common.util.concurrent.RateLimiter; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index 9c1a1ac5a8669..74f88a3190cbe 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -19,11 +19,11 @@ package org.apache.iotdb.db.engine.compaction.cross.rewrite.task; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -44,8 +44,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { private static final Logger logger = diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 183e8daf1acf5..354e3296589a1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTaskFactory; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -45,10 +46,10 @@ * selector traverses the file list from old to new. If the size of selected files or the number of * select files exceed given threshold, a compaction task will be submitted to task queue in * CompactionTaskManager. In CompactionTaskManager, tasks are ordered by {@link - * org.apache.iotdb.db.engine.compaction.CompactionTaskComparator}. To maximize compaction - * efficiency, selector searches compaction task from 0 compaction files(that is, file that never - * been compacted, named level 0 file) to higher level files. If a compaction task is found in some - * level, selector will not search higher level anymore. + * CompactionTaskComparator}. To maximize compaction efficiency, selector searches compaction task + * from 0 compaction files(that is, file that never been compacted, named level 0 file) to higher + * level files. If a compaction task is found in some level, selector will not search higher level + * anymore. */ public class SizeTieredCompactionSelector extends AbstractInnerSpaceCompactionSelector { private static final Logger LOGGER = diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 6314eca040c33..6058b84a445c3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -19,12 +19,12 @@ package org.apache.iotdb.db.engine.compaction.inner.sizetiered; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogAnalyzer.java similarity index 84% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogAnalyzer.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogAnalyzer.java index c827cf8a456db..62c293d09056d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogAnalyzer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogAnalyzer.java @@ -16,10 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.utils.log; +package org.apache.iotdb.db.engine.compaction.log; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.engine.compaction.TsFileIdentifier; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import java.io.BufferedReader; @@ -29,15 +28,15 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.SEQUENCE_NAME_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_MERGE_START_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SEQ_FILES_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_UNSEQ_FILES_FROM_OLD; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.UNSEQUENCE_NAME_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.SEQUENCE_NAME_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_MERGE_START_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SEQ_FILES_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_UNSEQ_FILES_FROM_OLD; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.UNSEQUENCE_NAME_FROM_OLD; public class CompactionLogAnalyzer { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogger.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogger.java similarity index 96% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogger.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogger.java index c22f82ddf3bcd..8f8204ade3386 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/utils/log/CompactionLogger.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/CompactionLogger.java @@ -17,9 +17,8 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.utils.log; +package org.apache.iotdb.db.engine.compaction.log; -import org.apache.iotdb.db.engine.compaction.TsFileIdentifier; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.io.BufferedWriter; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/TsFileIdentifier.java similarity index 99% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/log/TsFileIdentifier.java index 503eafc570a43..d003ed4598bbc 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/log/TsFileIdentifier.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction; +package org.apache.iotdb.db.engine.compaction.log; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java index a4f82eb11854a..cdb107ee5c03f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java @@ -21,9 +21,9 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.TsFileIdentifier; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogAnalyzer; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogAnalyzer; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.log.TsFileIdentifier; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparator.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java index f05f5c81053eb..2ca4b016afed3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparator.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java @@ -17,14 +17,13 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction; +package org.apache.iotdb.db.engine.compaction.task; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.Comparator; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java index 87d7b4ec143bb..cf89e54e00191 100755 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java @@ -28,9 +28,9 @@ import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; +import org.apache.iotdb.db.engine.compaction.CompactionRecoverManager; import org.apache.iotdb.db.engine.compaction.CompactionScheduler; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverManager; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.flush.CloseFileListener; import org.apache.iotdb.db.engine.flush.FlushListener; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 53b039f4ba467..21852246622b5 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java index 5824bf579063c..06cc8667f5ab0 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.log.TsFileIdentifier; import org.apache.iotdb.db.utils.FileUtils; import org.junit.Assert; @@ -31,7 +32,7 @@ import java.lang.reflect.Field; import java.nio.file.Files; -import static org.apache.iotdb.db.engine.compaction.TsFileIdentifier.INFO_SEPARATOR; +import static org.apache.iotdb.db.engine.compaction.log.TsFileIdentifier.INFO_SEPARATOR; public class TsFileIdentifierUT { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index e9ab9d8735471..06e19fbf5782a 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -21,11 +21,11 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.task.CompactionExceptionHandler; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -46,8 +46,8 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java index 736e6aa7629f7..327a15083967a 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.CompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -45,8 +45,8 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class RewriteCrossSpaceCompactionRecoverTest extends AbstractCompactionTest { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionLogTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionLogTest.java index bac5394f4d171..bb6a7b7b0af0d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionLogTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionLogTest.java @@ -35,7 +35,7 @@ import java.io.File; import java.io.IOException; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX; import static org.junit.Assert.assertFalse; public class InnerCompactionLogTest extends InnerCompactionTest { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java index 541e18309c086..da0b9a349da3d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -19,10 +19,10 @@ package org.apache.iotdb.db.engine.compaction.inner; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; -import org.apache.iotdb.db.engine.compaction.task.CompactionExceptionHandler; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java index 234cb889a9714..ee852e869224d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java index 277b916a342a0..4d1b240dfe4cd 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java @@ -24,10 +24,10 @@ import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -64,9 +64,9 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; import static org.junit.Assert.assertEquals; public class SizeTieredCompactionRecoverTest extends AbstractInnerSpaceCompactionTest { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java index 99357e64a033a..257122e9b4194 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/RewriteCrossSpaceCompactionRecoverCompatibleTest.java @@ -20,9 +20,9 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java index ad5877da52590..867f882a3ddfc 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java @@ -21,9 +21,9 @@ import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java index d1744ceea0a5d..6df79c7958f68 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java @@ -23,10 +23,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -61,8 +61,8 @@ import java.util.List; import java.util.Set; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger.STR_TARGET_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; public class SizeTieredCompactionRecoverTest { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionClearUtils.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionClearUtils.java index 0280c045ea86b..dc3bb0cca12f2 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionClearUtils.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionClearUtils.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.engine.cache.BloomFilterCache; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; From 7424fd6967690426c6aafdbec7e5d0b1be631dfd Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Thu, 31 Mar 2022 20:03:41 +0800 Subject: [PATCH 02/22] remove CompactionTaskFactory --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +- .../compaction/CompactionScheduler.java | 23 +-- .../CrossCompactionStrategy.java | 14 +- .../InnerCompactionStrategy.java | 10 +- .../AbstractCrossSpaceCompactionSelector.java | 5 +- .../CrossSpaceCompactionTaskFactory.java | 49 ------- .../RewriteCrossSpaceCompactionSelector.java | 24 +-- .../AbstractInnerSpaceCompactionSelector.java | 5 +- .../SizeTieredCompactionSelector.java | 28 ++-- .../utils/InnerSpaceCompactionUtils.java | 2 +- .../AbstractCompactionPerformer.java} | 37 ++--- .../ReadChunkCompactionPerformer.java | 26 ++++ .../ReadPointCompactionPerformer.java | 26 ++++ .../cross/CrossSpaceCompactionTest.java | 67 +++++---- .../inner/InnerCompactionSchedulerTest.java | 137 ------------------ .../FakedInnerSpaceCompactionTaskFactory.java | 50 ------- .../utils/CompactionConfigRestorer.java | 4 +- 18 files changed, 149 insertions(+), 366 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{cross => constant}/CrossCompactionStrategy.java (87%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{inner => constant}/InnerCompactionStrategy.java (90%) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{inner/InnerSpaceCompactionTaskFactory.java => performer/AbstractCompactionPerformer.java} (51%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java delete mode 100644 server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java delete mode 100644 server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTaskFactory.java diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 580d8f75521d9..5ff59c8ed939a 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -21,8 +21,8 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index e384b5c2b1e4b..d071e953e9ff1 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -22,8 +22,8 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; import org.apache.iotdb.db.exception.BadNodeUrlFormatException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.utils.DatetimeUtils; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index 62dc5f424e650..59a69207f945f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -22,9 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTaskFactory; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTaskFactory; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -51,22 +49,19 @@ public static void scheduleCompaction(TsFileManager tsFileManager, long timePart tsFileManager.getVirtualStorageGroup(), tsFileManager.getStorageGroupDir(), timePartition, - tsFileManager, - new CrossSpaceCompactionTaskFactory()); + tsFileManager); tryToSubmitInnerSpaceCompactionTask( tsFileManager.getStorageGroupName(), tsFileManager.getVirtualStorageGroup(), timePartition, tsFileManager, - true, - new InnerSpaceCompactionTaskFactory()); + true); tryToSubmitInnerSpaceCompactionTask( tsFileManager.getStorageGroupName(), tsFileManager.getVirtualStorageGroup(), timePartition, tsFileManager, - false, - new InnerSpaceCompactionTaskFactory()); + false); } public static void tryToSubmitInnerSpaceCompactionTask( @@ -74,8 +69,7 @@ public static void tryToSubmitInnerSpaceCompactionTask( String virtualStorageGroupName, long timePartition, TsFileManager tsFileManager, - boolean sequence, - InnerSpaceCompactionTaskFactory taskFactory) { + boolean sequence) { if ((!config.isEnableSeqSpaceCompaction() && sequence) || (!config.isEnableUnseqSpaceCompaction() && !sequence)) { return; @@ -89,8 +83,7 @@ public static void tryToSubmitInnerSpaceCompactionTask( virtualStorageGroupName, timePartition, tsFileManager, - sequence, - taskFactory); + sequence); innerSpaceCompactionSelector.selectAndSubmit(); } @@ -99,8 +92,7 @@ private static void tryToSubmitCrossSpaceCompactionTask( String virtualStorageGroupName, String storageGroupDir, long timePartition, - TsFileManager tsFileManager, - CrossSpaceCompactionTaskFactory taskFactory) { + TsFileManager tsFileManager) { if (!config.isEnableCrossSpaceCompaction()) { return; } @@ -112,8 +104,7 @@ private static void tryToSubmitCrossSpaceCompactionTask( virtualStorageGroupName, storageGroupDir, timePartition, - tsFileManager, - taskFactory); + tsFileManager); crossSpaceCompactionSelector.selectAndSubmit(); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java similarity index 87% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java index 05ae8acf5a960..c348ea6562c5a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java @@ -16,9 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.cross; +package org.apache.iotdb.db.engine.compaction.constant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; +import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -62,18 +64,12 @@ public AbstractCrossSpaceCompactionSelector getCompactionSelector( String virtualGroupId, String storageGroupDir, long timePartition, - TsFileManager tsFileManager, - CrossSpaceCompactionTaskFactory taskFactory) { + TsFileManager tsFileManager) { switch (this) { case REWRITE_COMPACTION: default: return new RewriteCrossSpaceCompactionSelector( - logicalStorageGroupName, - virtualGroupId, - storageGroupDir, - timePartition, - tsFileManager, - taskFactory); + logicalStorageGroupName, virtualGroupId, storageGroupDir, timePartition, tsFileManager); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java similarity index 90% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java index 35d8c7426cb11..5e766e0c041f7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java @@ -17,9 +17,11 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.inner; +package org.apache.iotdb.db.engine.compaction.constant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -63,8 +65,7 @@ public AbstractInnerSpaceCompactionSelector getCompactionSelector( String virtualStorageGroupName, long timePartition, TsFileManager tsFileManager, - boolean sequence, - InnerSpaceCompactionTaskFactory taskFactory) { + boolean sequence) { switch (this) { case SIZE_TIERED_COMPACTION: default: @@ -73,8 +74,7 @@ public AbstractInnerSpaceCompactionSelector getCompactionSelector( virtualStorageGroupName, timePartition, tsFileManager, - sequence, - taskFactory); + sequence); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java index 99e2c936e3257..743001161db80 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java @@ -32,22 +32,19 @@ public abstract class AbstractCrossSpaceCompactionSelector extends AbstractCompa protected TsFileManager tsFileManager; protected List sequenceFileList; protected List unsequenceFileList; - protected CrossSpaceCompactionTaskFactory taskFactory; public AbstractCrossSpaceCompactionSelector( String logicalStorageGroupName, String virtualGroupId, String storageGroupDir, long timePartition, - TsFileManager tsFileManager, - CrossSpaceCompactionTaskFactory taskFactory) { + TsFileManager tsFileManager) { this.logicalStorageGroupName = logicalStorageGroupName; this.virtualGroupId = virtualGroupId; this.storageGroupDir = storageGroupDir; this.timePartition = timePartition; this.sequenceFileList = tsFileManager.getSequenceListByTimePartition(timePartition); this.unsequenceFileList = tsFileManager.getUnsequenceListByTimePartition(timePartition); - this.taskFactory = taskFactory; this.tsFileManager = tsFileManager; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java deleted file mode 100644 index 3c98410ef37bf..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTaskFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.cross; - -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.List; - -public class CrossSpaceCompactionTaskFactory { - - public AbstractCompactionTask createTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartitionId, - TsFileManager tsFileManager, - List selectedSeqTsFileResourceList, - List selectedUnSeqTsFileResourceList) { - return IoTDBDescriptor.getInstance() - .getConfig() - .getCrossCompactionStrategy() - .getCompactionTask( - logicalStorageGroupName, - virtualStorageGroupName, - timePartitionId, - tsFileManager, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList); - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 05423ab53223f..2575d51d8785e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -23,7 +23,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTaskFactory; import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; @@ -50,15 +49,13 @@ public RewriteCrossSpaceCompactionSelector( String virtualStorageGroupId, String storageGroupDir, long timePartition, - TsFileManager tsFileManager, - CrossSpaceCompactionTaskFactory taskFactory) { + TsFileManager tsFileManager) { super( logicalStorageGroupName, virtualStorageGroupId, storageGroupDir, timePartition, - tsFileManager, - taskFactory); + tsFileManager); } /** @@ -116,13 +113,16 @@ public void selectAndSubmit() { if (mergeFiles[0].size() > 0 && mergeFiles[1].size() > 0) { AbstractCompactionTask compactionTask = - taskFactory.createTask( - logicalStorageGroupName, - virtualGroupId, - timePartition, - tsFileManager, - mergeFiles[0], - mergeFiles[1]); + IoTDBDescriptor.getInstance() + .getConfig() + .getCrossCompactionStrategy() + .getCompactionTask( + logicalStorageGroupName, + virtualGroupId, + timePartition, + tsFileManager, + mergeFiles[0], + mergeFiles[1]); CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); LOGGER.info( "{} [Compaction] submit a task with {} sequence file and {} unseq files", diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java index 405e9c901d803..ffed6030ed695 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java @@ -30,7 +30,6 @@ public abstract class AbstractInnerSpaceCompactionSelector extends AbstractCompa protected long timePartition; protected List tsFileResources; protected boolean sequence; - protected InnerSpaceCompactionTaskFactory taskFactory; protected TsFileManager tsFileManager; public AbstractInnerSpaceCompactionSelector( @@ -38,14 +37,12 @@ public AbstractInnerSpaceCompactionSelector( String virtualStorageGroupName, long timePartition, TsFileManager tsFileManager, - boolean sequence, - InnerSpaceCompactionTaskFactory taskFactory) { + boolean sequence) { this.logicalStorageGroupName = logicalStorageGroupName; this.virtualStorageGroupName = virtualStorageGroupName; this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.sequence = sequence; - this.taskFactory = taskFactory; if (sequence) { tsFileResources = tsFileManager.getSequenceListByTimePartition(timePartition).getArrayList(); } else { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 354e3296589a1..9f3d914f0ce1d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -23,7 +23,6 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTaskFactory; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -37,7 +36,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.PriorityQueue; @@ -54,22 +52,15 @@ public class SizeTieredCompactionSelector extends AbstractInnerSpaceCompactionSelector { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); public SizeTieredCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, TsFileManager tsFileManager, - boolean sequence, - InnerSpaceCompactionTaskFactory taskFactory) { - super( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - sequence, - taskFactory); + boolean sequence) { + super(logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, sequence); } /** @@ -156,9 +147,7 @@ private boolean selectLevelTask( private int searchMaxFileLevel() throws IOException { int maxLevel = -1; - Iterator iterator = tsFileResources.iterator(); - while (iterator.hasNext()) { - TsFileResource currentFile = iterator.next(); + for (TsFileResource currentFile : tsFileResources) { TsFileNameGenerator.TsFileName currentName = TsFileNameGenerator.getTsFileName(currentFile.getTsFile().getName()); if (currentName.getInnerCompactionCnt() > maxLevel) { @@ -168,17 +157,18 @@ private int searchMaxFileLevel() throws IOException { return maxLevel; } - private boolean createAndSubmitTask(List selectedFileList) + private void createAndSubmitTask(List selectedFileList) throws InterruptedException { AbstractCompactionTask compactionTask = - taskFactory.createTask( + new SizeTieredCompactionTask( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, selectedFileList, - sequence); - return CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); + sequence, + CompactionTaskManager.currentTaskNum); + CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); } private class SizeTieredCompactionTaskComparator diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java index d1753cb3ae139..a9536ee873159 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTaskFactory.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java similarity index 51% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTaskFactory.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java index 49ba832071e8b..95e57c2b734c7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTaskFactory.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java @@ -16,33 +16,22 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.iotdb.db.engine.compaction.performer; -package org.apache.iotdb.db.engine.compaction.inner; - -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import java.io.IOException; import java.util.List; -public class InnerSpaceCompactionTaskFactory { - public AbstractCompactionTask createTask( - String logicalStorageGroupName, - String virtualStorageGroup, - long timePartition, - TsFileManager tsFileManager, - List selectedTsFileResourceList, - boolean sequence) { - return IoTDBDescriptor.getInstance() - .getConfig() - .getInnerCompactionStrategy() - .getCompactionTask( - logicalStorageGroupName, - virtualStorageGroup, - timePartition, - tsFileManager, - selectedTsFileResourceList, - sequence); - } +/** + * CompactionPerformer is used to compact multiple files into one or multiple files. Different + * performers may use different implementation to achieve this goal. Some may read chunk directly + * from tsfile, and some may using query tools to read data point by point from tsfile. Notice, not + * all kinds of Performer can be used for all kinds of compaction tasks! + */ +public abstract class AbstractCompactionPerformer { + protected List seqFiles; + protected List unseqFiles; + + public abstract void perform() throws IOException; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java new file mode 100644 index 0000000000000..95ff206760b82 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.performer; + +import java.io.IOException; + +public class ReadChunkCompactionPerformer extends AbstractCompactionPerformer { + @Override + public void perform() throws IOException {} +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java new file mode 100644 index 0000000000000..97f692129ca29 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.performer; + +import java.io.IOException; + +public class ReadPointCompactionPerformer extends AbstractCompactionPerformer { + @Override + public void perform() throws IOException {} +} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java index a413419f32576..7a052fd50be6f 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.compaction.cross; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; @@ -424,18 +425,20 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { mergeResource.clear(); index++; if (mergeFiles.length > 0) { - CrossSpaceCompactionTaskFactory taskFactory = new CrossSpaceCompactionTaskFactory(); AbstractCompactionTask compactionTask = - taskFactory.createTask( - COMPACTION_TEST_SG, - "0", - 0, - new TsFileManager( - "root.compactionTest", + IoTDBDescriptor.getInstance() + .getConfig() + .getCrossCompactionStrategy() + .getCompactionTask( + COMPACTION_TEST_SG, "0", - "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - mergeResource.getSeqFiles(), - mergeResource.getUnseqFiles()); + 0, + new TsFileManager( + "root.compactionTest", + "0", + "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), + mergeResource.getSeqFiles(), + mergeResource.getUnseqFiles()); compactionTask.call(); List targetTsfileResourceList = new ArrayList<>(); for (TsFileResource seqResource : seqResources) { @@ -727,18 +730,20 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E List[] mergeFiles = fileSelector.select(); mergeResource.clear(); if (mergeFiles.length > 0) { - CrossSpaceCompactionTaskFactory taskFactory = new CrossSpaceCompactionTaskFactory(); AbstractCompactionTask compactionTask = - taskFactory.createTask( - COMPACTION_TEST_SG, - "0", - 0, - new TsFileManager( - "root.compactionTest", + IoTDBDescriptor.getInstance() + .getConfig() + .getCrossCompactionStrategy() + .getCompactionTask( + COMPACTION_TEST_SG, "0", - "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - mergeResource.getSeqFiles(), - mergeResource.getUnseqFiles()); + 0, + new TsFileManager( + "root.compactionTest", + "0", + "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), + mergeResource.getSeqFiles(), + mergeResource.getUnseqFiles()); compactionTask.call(); List targetTsfileResourceList = new ArrayList<>(); for (TsFileResource seqResource : seqResources.subList(1, 4)) { @@ -1029,18 +1034,20 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { List[] mergeFiles = fileSelector.select(); mergeResource.clear(); if (mergeFiles.length > 0) { - CrossSpaceCompactionTaskFactory taskFactory = new CrossSpaceCompactionTaskFactory(); AbstractCompactionTask compactionTask = - taskFactory.createTask( - COMPACTION_TEST_SG, - "0", - 0, - new TsFileManager( - "root.compactionTest", + IoTDBDescriptor.getInstance() + .getConfig() + .getCrossCompactionStrategy() + .getCompactionTask( + COMPACTION_TEST_SG, "0", - "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), - mergeResource.getSeqFiles(), - mergeResource.getUnseqFiles()); + 0, + new TsFileManager( + "root.compactionTest", + "0", + "target\\data\\sequence\\test\\root.compactionTest\\0\\0\\"), + mergeResource.getSeqFiles(), + mergeResource.getUnseqFiles()); compactionTask.call(); List targetTsfileResourceList = new ArrayList<>(); for (TsFileResource seqResource : seqResources.subList(1, 4)) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java deleted file mode 100644 index 2b053705d937c..0000000000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.inner; - -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; -import org.apache.iotdb.db.engine.compaction.CompactionScheduler; -import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.task.FakedInnerSpaceCompactionTaskFactory; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; -import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.exception.metadata.MetadataException; -import org.apache.iotdb.tsfile.exception.write.WriteProcessException; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -public class InnerCompactionSchedulerTest extends AbstractCompactionTest { - - private long originFileSize; - long MAX_WAITING_TIME = 120_000L; - boolean oldEnableSeqSpaceCompaction = - IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction(); - boolean oldEnableUnSeqSpaceCompaction = - IoTDBDescriptor.getInstance().getConfig().isEnableUnseqSpaceCompaction(); - int oldConcurrentCompactionThread = - IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread(); - int oldMaxCompactionCandidateFileNum = - IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum(); - - @Before - public void setUp() throws IOException, WriteProcessException, MetadataException { - CompactionTaskManager.getInstance().start(); - super.setUp(); - originFileSize = IoTDBDescriptor.getInstance().getConfig().getTargetCompactionFileSize(); - IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(90); - } - - @After - public void tearDown() throws IOException, StorageEngineException { - CompactionTaskManager.getInstance().stop(); - IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(originFileSize); - IoTDBDescriptor.getInstance() - .getConfig() - .setEnableSeqSpaceCompaction(oldEnableSeqSpaceCompaction); - IoTDBDescriptor.getInstance() - .getConfig() - .setEnableUnseqSpaceCompaction(oldEnableUnSeqSpaceCompaction); - IoTDBDescriptor.getInstance() - .getConfig() - .setConcurrentCompactionThread(oldConcurrentCompactionThread); - IoTDBDescriptor.getInstance() - .getConfig() - .setMaxInnerCompactionCandidateFileNum(oldMaxCompactionCandidateFileNum); - super.tearDown(); - } - - @Test - public void testFileSelector1() throws IOException, MetadataException, WriteProcessException { - IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(true); - IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(true); - IoTDBDescriptor.getInstance().getConfig().setConcurrentCompactionThread(50); - IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(4); - IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1000000); - createFiles(2, 2, 3, 100, 0, 0, 50, 50, false, true); - registerTimeseriesInMManger(2, 3, false); - createFiles(2, 3, 5, 50, 250, 250, 50, 50, false, true); - registerTimeseriesInMManger(3, 5, false); - createFiles(2, 5, 5, 50, 600, 800, 50, 50, false, true); - registerTimeseriesInMManger(5, 5, false); - TsFileManager tsFileManager = new TsFileManager("testSG", "0", "tmp"); - tsFileManager.addAll(seqResources, true); - - CompactionScheduler.tryToSubmitInnerSpaceCompactionTask( - "testSG", "0", 0L, tsFileManager, true, new FakedInnerSpaceCompactionTaskFactory()); - CompactionTaskManager.getInstance().submitTaskFromTaskQueue(); - try { - Thread.sleep(5000); - } catch (Exception e) { - - } - Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); - } - - @Test - public void testFileSelector2() throws IOException, MetadataException, WriteProcessException { - IoTDBDescriptor.getInstance().getConfig().setConcurrentCompactionThread(50); - IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(50); - TsFileResourceList tsFileResources = new TsFileResourceList(); - createFiles(2, 2, 3, 100, 0, 0, 50, 50, false, true); - createFiles(2, 3, 5, 50, 250, 250, 50, 50, false, true); - seqResources.get(0).setStatus(TsFileResourceStatus.COMPACTION_CANDIDATE); - seqResources.get(0).setStatus(TsFileResourceStatus.COMPACTING); - TsFileManager tsFileManager = new TsFileManager("testSG", "0", "tmp"); - tsFileManager.addAll(seqResources, true); - CompactionScheduler.tryToSubmitInnerSpaceCompactionTask( - "testSG", "0", 0L, tsFileManager, true, new FakedInnerSpaceCompactionTaskFactory()); - CompactionTaskManager.getInstance().submitTaskFromTaskQueue(); - - long waitingTime = 0; - while (CompactionTaskManager.getInstance().getExecutingTaskCount() != 0) { - try { - Thread.sleep(100); - waitingTime += 100; - if (waitingTime > MAX_WAITING_TIME) { - Assert.fail(); - break; - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - Assert.assertEquals(4, tsFileManager.getTsFileList(true).size()); - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTaskFactory.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTaskFactory.java deleted file mode 100644 index a54fb6dffed2b..0000000000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTaskFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.task; - -import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTaskFactory; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; - -import java.util.List; - -public class FakedInnerSpaceCompactionTaskFactory extends InnerSpaceCompactionTaskFactory { - - public FakedInnerSpaceCompactionTaskFactory() {} - - public AbstractCompactionTask createTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartition, - TsFileManager tsFileManager, - TsFileResourceList tsFileResourceList, - List selectedTsFileResourceList, - boolean sequence) { - return new FakedInnerSpaceCompactionTask( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - selectedTsFileResourceList, - sequence, - CompactionTaskManager.currentTaskNum); - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java index 014ad1afd5259..62853b5d9dcce 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java @@ -22,8 +22,8 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.cross.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; public class CompactionConfigRestorer { private boolean enableSeqSpaceCompaction = true; From d8951e8e6dc99c0057e80f87cd1ce71145a3bd5b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 1 Apr 2022 16:09:43 +0800 Subject: [PATCH 03/22] replace compact in InnerSpaceCompactionUtils with RCCP --- .../sizetiered/SizeTieredCompactionTask.java | 7 +- .../utils/InnerSpaceCompactionUtils.java | 137 ------------------ .../AbstractCompactionPerformer.java | 3 +- .../ReadChunkCompactionPerformer.java | 116 ++++++++++++++- .../inner/InnerSeqCompactionTest.java | 7 +- .../InnerSpaceCompactionExceptionTest.java | 15 +- .../InnerSpaceCompactionUtilsAlignedTest.java | 16 +- ...nnerSpaceCompactionUtilsNoAlignedTest.java | 20 +-- .../InnerSpaceCompactionUtilsOldTest.java | 3 +- .../SizeTieredCompactionRecoverTest.java | 42 +++--- ...TieredCompactionRecoverCompatibleTest.java | 9 +- .../SizeTieredCompactionRecoverTest.java | 17 ++- 12 files changed, 194 insertions(+), 198 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 6058b84a445c3..80be49f4817c4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -24,6 +24,8 @@ import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -51,6 +53,7 @@ public class SizeTieredCompactionTask extends AbstractInnerSpaceCompactionTask { protected TsFileResourceList tsFileResourceList; protected boolean[] isHoldingReadLock; protected boolean[] isHoldingWriteLock; + protected AbstractCompactionPerformer performer; public SizeTieredCompactionTask( String logicalStorageGroupName, @@ -116,7 +119,9 @@ protected void doCompaction() throws Exception { // carry out the compaction if (sequence) { - InnerSpaceCompactionUtils.compact(targetTsFileResource, selectedTsFileResourceList); + performer = + new ReadChunkCompactionPerformer(selectedTsFileResourceList, targetTsFileResource); + performer.perform(); } else { CompactionUtils.compact( Collections.emptyList(), diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java index a9536ee873159..57342a0110b3f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java @@ -27,25 +27,12 @@ import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; -import org.apache.iotdb.db.exception.metadata.MetadataException; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; -import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; -import org.apache.iotdb.tsfile.read.TsFileSequenceReader; -import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; -import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,8 +40,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; -import java.util.LinkedList; import java.util.List; public class InnerSpaceCompactionUtils { @@ -66,93 +51,6 @@ private InnerSpaceCompactionUtils() { throw new IllegalStateException("Utility class"); } - public static void compact(TsFileResource targetResource, List tsFileResources) - throws IOException, MetadataException, InterruptedException { - - try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(tsFileResources); - TsFileIOWriter writer = new TsFileIOWriter(targetResource.getTsFile())) { - while (deviceIterator.hasNextDevice()) { - Pair deviceInfo = deviceIterator.nextDevice(); - String device = deviceInfo.left; - boolean aligned = deviceInfo.right; - - writer.startChunkGroup(device); - if (aligned) { - compactAlignedSeries(device, targetResource, writer, deviceIterator); - } else { - compactNotAlignedSeries(device, targetResource, writer, deviceIterator); - } - writer.endChunkGroup(); - } - - for (TsFileResource tsFileResource : tsFileResources) { - targetResource.updatePlanIndexes(tsFileResource); - } - writer.endFile(); - targetResource.close(); - } - } - - private static void checkThreadInterrupted(TsFileResource tsFileResource) - throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException( - String.format( - "[Compaction] compaction for target file %s abort", tsFileResource.toString())); - } - } - - private static void compactNotAlignedSeries( - String device, - TsFileResource targetResource, - TsFileIOWriter writer, - MultiTsFileDeviceIterator deviceIterator) - throws IOException, MetadataException, InterruptedException { - MultiTsFileDeviceIterator.MeasurementIterator seriesIterator = - deviceIterator.iterateNotAlignedSeries(device, true); - while (seriesIterator.hasNextSeries()) { - checkThreadInterrupted(targetResource); - // TODO: we can provide a configuration item to enable concurrent between each series - PartialPath p = new PartialPath(device, seriesIterator.nextSeries()); - IMeasurementSchema measurementSchema; - // TODO: seriesIterator needs to be refactor. - // This statement must be called before next hasNextSeries() called, or it may be trapped in a - // dead-loop. - LinkedList>> readerAndChunkMetadataList = - seriesIterator.getMetadataListForCurrentSeries(); - try { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { - measurementSchema = - IDTableManager.getInstance().getSeriesSchema(device, p.getMeasurement()); - } else { - measurementSchema = IoTDB.schemaProcessor.getSeriesSchema(p); - } - } catch (PathNotExistException e) { - logger.info("A deleted path is skipped: {}", e.getMessage()); - continue; - } - SingleSeriesCompactionExecutor compactionExecutorOfCurrentTimeSeries = - new SingleSeriesCompactionExecutor( - p, measurementSchema, readerAndChunkMetadataList, writer, targetResource); - compactionExecutorOfCurrentTimeSeries.execute(); - } - } - - private static void compactAlignedSeries( - String device, - TsFileResource targetResource, - TsFileIOWriter writer, - MultiTsFileDeviceIterator deviceIterator) - throws IOException, InterruptedException { - checkThreadInterrupted(targetResource); - LinkedList>> readerAndChunkMetadataList = - deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); - AlignedSeriesCompactionExecutor compactionExecutor = - new AlignedSeriesCompactionExecutor( - device, targetResource, readerAndChunkMetadataList, writer); - compactionExecutor.execute(); - } - public static boolean deleteTsFilesInDisk( Collection mergeTsFiles, String storageGroupName) { logger.info("{} [Compaction] Compaction starts to delete real file ", storageGroupName); @@ -185,34 +83,6 @@ public static void deleteModificationForSourceFile( } } - /** - * This method is called to recover modifications while an exception occurs during compaction. It - * append new modifications of each selected tsfile to its corresponding old mods file and delete - * the compaction mods file. - * - * @param selectedTsFileResources - * @throws IOException - */ - public static void appendNewModificationsToOldModsFile( - List selectedTsFileResources) throws IOException { - for (TsFileResource sourceFile : selectedTsFileResources) { - // if there are modifications to this seqFile during compaction - if (sourceFile.getCompactionModFile().exists()) { - ModificationFile compactionModificationFile = - ModificationFile.getCompactionMods(sourceFile); - Collection newModification = compactionModificationFile.getModifications(); - compactionModificationFile.close(); - // write the new modifications to its old modification file - try (ModificationFile oldModificationFile = sourceFile.getModFile()) { - for (Modification modification : newModification) { - oldModificationFile.write(modification); - } - } - FileUtils.delete(new File(ModificationFile.getCompactionMods(sourceFile).getFilePath())); - } - } - } - /** * Collect all the compaction modification files of source files, and combines them as the * modification file of target file. @@ -262,13 +132,6 @@ public static ICrossSpaceMergeFileSelector getCrossSpaceFileSelector( } } - public static class TsFileNameComparator implements Comparator { - - @Override - public int compare(TsFileSequenceReader o1, TsFileSequenceReader o2) { - return TsFileManager.compareFileName(new File(o1.getFileName()), new File(o2.getFileName())); - } - } /** * Update the targetResource. Move xxx.target to xxx.tsfile and serialize xxx.tsfile.resource . * diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java index 95e57c2b734c7..2345847a5a6be 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.compaction.performer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.metadata.MetadataException; import java.io.IOException; import java.util.List; @@ -33,5 +34,5 @@ public abstract class AbstractCompactionPerformer { protected List seqFiles; protected List unseqFiles; - public abstract void perform() throws IOException; + public abstract void perform() throws IOException, MetadataException, InterruptedException; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java index 95ff206760b82..56b61c623ba92 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java @@ -18,9 +18,123 @@ */ package org.apache.iotdb.db.engine.compaction.performer; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.inner.utils.AlignedSeriesCompactionExecutor; +import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; +import org.apache.iotdb.db.engine.compaction.inner.utils.SingleSeriesCompactionExecutor; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.path.PartialPath; +import org.apache.iotdb.db.service.IoTDB; +import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; +import org.apache.iotdb.tsfile.read.TsFileSequenceReader; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; +import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; +import java.util.LinkedList; +import java.util.List; public class ReadChunkCompactionPerformer extends AbstractCompactionPerformer { + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); + private TsFileResource targetResource; + + public ReadChunkCompactionPerformer(List sourceFiles, TsFileResource targetFile) { + this.seqFiles = sourceFiles; + this.targetResource = targetFile; + } + @Override - public void perform() throws IOException {} + public void perform() throws IOException, MetadataException, InterruptedException { + try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles); + TsFileIOWriter writer = new TsFileIOWriter(targetResource.getTsFile())) { + while (deviceIterator.hasNextDevice()) { + Pair deviceInfo = deviceIterator.nextDevice(); + String device = deviceInfo.left; + boolean aligned = deviceInfo.right; + + writer.startChunkGroup(device); + if (aligned) { + compactAlignedSeries(device, targetResource, writer, deviceIterator); + } else { + compactNotAlignedSeries(device, targetResource, writer, deviceIterator); + } + writer.endChunkGroup(); + } + + for (TsFileResource tsFileResource : seqFiles) { + targetResource.updatePlanIndexes(tsFileResource); + } + writer.endFile(); + targetResource.close(); + } + } + + private void compactAlignedSeries( + String device, + TsFileResource targetResource, + TsFileIOWriter writer, + MultiTsFileDeviceIterator deviceIterator) + throws IOException, InterruptedException { + checkThreadInterrupted(); + LinkedList>> readerAndChunkMetadataList = + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + AlignedSeriesCompactionExecutor compactionExecutor = + new AlignedSeriesCompactionExecutor( + device, targetResource, readerAndChunkMetadataList, writer); + compactionExecutor.execute(); + } + + private void checkThreadInterrupted() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException( + String.format( + "[Compaction] compaction for target file %s abort", targetResource.toString())); + } + } + + private void compactNotAlignedSeries( + String device, + TsFileResource targetResource, + TsFileIOWriter writer, + MultiTsFileDeviceIterator deviceIterator) + throws IOException, MetadataException, InterruptedException { + MultiTsFileDeviceIterator.MeasurementIterator seriesIterator = + deviceIterator.iterateNotAlignedSeries(device, true); + while (seriesIterator.hasNextSeries()) { + checkThreadInterrupted(); + // TODO: we can provide a configuration item to enable concurrent between each series + PartialPath p = new PartialPath(device, seriesIterator.nextSeries()); + IMeasurementSchema measurementSchema; + // TODO: seriesIterator needs to be refactor. + // This statement must be called before next hasNextSeries() called, or it may be trapped in a + // dead-loop. + LinkedList>> readerAndChunkMetadataList = + seriesIterator.getMetadataListForCurrentSeries(); + try { + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + measurementSchema = + IDTableManager.getInstance().getSeriesSchema(device, p.getMeasurement()); + } else { + measurementSchema = IoTDB.schemaProcessor.getSeriesSchema(p); + } + } catch (PathNotExistException e) { + LOGGER.info("A deleted path is skipped: {}", e.getMessage()); + continue; + } + SingleSeriesCompactionExecutor compactionExecutorOfCurrentTimeSeries = + new SingleSeriesCompactionExecutor( + p, measurementSchema, readerAndChunkMetadataList, writer, targetResource); + compactionExecutorOfCurrentTimeSeries.execute(); + } + } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java index 95f2ae0a7f79a..248553497d1f5 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; @@ -227,7 +228,7 @@ public void testDeserializePage() throws MetadataException, IOException { timeValuePair.getTimestamp() >= 250L && timeValuePair.getTimestamp() <= 300L); } - InnerSpaceCompactionUtils.compact(targetTsFileResource, sourceResources); + new ReadChunkCompactionPerformer(sourceResources, targetTsFileResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); InnerSpaceCompactionUtils.combineModsInCompaction( sourceResources, targetTsFileResource); @@ -452,7 +453,7 @@ public void testAppendPage() throws IOException, MetadataException, InterruptedE timeValuePair -> timeValuePair.getTimestamp() >= 250L && timeValuePair.getTimestamp() <= 300L); } - InnerSpaceCompactionUtils.compact(targetTsFileResource, toMergeResources); + new ReadChunkCompactionPerformer(toMergeResources, targetTsFileResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); InnerSpaceCompactionUtils.combineModsInCompaction( toMergeResources, targetTsFileResource); @@ -727,7 +728,7 @@ public void testAppendChunk() throws IOException, IllegalPathException, Metadata timeValuePair.getTimestamp() >= 250L && timeValuePair.getTimestamp() <= 300L); } - InnerSpaceCompactionUtils.compact(targetTsFileResource, toMergeResources); + new ReadChunkCompactionPerformer(toMergeResources, targetTsFileResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); InnerSpaceCompactionUtils.combineModsInCompaction( toMergeResources, targetTsFileResource); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java index da0b9a349da3d..c137f1bddbf70 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; @@ -63,7 +64,7 @@ public void testWhenAllSourceExistsAndTargetNotComplete() throws Exception { compactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); try (FileOutputStream os = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = os.getChannel()) { @@ -111,7 +112,7 @@ public void testWhenAllSourceExistsAndTargetComplete() throws Exception { compactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); compactionLogger.close(); CompactionExceptionHandler.handleException( @@ -155,7 +156,7 @@ public void testWhenSomeSourceLostAndTargetComplete() throws Exception { compactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); for (TsFileResource resource : seqResources) { tsFileManager.getSequenceListByTimePartition(0).remove(resource); @@ -210,7 +211,7 @@ public void testWhenSomeSourceLostAndTargetNotComplete() throws Exception { compactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); seqResources.get(0).remove(); try (FileOutputStream os = new FileOutputStream(targetResource.getTsFile(), true); @@ -263,7 +264,7 @@ public void testHandleWithCompactionMods() throws Exception { compactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); @@ -337,7 +338,7 @@ public void testHandleWithNormalMods() throws Exception { new Pair<>(i * ptNum, i * ptNum + 10)); CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); seqResources.get(0).remove(); compactionLogger.close(); @@ -395,7 +396,7 @@ public void testHandleWithCompactionModsAndNormalMods() throws Exception { new Pair<>(i * ptNum, i * ptNum + 5)); CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java index 8541f39066166..164e67f5e2eef 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.TestUtilsForAlignedSeries; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; @@ -133,7 +133,7 @@ public void testSimpleAlignedTsFileCompaction() throws Exception { Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -201,7 +201,7 @@ public void testAlignedTsFileWithModificationCompaction() throws Exception { Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -263,7 +263,7 @@ public void testAlignedTsFileWithNullValueCompaction() throws Exception { Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -328,7 +328,7 @@ public void testAlignedTsFileWithDifferentSchemaInDifferentTsFileCompaction() th Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -391,7 +391,7 @@ public void testAlignedTsFileWithDifferentDataTypeCompaction() throws Exception Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -456,7 +456,7 @@ public void testAlignedTsFileWithDifferentDataTypeInDifferentTsFileCompaction() Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, @@ -522,7 +522,7 @@ public void testAlignedTsFileWithBadSchemaCompaction() throws Exception { Map> originData = CompactionCheckerUtils.getDataByQuery( fullPaths, iMeasurementSchemas, resources, new ArrayList<>()); - InnerSpaceCompactionUtils.compact(targetResource, resources); + new ReadChunkCompactionPerformer(resources, targetResource).perform(); Map> compactedData = CompactionCheckerUtils.getDataByQuery( fullPaths, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java index e63f53094fbd4..d5dfe6f8dee2b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; @@ -201,7 +201,7 @@ public void testDirectlyFlushChunk() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); long[] points = new long[fileNum]; for (int i = 1; i <= fileNum; i++) { @@ -291,7 +291,7 @@ public void testLargeChunkMergeWithCacheChunkAndFlush() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page for (String path : fullPathSet) { @@ -383,7 +383,7 @@ public void testLargeChunkDeserializeIntoPoint() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page for (String path : fullPathSet) { @@ -462,7 +462,7 @@ public void testMergeChunk() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page List> chunkPointsArray = new ArrayList<>(); @@ -571,7 +571,7 @@ public void testMiddleChunkDeserialize() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page for (String path : fullPathSet) { @@ -650,7 +650,7 @@ public void testDeserializePage() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page List> chunkPointsArray = new ArrayList<>(); @@ -732,7 +732,7 @@ public void testDeserializeCachedChunk() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page List> chunkPointsArray = new ArrayList<>(); @@ -808,7 +808,7 @@ public void testMixCompact1() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page for (String path : fullPathSet) { @@ -886,7 +886,7 @@ public void testMixCompact2() throws Exception { tsFileName.getVersion(), tsFileName.getInnerCompactionCnt() + 1, tsFileName.getCrossCompactionCnt()))); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); Map>> chunkPagePointsNumMerged = new HashMap<>(); // outer list is a chunk, inner list is point num in each page for (String path : fullPathSet) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java index ee852e869224d..76c2d92a8fddf 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -105,7 +106,7 @@ public void testCompact() throws IOException, MetadataException, InterruptedExce new CompactionLogger( new File(targetTsFileResource.getTsFilePath().concat(".compaction.log"))); sizeTieredCompactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); - InnerSpaceCompactionUtils.compact(targetTsFileResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetTsFileResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); sizeTieredCompactionLogger.close(); Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java index 4d1b240dfe4cd..2d3207cde4cdd 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; @@ -143,8 +144,9 @@ public void testCompactionRecoverWithUncompletedTargetFileAndLog() throws Except compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); compactionLogger.close(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); BufferedReader logReader = new BufferedReader(new FileReader(compactionLogFile)); @@ -263,8 +265,9 @@ public void testRecoverWithAllSourceFilesExisted() throws Exception { compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); compactionLogger.close(); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, true) .doCompaction(); @@ -373,8 +376,9 @@ public void testRecoverWithAllSourceFilesExistedAndTargetFileNotExist() throws E compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); // target file may not exist targetTsFileResource.remove(); compactionLogger.close(); @@ -485,8 +489,9 @@ public void testRecoverWithoutAllSourceFilesExisted() throws Exception { compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); // delete one source file seqResources.get(0).remove(); @@ -558,7 +563,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExist() throws Excep CompactionLogger compactionLogger = new CompactionLogger(logFile); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); @@ -626,7 +631,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExistAndTargetFileNo CompactionLogger compactionLogger = new CompactionLogger(logFile); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); // target file may not exist targetResource.remove(); for (int i = 0; i < seqResources.size(); i++) { @@ -697,7 +702,7 @@ public void testRecoverWithoutAllSourceFilesExistAndModFiles() throws Exception CompactionLogger compactionLogger = new CompactionLogger(logFile); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); @@ -808,8 +813,9 @@ public void testRecoverCompleteTargetFileAndCompactionLog() throws Exception { compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); compactionLogger.close(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); tsFileManager.add(targetTsFileResource, true); @@ -905,8 +911,9 @@ public void testCompactionRecoverWithCompletedTargetFileAndLog() throws Exceptio compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); compactionLogger.close(); for (TsFileResource resource : new ArrayList<>(seqResources.subList(0, 3))) { @@ -1005,8 +1012,9 @@ public void testCompactionRecoverWithCompletedTargetFile() throws Exception { compactionLogger.logFiles(tmpSeqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetTsFileResource), STR_TARGET_FILES); deleteFileIfExists(targetTsFileResource.getTsFile()); - InnerSpaceCompactionUtils.compact( - targetTsFileResource, new ArrayList<>(seqResources.subList(0, 3))); + new ReadChunkCompactionPerformer( + new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) + .perform(); compactionLogger.close(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); deleteFileIfExists(compactionLogFile); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java index 867f882a3ddfc..b25168c6575b4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -67,7 +68,7 @@ public void testCompatibleWithAllSourceFilesExistWithFilePath() throws Exception registerTimeseriesInMManger(2, 3, false); TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); RandomAccessFile targetFile = new RandomAccessFile(targetResource.getTsFile(), "rw"); long fileLength = targetFile.length(); targetFile.getChannel().truncate(fileLength - 20); @@ -124,7 +125,7 @@ public void testCompatibleWithSomeSourceFilesLostWithFilePath() throws Exception registerTimeseriesInMManger(2, 3, false); TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, "root.compactionTest"); // first source file does not exist @@ -180,7 +181,7 @@ public void testCompatibleWithAllSourceFilesExistWithFileInfo() throws Exception registerTimeseriesInMManger(2, 3, false); TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true); - InnerSpaceCompactionUtils.compact(targetResource, seqResources); + new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); RandomAccessFile targetFile = new RandomAccessFile(targetResource.getTsFile(), "rw"); long fileLength = targetFile.length(); targetFile.getChannel().truncate(fileLength - 20); @@ -217,7 +218,7 @@ public void testCompatibleWithSomeSourceFilesLostWithFileInfo() throws Exception registerTimeseriesInMManger(2, 3, false); TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(unseqResources, true); - InnerSpaceCompactionUtils.compact(targetResource, unseqResources); + new ReadChunkCompactionPerformer(unseqResources, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, "root.compactionTest"); // first source file does not exist diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java index 6df79c7958f68..7a6ae01b80a3b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; @@ -234,7 +235,7 @@ public void testRecoverWithCompleteTargetFileUsingFileInfo() throws Exception { compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); CompactionRecoverTask recoverTask = new CompactionRecoverTask( @@ -284,7 +285,7 @@ public void testRecoverWithIncompleteTargetFileUsingFileInfo() throws Exception compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); @@ -339,7 +340,7 @@ public void testRecoverWithCompleteTargetFileUsingFilePath() throws Exception { logger.logFiles(sourceFiles, CompactionLogger.STR_SOURCE_FILES); logger.logFiles(Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); logger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); CompactionRecoverTask recoverTask = new CompactionRecoverTask( @@ -389,7 +390,7 @@ public void testRecoverWithIncompleteTargetFileUsingFilePath() throws Exception compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); @@ -447,7 +448,7 @@ public void testRecoverWithCompleteTargetFileUsingFileInfoAndChangingDataDirs() compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); long sizeOfTargetFile = targetResource.getTsFileSize(); FileUtils.moveDirectory( @@ -543,7 +544,7 @@ public void testRecoverWithIncompleteTargetFileUsingFileInfoAndChangingDataDirs( compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); @@ -636,7 +637,7 @@ public void testRecoverWithCompleteTargetFileUsingFilePathAndChangingDataDirs() compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); long sizeOfTargetFile = targetResource.getTsFileSize(); FileUtils.moveDirectory( @@ -732,7 +733,7 @@ public void testRecoverWithIncompleteTargetFileUsingFilePathAndChangingDataDirs( compactionLogger.logFiles(sourceFiles, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); - InnerSpaceCompactionUtils.compact(targetResource, sourceFiles); + new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); From b228a020687bab51e5e4d27ddf176ab0c7cbab42 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 1 Apr 2022 16:14:02 +0800 Subject: [PATCH 04/22] rename test --- ...edTest.java => ReadChunkCompactionPerformerAlignedTest.java} | 2 +- ...Test.java => ReadChunkCompactionPerformerNoAlignedTest.java} | 2 +- ...ilsOldTest.java => ReadChunkCompactionPerformerOldTest.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/{InnerSpaceCompactionUtilsAlignedTest.java => ReadChunkCompactionPerformerAlignedTest.java} (99%) rename server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/{InnerSpaceCompactionUtilsNoAlignedTest.java => ReadChunkCompactionPerformerNoAlignedTest.java} (99%) rename server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/{InnerSpaceCompactionUtilsOldTest.java => ReadChunkCompactionPerformerOldTest.java} (98%) diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java similarity index 99% rename from server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java rename to server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java index 164e67f5e2eef..c1e85647fe3e9 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java @@ -52,7 +52,7 @@ import java.util.Map; import java.util.Random; -public class InnerSpaceCompactionUtilsAlignedTest { +public class ReadChunkCompactionPerformerAlignedTest { private static final String storageGroup = "root.testAlignedCompaction"; private static File dataDirectory = new File( diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java similarity index 99% rename from server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java rename to server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java index d5dfe6f8dee2b..71523083b6e5d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java @@ -60,7 +60,7 @@ * This class is used to test InnerSpaceCompactionUtils.compact. Notice, it just tests not aligned * timeseries. */ -public class InnerSpaceCompactionUtilsNoAlignedTest { +public class ReadChunkCompactionPerformerNoAlignedTest { private final String storageGroup = "root.compactionTest"; private final String[] devices = new String[] {"device0", "device1", "device2", "device3"}; private PartialPath[] devicePath = new PartialPath[devices.length]; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java similarity index 98% rename from server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java rename to server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java index 76c2d92a8fddf..a6b8914d5b653 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java @@ -51,7 +51,7 @@ * current compaction. However, due to this test's strong coupling with an older version of * compaction, we may remove it in the future. */ -public class InnerSpaceCompactionUtilsOldTest extends InnerCompactionTest { +public class ReadChunkCompactionPerformerOldTest extends InnerCompactionTest { File tempSGDir; From 9e4f46a3b6f6280e5569b3dd4661eb0e6a37624b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 1 Apr 2022 16:51:00 +0800 Subject: [PATCH 05/22] replace CompactionUtils with ReadPointCompactionPerformer --- .../db/engine/compaction/CompactionUtils.java | 257 ----------------- .../task/RewriteCrossSpaceCompactionTask.java | 11 +- .../sizetiered/SizeTieredCompactionTask.java | 13 +- .../AbstractCompactionPerformer.java | 4 +- .../ReadChunkCompactionPerformer.java | 4 +- .../ReadPointCompactionPerformer.java | 273 +++++++++++++++++- ... => ReadPointCompactionPerformerTest.java} | 57 ++-- .../CrossSpaceCompactionExceptionTest.java | 11 +- ...ewriteCrossSpaceCompactionRecoverTest.java | 15 +- .../inner/InnerSeqCompactionTest.java | 8 +- .../inner/InnerUnseqCompactionTest.java | 10 +- .../ReadChunkCompactionPerformerOldTest.java | 3 +- 12 files changed, 350 insertions(+), 316 deletions(-) rename server/src/test/java/org/apache/iotdb/db/engine/compaction/{CompactionUtilsTest.java => ReadPointCompactionPerformerTest.java} (98%) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java index 6e9cf68291859..274295b2d78d2 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java @@ -19,50 +19,22 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; -import org.apache.iotdb.db.engine.compaction.writer.AbstractCompactionWriter; -import org.apache.iotdb.db.engine.compaction.writer.CrossSpaceCompactionWriter; -import org.apache.iotdb.db.engine.compaction.writer.InnerSpaceCompactionWriter; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; -import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.exception.metadata.IllegalPathException; -import org.apache.iotdb.db.exception.metadata.MetadataException; -import org.apache.iotdb.db.exception.metadata.PathNotExistException; -import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.metadata.path.AlignedPath; -import org.apache.iotdb.db.metadata.path.MeasurementPath; -import org.apache.iotdb.db.metadata.path.PartialPath; -import org.apache.iotdb.db.query.context.QueryContext; -import org.apache.iotdb.db.query.control.QueryResourceManager; -import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader; -import org.apache.iotdb.db.service.IoTDB; -import org.apache.iotdb.db.utils.QueryUtils; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; -import org.apache.iotdb.tsfile.read.common.BatchData; -import org.apache.iotdb.tsfile.read.reader.IBatchReader; -import org.apache.iotdb.tsfile.utils.Pair; -import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; /** * This tool can be used to perform inner space or cross space compaction of aligned and non aligned @@ -74,226 +46,6 @@ public class CompactionUtils { private static final Logger logger = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - public static void compact( - List seqFileResources, - List unseqFileResources, - List targetFileResources) - throws IOException, MetadataException, StorageEngineException, InterruptedException { - long queryId = QueryResourceManager.getInstance().assignCompactionQueryId(); - QueryContext queryContext = new QueryContext(queryId); - QueryDataSource queryDataSource = new QueryDataSource(seqFileResources, unseqFileResources); - QueryResourceManager.getInstance() - .getQueryFileManager() - .addUsedFilesForQuery(queryId, queryDataSource); - - try (AbstractCompactionWriter compactionWriter = - getCompactionWriter(seqFileResources, unseqFileResources, targetFileResources)) { - // Do not close device iterator, because tsfile reader is managed by FileReaderManager. - MultiTsFileDeviceIterator deviceIterator = - new MultiTsFileDeviceIterator(seqFileResources, unseqFileResources); - while (deviceIterator.hasNextDevice()) { - checkThreadInterrupted(targetFileResources); - Pair deviceInfo = deviceIterator.nextDevice(); - String device = deviceInfo.left; - boolean isAligned = deviceInfo.right; - QueryUtils.fillOrderIndexes(queryDataSource, device, true); - - if (isAligned) { - compactAlignedSeries( - device, deviceIterator, compactionWriter, queryContext, queryDataSource); - } else { - compactNonAlignedSeries( - device, deviceIterator, compactionWriter, queryContext, queryDataSource); - } - } - - compactionWriter.endFile(); - updatePlanIndexes(targetFileResources, seqFileResources, unseqFileResources); - } finally { - QueryResourceManager.getInstance().endQuery(queryId); - } - } - - private static void compactAlignedSeries( - String device, - MultiTsFileDeviceIterator deviceIterator, - AbstractCompactionWriter compactionWriter, - QueryContext queryContext, - QueryDataSource queryDataSource) - throws IOException, MetadataException { - MultiTsFileDeviceIterator.AlignedMeasurementIterator alignedMeasurementIterator = - deviceIterator.iterateAlignedSeries(device); - Set allMeasurements = alignedMeasurementIterator.getAllMeasurements(); - List measurementSchemas = new ArrayList<>(); - for (String measurement : allMeasurements) { - try { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { - measurementSchemas.add(IDTableManager.getInstance().getSeriesSchema(device, measurement)); - } else { - measurementSchemas.add( - IoTDB.schemaProcessor.getSeriesSchema(new PartialPath(device, measurement))); - } - } catch (PathNotExistException e) { - logger.info("A deleted path is skipped: {}", e.getMessage()); - } - } - if (measurementSchemas.isEmpty()) { - return; - } - List existedMeasurements = - measurementSchemas.stream() - .map(IMeasurementSchema::getMeasurementId) - .collect(Collectors.toList()); - IBatchReader dataBatchReader = - constructReader( - device, - existedMeasurements, - measurementSchemas, - allMeasurements, - queryContext, - queryDataSource, - true); - - if (dataBatchReader.hasNextBatch()) { - // chunkgroup is serialized only when at least one timeseries under this device has data - compactionWriter.startChunkGroup(device, true); - compactionWriter.startMeasurement(measurementSchemas); - writeWithReader(compactionWriter, dataBatchReader); - compactionWriter.endMeasurement(); - compactionWriter.endChunkGroup(); - } - } - - private static void compactNonAlignedSeries( - String device, - MultiTsFileDeviceIterator deviceIterator, - AbstractCompactionWriter compactionWriter, - QueryContext queryContext, - QueryDataSource queryDataSource) - throws MetadataException, IOException { - boolean hasStartChunkGroup = false; - MultiTsFileDeviceIterator.MeasurementIterator measurementIterator = - deviceIterator.iterateNotAlignedSeries(device, false); - Set allMeasurements = measurementIterator.getAllMeasurements(); - for (String measurement : allMeasurements) { - List measurementSchemas = new ArrayList<>(); - try { - if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { - measurementSchemas.add(IDTableManager.getInstance().getSeriesSchema(device, measurement)); - } else { - measurementSchemas.add( - IoTDB.schemaProcessor.getSeriesSchema(new PartialPath(device, measurement))); - } - } catch (PathNotExistException e) { - logger.info("A deleted path is skipped: {}", e.getMessage()); - continue; - } - - IBatchReader dataBatchReader = - constructReader( - device, - Collections.singletonList(measurement), - measurementSchemas, - allMeasurements, - queryContext, - queryDataSource, - false); - - if (dataBatchReader.hasNextBatch()) { - if (!hasStartChunkGroup) { - // chunkgroup is serialized only when at least one timeseries under this device has - // data - compactionWriter.startChunkGroup(device, false); - hasStartChunkGroup = true; - } - compactionWriter.startMeasurement(measurementSchemas); - writeWithReader(compactionWriter, dataBatchReader); - compactionWriter.endMeasurement(); - } - } - - if (hasStartChunkGroup) { - compactionWriter.endChunkGroup(); - } - } - - private static void writeWithReader(AbstractCompactionWriter writer, IBatchReader reader) - throws IOException { - while (reader.hasNextBatch()) { - BatchData batchData = reader.nextBatch(); - while (batchData.hasCurrent()) { - writer.write(batchData.currentTime(), batchData.currentValue()); - batchData.next(); - } - } - } - - /** - * @param measurementIds if device is aligned, then measurementIds contain all measurements. If - * device is not aligned, then measurementIds only contain one measurement. - */ - private static IBatchReader constructReader( - String deviceId, - List measurementIds, - List measurementSchemas, - Set allSensors, - QueryContext queryContext, - QueryDataSource queryDataSource, - boolean isAlign) - throws IllegalPathException { - PartialPath seriesPath; - TSDataType tsDataType; - if (isAlign) { - seriesPath = new AlignedPath(deviceId, measurementIds, measurementSchemas); - tsDataType = TSDataType.VECTOR; - } else { - seriesPath = new MeasurementPath(deviceId, measurementIds.get(0), measurementSchemas.get(0)); - tsDataType = measurementSchemas.get(0).getType(); - } - return new SeriesRawDataBatchReader( - seriesPath, allSensors, tsDataType, queryContext, queryDataSource, null, null, null, true); - } - - private static AbstractCompactionWriter getCompactionWriter( - List seqFileResources, - List unseqFileResources, - List targetFileResources) - throws IOException { - if (!seqFileResources.isEmpty() && !unseqFileResources.isEmpty()) { - // cross space - return new CrossSpaceCompactionWriter(targetFileResources, seqFileResources); - } else { - // inner space - return new InnerSpaceCompactionWriter(targetFileResources.get(0)); - } - } - - private static void updatePlanIndexes( - List targetResources, - List seqResources, - List unseqResources) { - // as the new file contains data of other files, track their plan indexes in the new file - // so that we will be able to compare data across different IoTDBs that share the same index - // generation policy - // however, since the data of unseq files are mixed together, we won't be able to know - // which files are exactly contained in the new file, so we have to record all unseq files - // in the new file - for (int i = 0; i < targetResources.size(); i++) { - TsFileResource targetResource = targetResources.get(i); - // remove the target file been deleted from list - if (!targetResource.getTsFile().exists()) { - targetResources.remove(i--); - continue; - } - for (TsFileResource unseqResource : unseqResources) { - targetResource.updatePlanIndexes(unseqResource); - } - for (TsFileResource seqResource : seqResources) { - targetResource.updatePlanIndexes(seqResource); - } - } - } - /** * Update the targetResource. Move tmp target file to target file and serialize * xxx.tsfile.resource. @@ -398,13 +150,4 @@ public static void deleteCompactionModsFile( } } } - - private static void checkThreadInterrupted(List tsFileResource) - throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException( - String.format( - "[Compaction] compaction for target file %s abort", tsFileResource.toString())); - } - } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index 74f88a3190cbe..1ceba99b38fb0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -23,6 +23,8 @@ import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -55,6 +57,7 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio protected TsFileResourceList seqTsFileResourceList; protected TsFileResourceList unseqTsFileResourceList; private File logFile; + private AbstractCompactionPerformer performer; private List targetTsfileResourceList; private List holdReadLockList = new ArrayList<>(); @@ -144,8 +147,12 @@ private void executeCompaction() // restart recovery compactionLogger.close(); - CompactionUtils.compact( - selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, targetTsfileResourceList); + performer = + new ReadPointCompactionPerformer( + selectedSeqTsFileResourceList, + selectedUnSeqTsFileResourceList, + targetTsfileResourceList); + performer.perform(); CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); CompactionUtils.combineModsInCompaction( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 80be49f4817c4..0b4c7bfa913a7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -20,12 +20,12 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -121,13 +121,14 @@ protected void doCompaction() throws Exception { if (sequence) { performer = new ReadChunkCompactionPerformer(selectedTsFileResourceList, targetTsFileResource); - performer.perform(); } else { - CompactionUtils.compact( - Collections.emptyList(), - selectedTsFileResourceList, - Collections.singletonList(targetTsFileResource)); + performer = + new ReadPointCompactionPerformer( + Collections.emptyList(), + selectedTsFileResourceList, + Collections.singletonList(targetTsFileResource)); } + performer.perform(); InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, fullStorageGroupName); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java index 2345847a5a6be..5dbfa741fd1d4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.compaction.performer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; import java.io.IOException; @@ -34,5 +35,6 @@ public abstract class AbstractCompactionPerformer { protected List seqFiles; protected List unseqFiles; - public abstract void perform() throws IOException, MetadataException, InterruptedException; + public abstract void perform() + throws IOException, MetadataException, StorageEngineException, InterruptedException; } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java index 56b61c623ba92..67268a95160a5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; import org.apache.iotdb.db.engine.compaction.inner.utils.SingleSeriesCompactionExecutor; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.metadata.idtable.IDTableManager; @@ -54,7 +55,8 @@ public ReadChunkCompactionPerformer(List sourceFiles, TsFileReso } @Override - public void perform() throws IOException, MetadataException, InterruptedException { + public void perform() + throws IOException, MetadataException, InterruptedException, StorageEngineException { try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles); TsFileIOWriter writer = new TsFileIOWriter(targetResource.getTsFile())) { while (deviceIterator.hasNextDevice()) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java index 97f692129ca29..1ad1af53c5ad6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java @@ -18,9 +18,280 @@ */ package org.apache.iotdb.db.engine.compaction.performer; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; +import org.apache.iotdb.db.engine.compaction.writer.AbstractCompactionWriter; +import org.apache.iotdb.db.engine.compaction.writer.CrossSpaceCompactionWriter; +import org.apache.iotdb.db.engine.compaction.writer.InnerSpaceCompactionWriter; +import org.apache.iotdb.db.engine.querycontext.QueryDataSource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; +import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.metadata.idtable.IDTableManager; +import org.apache.iotdb.db.metadata.path.AlignedPath; +import org.apache.iotdb.db.metadata.path.MeasurementPath; +import org.apache.iotdb.db.metadata.path.PartialPath; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.control.QueryResourceManager; +import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader; +import org.apache.iotdb.db.service.IoTDB; +import org.apache.iotdb.db.utils.QueryUtils; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.BatchData; +import org.apache.iotdb.tsfile.read.reader.IBatchReader; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; public class ReadPointCompactionPerformer extends AbstractCompactionPerformer { + private Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); + + private List targetFiles; + + public ReadPointCompactionPerformer( + List seqFiles, + List unseqFiles, + List targetFiles) { + this.seqFiles = seqFiles; + this.unseqFiles = unseqFiles; + this.targetFiles = targetFiles; + } + @Override - public void perform() throws IOException {} + public void perform() + throws IOException, MetadataException, StorageEngineException, InterruptedException { + long queryId = QueryResourceManager.getInstance().assignCompactionQueryId(); + QueryContext queryContext = new QueryContext(queryId); + QueryDataSource queryDataSource = new QueryDataSource(seqFiles, unseqFiles); + QueryResourceManager.getInstance() + .getQueryFileManager() + .addUsedFilesForQuery(queryId, queryDataSource); + + try (AbstractCompactionWriter compactionWriter = + getCompactionWriter(seqFiles, unseqFiles, targetFiles)) { + // Do not close device iterator, because tsfile reader is managed by FileReaderManager. + MultiTsFileDeviceIterator deviceIterator = + new MultiTsFileDeviceIterator(seqFiles, unseqFiles); + while (deviceIterator.hasNextDevice()) { + checkThreadInterrupted(); + Pair deviceInfo = deviceIterator.nextDevice(); + String device = deviceInfo.left; + boolean isAligned = deviceInfo.right; + QueryUtils.fillOrderIndexes(queryDataSource, device, true); + + if (isAligned) { + compactAlignedSeries( + device, deviceIterator, compactionWriter, queryContext, queryDataSource); + } else { + compactNonAlignedSeries( + device, deviceIterator, compactionWriter, queryContext, queryDataSource); + } + } + + compactionWriter.endFile(); + updatePlanIndexes(targetFiles, seqFiles, unseqFiles); + } finally { + QueryResourceManager.getInstance().endQuery(queryId); + } + } + + private void compactAlignedSeries( + String device, + MultiTsFileDeviceIterator deviceIterator, + AbstractCompactionWriter compactionWriter, + QueryContext queryContext, + QueryDataSource queryDataSource) + throws IOException, MetadataException { + MultiTsFileDeviceIterator.AlignedMeasurementIterator alignedMeasurementIterator = + deviceIterator.iterateAlignedSeries(device); + Set allMeasurements = alignedMeasurementIterator.getAllMeasurements(); + List measurementSchemas = new ArrayList<>(); + for (String measurement : allMeasurements) { + try { + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + measurementSchemas.add(IDTableManager.getInstance().getSeriesSchema(device, measurement)); + } else { + measurementSchemas.add( + IoTDB.schemaProcessor.getSeriesSchema(new PartialPath(device, measurement))); + } + } catch (PathNotExistException e) { + LOGGER.info("A deleted path is skipped: {}", e.getMessage()); + } + } + if (measurementSchemas.isEmpty()) { + return; + } + List existedMeasurements = + measurementSchemas.stream() + .map(IMeasurementSchema::getMeasurementId) + .collect(Collectors.toList()); + IBatchReader dataBatchReader = + constructReader( + device, + existedMeasurements, + measurementSchemas, + allMeasurements, + queryContext, + queryDataSource, + true); + + if (dataBatchReader.hasNextBatch()) { + // chunkgroup is serialized only when at least one timeseries under this device has data + compactionWriter.startChunkGroup(device, true); + compactionWriter.startMeasurement(measurementSchemas); + writeWithReader(compactionWriter, dataBatchReader); + compactionWriter.endMeasurement(); + compactionWriter.endChunkGroup(); + } + } + + private void compactNonAlignedSeries( + String device, + MultiTsFileDeviceIterator deviceIterator, + AbstractCompactionWriter compactionWriter, + QueryContext queryContext, + QueryDataSource queryDataSource) + throws MetadataException, IOException { + boolean hasStartChunkGroup = false; + MultiTsFileDeviceIterator.MeasurementIterator measurementIterator = + deviceIterator.iterateNotAlignedSeries(device, false); + Set allMeasurements = measurementIterator.getAllMeasurements(); + for (String measurement : allMeasurements) { + List measurementSchemas = new ArrayList<>(); + try { + if (IoTDBDescriptor.getInstance().getConfig().isEnableIDTable()) { + measurementSchemas.add(IDTableManager.getInstance().getSeriesSchema(device, measurement)); + } else { + measurementSchemas.add( + IoTDB.schemaProcessor.getSeriesSchema(new PartialPath(device, measurement))); + } + } catch (PathNotExistException e) { + LOGGER.info("A deleted path is skipped: {}", e.getMessage()); + continue; + } + + IBatchReader dataBatchReader = + constructReader( + device, + Collections.singletonList(measurement), + measurementSchemas, + allMeasurements, + queryContext, + queryDataSource, + false); + + if (dataBatchReader.hasNextBatch()) { + if (!hasStartChunkGroup) { + // chunkgroup is serialized only when at least one timeseries under this device has + // data + compactionWriter.startChunkGroup(device, false); + hasStartChunkGroup = true; + } + compactionWriter.startMeasurement(measurementSchemas); + writeWithReader(compactionWriter, dataBatchReader); + compactionWriter.endMeasurement(); + } + } + + if (hasStartChunkGroup) { + compactionWriter.endChunkGroup(); + } + } + + /** + * @param measurementIds if device is aligned, then measurementIds contain all measurements. If + * device is not aligned, then measurementIds only contain one measurement. + */ + private IBatchReader constructReader( + String deviceId, + List measurementIds, + List measurementSchemas, + Set allSensors, + QueryContext queryContext, + QueryDataSource queryDataSource, + boolean isAlign) + throws IllegalPathException { + PartialPath seriesPath; + TSDataType tsDataType; + if (isAlign) { + seriesPath = new AlignedPath(deviceId, measurementIds, measurementSchemas); + tsDataType = TSDataType.VECTOR; + } else { + seriesPath = new MeasurementPath(deviceId, measurementIds.get(0), measurementSchemas.get(0)); + tsDataType = measurementSchemas.get(0).getType(); + } + return new SeriesRawDataBatchReader( + seriesPath, allSensors, tsDataType, queryContext, queryDataSource, null, null, null, true); + } + + private void writeWithReader(AbstractCompactionWriter writer, IBatchReader reader) + throws IOException { + while (reader.hasNextBatch()) { + BatchData batchData = reader.nextBatch(); + while (batchData.hasCurrent()) { + writer.write(batchData.currentTime(), batchData.currentValue()); + batchData.next(); + } + } + } + + private AbstractCompactionWriter getCompactionWriter( + List seqFileResources, + List unseqFileResources, + List targetFileResources) + throws IOException { + if (!seqFileResources.isEmpty() && !unseqFileResources.isEmpty()) { + // cross space + return new CrossSpaceCompactionWriter(targetFileResources, seqFileResources); + } else { + // inner space + return new InnerSpaceCompactionWriter(targetFileResources.get(0)); + } + } + + private static void updatePlanIndexes( + List targetResources, + List seqResources, + List unseqResources) { + // as the new file contains data of other files, track their plan indexes in the new file + // so that we will be able to compare data across different IoTDBs that share the same index + // generation policy + // however, since the data of unseq files are mixed together, we won't be able to know + // which files are exactly contained in the new file, so we have to record all unseq files + // in the new file + for (int i = 0; i < targetResources.size(); i++) { + TsFileResource targetResource = targetResources.get(i); + // remove the target file been deleted from list + if (!targetResource.getTsFile().exists()) { + targetResources.remove(i--); + continue; + } + for (TsFileResource unseqResource : unseqResources) { + targetResource.updatePlanIndexes(unseqResource); + } + for (TsFileResource seqResource : seqResources) { + targetResource.updatePlanIndexes(seqResource); + } + } + } + + private void checkThreadInterrupted() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException( + String.format( + "[Compaction] compaction for target file %s abort", targetFiles.toString())); + } + } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionUtilsTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java similarity index 98% rename from server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionUtilsTest.java rename to server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java index a6aab11e1c625..fb163b33c03e4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionUtilsTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; @@ -56,7 +57,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; import static org.junit.Assert.assertEquals; -public class CompactionUtilsTest extends AbstractCompactionTest { +public class ReadPointCompactionPerformerTest extends AbstractCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @Before @@ -115,7 +116,7 @@ public void testSeqInnerSpaceCompactionWithSameTimeseries() List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); tsFilesReader = @@ -200,7 +201,7 @@ public void testSeqInnerSpaceCompactionWithDifferentTimeseries() List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 5; i++) { @@ -288,7 +289,7 @@ public void testUnSeqInnerSpaceCompactionWithSameTimeseries() List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 2; i++) { @@ -393,7 +394,7 @@ public void testUnSeqInnerSpaceCompactionWithDifferentTimeseries() List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 9; i++) { @@ -528,7 +529,7 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInTimeseries() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 5; i++) { @@ -653,7 +654,7 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInDevice() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 5; i++) { @@ -764,7 +765,7 @@ public void testUnSeqInnerSpaceCompactionWithAllDataDeletedInTargetFile() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = 0; i < 5; i++) { @@ -844,7 +845,7 @@ public void testAlignedSeqInnerSpaceCompactionWithSameTimeseries() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -952,7 +953,7 @@ public void testAlignedSeqInnerSpaceCompactionWithDifferentTimeseriesAndEmptyPag } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1070,7 +1071,7 @@ public void testAlignedSeqInnerSpaceCompactionWithDifferentTimeseriesAndEmptyChu } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(seqResources, true); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1194,7 +1195,7 @@ public void testAlignedUnSeqInnerSpaceCompactionWithEmptyChunkAndEmptyPage() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1367,7 +1368,7 @@ public void testAlignedUnSeqInnerSpaceCompactionWithAllDataDeletedInTimeseries() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1521,7 +1522,7 @@ public void testAlignedUnSeqInnerSpaceCompactionWithAllDataDeletedInDevice() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1627,7 +1628,7 @@ public void testAlignedUnSeqInnerSpaceCompactionWithSameTimeseries() } List targetResources = CompactionFileGeneratorUtils.getInnerCompactionTargetTsFileResources(unseqResources, false); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, true, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -1720,7 +1721,7 @@ public void testCrossSpaceCompactionWithSameTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); tsFilesReader = @@ -1833,7 +1834,7 @@ public void testCrossSpaceCompactionWithDifferentTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); Map measurementMaxTime = new HashMap<>(); @@ -1999,7 +2000,7 @@ public void testCrossSpaceCompactionWithAllDataDeletedInTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); Map measurementMaxTime = new HashMap<>(); @@ -2162,7 +2163,7 @@ public void testCrossSpaceCompactionWithAllDataDeletedInDevice() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); Map measurementMaxTime = new HashMap<>(); @@ -2317,7 +2318,7 @@ public void testCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); Map measurementMaxTime = new HashMap<>(); @@ -2474,7 +2475,7 @@ public void testCrossSpaceCompactionWithAllDataDeletedInDeviceInSeqFiles() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); Map measurementMaxTime = new HashMap<>(); @@ -2603,7 +2604,7 @@ public void testAlignedCrossSpaceCompactionWithSameTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); tsFilesReader = @@ -2730,7 +2731,7 @@ public void testAlignedCrossSpaceCompactionWithDifferentTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -2930,7 +2931,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInTimeseries() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -3136,7 +3137,7 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -3346,7 +3347,7 @@ public void testAlignedCrossSpaceCompactionWithFileTimeIndexResource() List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = TsFileGeneratorUtils.getAlignDeviceOffset(); @@ -3431,7 +3432,7 @@ public void testCrossSpaceCompactionWithNewDeviceInUnseqFile() { List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); } catch (MetadataException | IOException @@ -3453,7 +3454,7 @@ public void testCrossSpaceCompactionWithDeviceMaxTimeLaterInUnseqFile() { List targetResources = CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = 0; i < 4; i++) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index 06e19fbf5782a..80691b28800a4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.ModificationFile; @@ -90,7 +91,7 @@ public void testHandleWithAllSourceFilesExisted() throws Exception { compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); compactionLogger.close(); CompactionExceptionHandler.handleException( COMPACTION_TEST_SG, @@ -161,7 +162,7 @@ public void testHandleWithAllSourceFilesExistedAndTargetFilesMoved() throws Exce compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); compactionLogger.close(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); CompactionExceptionHandler.handleException( @@ -233,7 +234,7 @@ public void testHandleWithSomeSourceFilesExisted() throws Exception { compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (TsFileResource resource : seqResources) { tsFileManager.getSequenceListByTimePartition(0).remove(resource); @@ -315,7 +316,7 @@ public void testHandleWithoutAllSourceFilesAndModFilesExist() throws Exception { compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); compactionLogger.close(); for (int i = 0; i < seqResources.size(); i++) { @@ -429,7 +430,7 @@ public void testHandleWithAllSourcesFileAndCompactonModFileExist() throws Except compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); compactionLogger.close(); for (int i = 0; i < seqResources.size(); i++) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java index 327a15083967a..107789ca74c43 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.ModificationFile; @@ -87,7 +88,7 @@ public void testRecoverWithAllSourceFilesExisted() throws Exception { compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); compactionLogger.close(); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, false) .doCompaction(); @@ -147,7 +148,7 @@ public void testRecoverWithAllSourceFilesExistedAndSomeTargetFilesNotExist() thr compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); // Target files may not exist for (int i = 0; i < targetResources.size(); i++) { if (i < 2) { @@ -215,7 +216,7 @@ public void testRecoverWithAllSourceFilesExistedAndTargetFilesMoved() throws Exc compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); compactionLogger.close(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, false) @@ -276,7 +277,7 @@ public void testRecoverWithSomeSourceFilesExisted() throws Exception { compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); seqResources.get(0).getTsFile().delete(); compactionLogger.close(); @@ -338,7 +339,7 @@ public void testRecoverWithoutAllSourceFilesAndModFilesExist() throws Exception compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); compactionLogger.close(); for (int i = 0; i < seqResources.size(); i++) { @@ -433,7 +434,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExist() throws Excep compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); compactionLogger.close(); CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { @@ -540,7 +541,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExistAndSomeTargetFi compactionLogger.logFiles(targetResources, STR_TARGET_FILES); compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); - CompactionUtils.compact(seqResources, unseqResources, targetResources); + new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources).perform(); // Target files may not exist for (int i = 0; i < targetResources.size(); i++) { if (i < 2) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java index 248553497d1f5..8d08bb4eda208 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java @@ -335,7 +335,7 @@ public void testDeserializePage() throws MetadataException, IOException { } } } - } catch (InterruptedException e) { + } catch (InterruptedException | StorageEngineException e) { e.printStackTrace(); } finally { IoTDBDescriptor.getInstance() @@ -348,7 +348,8 @@ public void testDeserializePage() throws MetadataException, IOException { } @Test - public void testAppendPage() throws IOException, MetadataException, InterruptedException { + public void testAppendPage() + throws IOException, MetadataException, InterruptedException, StorageEngineException { for (int toMergeFileNum : toMergeFileNums) { for (CompactionTimeseriesType compactionTimeseriesType : compactionTimeseriesTypes) { @@ -610,7 +611,8 @@ public void testAppendPage() throws IOException, MetadataException, InterruptedE } @Test - public void testAppendChunk() throws IOException, IllegalPathException, MetadataException { + public void testAppendChunk() + throws IOException, IllegalPathException, MetadataException, StorageEngineException { long prevChunkPointNumLowerBoundInCompaction = IoTDBDescriptor.getInstance().getConfig().getChunkPointNumLowerBoundInCompaction(); IoTDBDescriptor.getInstance().getConfig().setChunkPointNumLowerBoundInCompaction(1); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java index 42419d9411ce6..4f599221f967c 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; @@ -366,10 +367,11 @@ public void test() timeValuePair.getTimestamp() >= 250L && timeValuePair.getTimestamp() <= 300L); } - CompactionUtils.compact( - Collections.emptyList(), - toMergeResources, - Collections.singletonList(targetTsFileResource)); + new ReadPointCompactionPerformer( + Collections.emptyList(), + toMergeResources, + Collections.singletonList(targetTsFileResource)) + .perform(); CompactionUtils.moveTargetFile( Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); InnerSpaceCompactionUtils.combineModsInCompaction( diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java index a6b8914d5b653..5a3cf75f4eea0 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java @@ -73,7 +73,8 @@ public void tearDown() throws IOException, StorageEngineException { } @Test - public void testCompact() throws IOException, MetadataException, InterruptedException { + public void testCompact() + throws IOException, MetadataException, InterruptedException, StorageEngineException { TsFileResource targetTsFileResource = new TsFileResource( new File( From 8a8c129e639f4c7c8610683ce7286b576ef52d28 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 1 Apr 2022 17:45:32 +0800 Subject: [PATCH 06/22] combine CompactionUtils and InnerSpaceCompactionUtils together --- .../db/engine/compaction/CompactionUtils.java | 96 +++++++++- .../RewriteCrossSpaceCompactionSelector.java | 4 +- .../task/RewriteCrossSpaceCompactionTask.java | 2 +- .../sizetiered/SizeTieredCompactionTask.java | 12 +- .../utils/InnerSpaceCompactionUtils.java | 165 ------------------ .../CrossSpaceCompactionExceptionTest.java | 4 +- ...ewriteCrossSpaceCompactionRecoverTest.java | 4 +- .../inner/InnerSeqCompactionTest.java | 29 +-- .../InnerSpaceCompactionExceptionTest.java | 25 ++- .../inner/InnerUnseqCompactionTest.java | 4 +- .../ReadChunkCompactionPerformerOldTest.java | 9 +- .../SizeTieredCompactionRecoverTest.java | 27 +-- ...TieredCompactionRecoverCompatibleTest.java | 9 +- .../SizeTieredCompactionRecoverTest.java | 26 ++- 14 files changed, 184 insertions(+), 232 deletions(-) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java index 274295b2d78d2..b6618eb3f8b00 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java @@ -19,10 +19,17 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; @@ -32,15 +39,15 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; /** * This tool can be used to perform inner space or cross space compaction of aligned and non aligned - * timeseries . Currently, we use {@link - * org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils} to speed up if it is - * an inner space compaction. + * timeseries. */ public class CompactionUtils { private static final Logger logger = @@ -92,7 +99,7 @@ private static void moveOneTargetFile( * Collect all the compaction modification files of source files, and combines them as the * modification file of target file. */ - public static void combineModsInCompaction( + public static void combineModsInCrossCompaction( List seqResources, List unseqResources, List targetResources) @@ -112,6 +119,31 @@ public static void combineModsInCompaction( } } + /** + * Collect all the compaction modification files of source files, and combines them as the + * modification file of target file. + */ + public static void combineModsInInnerCompaction( + Collection sourceFiles, TsFileResource targetTsFile) throws IOException { + List modifications = new ArrayList<>(); + for (TsFileResource mergeTsFile : sourceFiles) { + try (ModificationFile sourceCompactionModificationFile = + ModificationFile.getCompactionMods(mergeTsFile)) { + modifications.addAll(sourceCompactionModificationFile.getModifications()); + } + } + if (!modifications.isEmpty()) { + try (ModificationFile modificationFile = ModificationFile.getNormalMods(targetTsFile)) { + for (Modification modification : modifications) { + // we have to set modification offset to MAX_VALUE, as the offset of source chunk may + // change after compaction + modification.setFileOffset(Long.MAX_VALUE); + modificationFile.write(modification); + } + } + } + } + private static void updateOneTargetMods( TsFileResource targetFile, TsFileResource seqFile, List unseqFiles) throws IOException { @@ -150,4 +182,60 @@ public static void deleteCompactionModsFile( } } } + + public static boolean deleteTsFilesInDisk( + Collection mergeTsFiles, String storageGroupName) { + logger.info("{} [Compaction] Compaction starts to delete real file ", storageGroupName); + boolean result = true; + for (TsFileResource mergeTsFile : mergeTsFiles) { + if (!deleteTsFile(mergeTsFile)) { + result = false; + } + logger.info( + "{} [Compaction] delete TsFile {}", storageGroupName, mergeTsFile.getTsFilePath()); + } + return result; + } + + public static boolean deleteTsFile(TsFileResource seqFile) { + try { + FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath()); + seqFile.setStatus(TsFileResourceStatus.DELETED); + seqFile.delete(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return false; + } + return true; + } + + /** Delete all modification files for source files */ + public static void deleteModificationForSourceFile( + Collection sourceFiles, String storageGroupName) throws IOException { + logger.info("{} [Compaction] Start to delete modifications of source files", storageGroupName); + for (TsFileResource tsFileResource : sourceFiles) { + ModificationFile compactionModificationFile = + ModificationFile.getCompactionMods(tsFileResource); + if (compactionModificationFile.exists()) { + compactionModificationFile.remove(); + } + + ModificationFile normalModification = ModificationFile.getNormalMods(tsFileResource); + if (normalModification.exists()) { + normalModification.remove(); + } + } + } + + public static ICrossSpaceMergeFileSelector getCrossSpaceFileSelector( + long budget, CrossSpaceCompactionResource resource) { + CrossCompactionStrategy strategy = + IoTDBDescriptor.getInstance().getConfig().getCrossCompactionStrategy(); + switch (strategy) { + case REWRITE_COMPACTION: + return new RewriteCompactionFileSelector(resource, budget); + default: + throw new UnsupportedOperationException("Unknown CrossSpaceFileStrategy " + strategy); + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 2575d51d8785e..7150b758e8b1f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -22,10 +22,10 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -91,7 +91,7 @@ public void selectAndSubmit() { new CrossSpaceCompactionResource(seqFileList, unSeqFileList, timeLowerBound); ICrossSpaceMergeFileSelector fileSelector = - InnerSpaceCompactionUtils.getCrossSpaceFileSelector(budget, mergeResource); + CompactionUtils.getCrossSpaceFileSelector(budget, mergeResource); try { List[] mergeFiles = fileSelector.select(); // avoid pending tasks holds the metadata and streams diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index 1ceba99b38fb0..8104b1be4d2eb 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -155,7 +155,7 @@ private void executeCompaction() performer.perform(); CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); - CompactionUtils.combineModsInCompaction( + CompactionUtils.combineModsInCrossCompaction( selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, targetTsfileResourceList); // update tsfile resource in memory diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 0b4c7bfa913a7..78d9e4641e291 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -20,8 +20,8 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; @@ -130,10 +130,11 @@ protected void doCompaction() throws Exception { } performer.perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, fullStorageGroupName); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, fullStorageGroupName); LOGGER.info("{} [SizeTiredCompactionTask] start to rename mods file", fullStorageGroupName); - InnerSpaceCompactionUtils.combineModsInCompaction( + CompactionUtils.combineModsInInnerCompaction( selectedTsFileResourceList, targetTsFileResource); if (Thread.currentThread().isInterrupted()) { @@ -182,9 +183,8 @@ protected void doCompaction() throws Exception { LOGGER.info( "{} [Compaction] compaction finish, start to delete old files", fullStorageGroupName); // delete the old files - InnerSpaceCompactionUtils.deleteTsFilesInDisk( - selectedTsFileResourceList, fullStorageGroupName); - InnerSpaceCompactionUtils.deleteModificationForSourceFile( + CompactionUtils.deleteTsFilesInDisk(selectedTsFileResourceList, fullStorageGroupName); + CompactionUtils.deleteModificationForSourceFile( selectedTsFileResourceList, fullStorageGroupName); long costTime = System.currentTimeMillis() - startTime; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java deleted file mode 100644 index 57342a0110b3f..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/InnerSpaceCompactionUtils.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.inner.utils; - -import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; -import org.apache.iotdb.db.engine.modification.Modification; -import org.apache.iotdb.db.engine.modification.ModificationFile; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; -import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.tsfile.common.constant.TsFileConstant; -import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -public class InnerSpaceCompactionUtils { - - private static final Logger logger = - LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - - private InnerSpaceCompactionUtils() { - throw new IllegalStateException("Utility class"); - } - - public static boolean deleteTsFilesInDisk( - Collection mergeTsFiles, String storageGroupName) { - logger.info("{} [Compaction] Compaction starts to delete real file ", storageGroupName); - boolean result = true; - for (TsFileResource mergeTsFile : mergeTsFiles) { - if (!deleteTsFile(mergeTsFile)) { - result = false; - } - logger.info( - "{} [Compaction] delete TsFile {}", storageGroupName, mergeTsFile.getTsFilePath()); - } - return result; - } - - /** Delete all modification files for source files */ - public static void deleteModificationForSourceFile( - Collection sourceFiles, String storageGroupName) throws IOException { - logger.info("{} [Compaction] Start to delete modifications of source files", storageGroupName); - for (TsFileResource tsFileResource : sourceFiles) { - ModificationFile compactionModificationFile = - ModificationFile.getCompactionMods(tsFileResource); - if (compactionModificationFile.exists()) { - compactionModificationFile.remove(); - } - - ModificationFile normalModification = ModificationFile.getNormalMods(tsFileResource); - if (normalModification.exists()) { - normalModification.remove(); - } - } - } - - /** - * Collect all the compaction modification files of source files, and combines them as the - * modification file of target file. - */ - public static void combineModsInCompaction( - Collection mergeTsFiles, TsFileResource targetTsFile) throws IOException { - List modifications = new ArrayList<>(); - for (TsFileResource mergeTsFile : mergeTsFiles) { - try (ModificationFile sourceCompactionModificationFile = - ModificationFile.getCompactionMods(mergeTsFile)) { - modifications.addAll(sourceCompactionModificationFile.getModifications()); - } - } - if (!modifications.isEmpty()) { - try (ModificationFile modificationFile = ModificationFile.getNormalMods(targetTsFile)) { - for (Modification modification : modifications) { - // we have to set modification offset to MAX_VALUE, as the offset of source chunk may - // change after compaction - modification.setFileOffset(Long.MAX_VALUE); - modificationFile.write(modification); - } - } - } - } - - public static boolean deleteTsFile(TsFileResource seqFile) { - try { - FileReaderManager.getInstance().closeFileAndRemoveReader(seqFile.getTsFilePath()); - seqFile.setStatus(TsFileResourceStatus.DELETED); - seqFile.delete(); - } catch (IOException e) { - logger.error(e.getMessage(), e); - return false; - } - return true; - } - - public static ICrossSpaceMergeFileSelector getCrossSpaceFileSelector( - long budget, CrossSpaceCompactionResource resource) { - CrossCompactionStrategy strategy = - IoTDBDescriptor.getInstance().getConfig().getCrossCompactionStrategy(); - switch (strategy) { - case REWRITE_COMPACTION: - return new RewriteCompactionFileSelector(resource, budget); - default: - throw new UnsupportedOperationException("Unknown CrossSpaceFileStrategy " + strategy); - } - } - - /** - * Update the targetResource. Move xxx.target to xxx.tsfile and serialize xxx.tsfile.resource . - * - * @param targetResource the old tsfile to be moved, which is xxx.target - */ - public static void moveTargetFile(TsFileResource targetResource, String fullStorageGroupName) - throws IOException { - if (!targetResource.getTsFilePath().endsWith(IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)) { - logger.warn( - "{} [Compaction] Tmp target tsfile {} should be end with {}", - fullStorageGroupName, - targetResource.getTsFilePath(), - IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX); - return; - } - File oldFile = targetResource.getTsFile(); - - // move TsFile and delete old tsfile - String newFilePath = - targetResource - .getTsFilePath() - .replace(IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX, TsFileConstant.TSFILE_SUFFIX); - File newFile = new File(newFilePath); - FSFactoryProducer.getFSFactory().moveFile(oldFile, newFile); - - // serialize xxx.tsfile.resource - targetResource.setFile(newFile); - targetResource.serialize(); - targetResource.close(); - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index 80691b28800a4..cb79feb73fcdb 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -335,7 +335,7 @@ public void testHandleWithoutAllSourceFilesAndModFilesExist() throws Exception { CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); } - CompactionUtils.combineModsInCompaction(seqResources, unseqResources, targetResources); + CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); for (TsFileResource resource : seqResources) { tsFileManager.getSequenceListByTimePartition(0).remove(resource); } @@ -449,7 +449,7 @@ public void testHandleWithAllSourcesFileAndCompactonModFileExist() throws Except CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); } - CompactionUtils.combineModsInCompaction(seqResources, unseqResources, targetResources); + CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); CompactionExceptionHandler.handleException( COMPACTION_TEST_SG, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java index 107789ca74c43..b5f4a7f152e18 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java @@ -358,7 +358,7 @@ public void testRecoverWithoutAllSourceFilesAndModFilesExist() throws Exception CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); } - CompactionUtils.combineModsInCompaction(seqResources, unseqResources, targetResources); + CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); seqResources.get(0).remove(); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, false) @@ -453,7 +453,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExist() throws Excep CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); } - CompactionUtils.combineModsInCompaction(seqResources, unseqResources, targetResources); + CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, false) .doCompaction(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java index 8d08bb4eda208..c1a79b8d12199 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; @@ -37,6 +37,7 @@ import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.read.TimeValuePair; @@ -113,7 +114,7 @@ public void tearDown() throws IOException, StorageEngineException { } @Test - public void testDeserializePage() throws MetadataException, IOException { + public void testDeserializePage() throws MetadataException, IOException, WriteProcessException { long chunkSizeLowerBoundInCompaction = IoTDBDescriptor.getInstance().getConfig().getChunkSizeLowerBoundInCompaction(); @@ -229,9 +230,9 @@ public void testDeserializePage() throws MetadataException, IOException { && timeValuePair.getTimestamp() <= 300L); } new ReadChunkCompactionPerformer(sourceResources, targetTsFileResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); - InnerSpaceCompactionUtils.combineModsInCompaction( - sourceResources, targetTsFileResource); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); + CompactionUtils.combineModsInInnerCompaction(sourceResources, targetTsFileResource); List targetTsFileResources = new ArrayList<>(); targetTsFileResources.add(targetTsFileResource); // check data @@ -349,7 +350,8 @@ public void testDeserializePage() throws MetadataException, IOException { @Test public void testAppendPage() - throws IOException, MetadataException, InterruptedException, StorageEngineException { + throws IOException, MetadataException, InterruptedException, StorageEngineException, + WriteProcessException { for (int toMergeFileNum : toMergeFileNums) { for (CompactionTimeseriesType compactionTimeseriesType : compactionTimeseriesTypes) { @@ -455,9 +457,9 @@ public void testAppendPage() timeValuePair.getTimestamp() >= 250L && timeValuePair.getTimestamp() <= 300L); } new ReadChunkCompactionPerformer(toMergeResources, targetTsFileResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); - InnerSpaceCompactionUtils.combineModsInCompaction( - toMergeResources, targetTsFileResource); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); + CompactionUtils.combineModsInInnerCompaction(toMergeResources, targetTsFileResource); List targetTsFileResources = new ArrayList<>(); targetTsFileResources.add(targetTsFileResource); CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsFileResources); @@ -612,7 +614,8 @@ public void testAppendPage() @Test public void testAppendChunk() - throws IOException, IllegalPathException, MetadataException, StorageEngineException { + throws IOException, IllegalPathException, MetadataException, StorageEngineException, + WriteProcessException { long prevChunkPointNumLowerBoundInCompaction = IoTDBDescriptor.getInstance().getConfig().getChunkPointNumLowerBoundInCompaction(); IoTDBDescriptor.getInstance().getConfig().setChunkPointNumLowerBoundInCompaction(1); @@ -731,9 +734,9 @@ public void testAppendChunk() && timeValuePair.getTimestamp() <= 300L); } new ReadChunkCompactionPerformer(toMergeResources, targetTsFileResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); - InnerSpaceCompactionUtils.combineModsInCompaction( - toMergeResources, targetTsFileResource); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); + CompactionUtils.combineModsInInnerCompaction(toMergeResources, targetTsFileResource); List targetTsFileResources = new ArrayList<>(); targetTsFileResources.add(targetTsFileResource); CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsFileResources); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java index c137f1bddbf70..207d9483b4d77 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; @@ -65,7 +65,8 @@ public void testWhenAllSourceExistsAndTargetNotComplete() throws Exception { compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); try (FileOutputStream os = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = os.getChannel()) { channel.truncate(targetResource.getTsFileSize() - 10); @@ -113,7 +114,8 @@ public void testWhenAllSourceExistsAndTargetComplete() throws Exception { compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); compactionLogger.close(); CompactionExceptionHandler.handleException( COMPACTION_TEST_SG, @@ -157,7 +159,8 @@ public void testWhenSomeSourceLostAndTargetComplete() throws Exception { compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (TsFileResource resource : seqResources) { tsFileManager.getSequenceListByTimePartition(0).remove(resource); } @@ -212,7 +215,8 @@ public void testWhenSomeSourceLostAndTargetNotComplete() throws Exception { compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); seqResources.get(0).remove(); try (FileOutputStream os = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = os.getChannel()) { @@ -265,7 +269,8 @@ public void testHandleWithCompactionMods() throws Exception { compactionLogger.logFiles( Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); deleteMap.put( @@ -273,7 +278,7 @@ public void testHandleWithCompactionMods() throws Exception { new Pair<>(i * ptNum, i * ptNum + 10)); CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), true); } - InnerSpaceCompactionUtils.combineModsInCompaction(seqResources, targetResource); + CompactionUtils.combineModsInInnerCompaction(seqResources, targetResource); seqResources.get(0).remove(); compactionLogger.close(); @@ -339,7 +344,8 @@ public void testHandleWithNormalMods() throws Exception { CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); seqResources.get(0).remove(); compactionLogger.close(); @@ -397,7 +403,8 @@ public void testHandleWithCompactionModsAndNormalMods() throws Exception { CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); deleteMap.put( diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java index 4f599221f967c..de9d62233b0cd 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java @@ -22,7 +22,6 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; @@ -374,8 +373,7 @@ public void test() .perform(); CompactionUtils.moveTargetFile( Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); - InnerSpaceCompactionUtils.combineModsInCompaction( - toMergeResources, targetTsFileResource); + CompactionUtils.combineModsInInnerCompaction(toMergeResources, targetTsFileResource); List targetTsFileResources = new ArrayList<>(); targetTsFileResources.add(targetTsFileResource); CompactionCheckerUtils.checkDataAndResource(sourceData, targetTsFileResources); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java index 5a3cf75f4eea0..40a3b8000e37a 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java @@ -21,12 +21,13 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.tsfile.exception.write.WriteProcessException; import org.apache.iotdb.tsfile.read.TsFileReader; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.Path; @@ -74,7 +75,8 @@ public void tearDown() throws IOException, StorageEngineException { @Test public void testCompact() - throws IOException, MetadataException, InterruptedException, StorageEngineException { + throws IOException, MetadataException, InterruptedException, StorageEngineException, + WriteProcessException { TsFileResource targetTsFileResource = new TsFileResource( new File( @@ -108,7 +110,8 @@ public void testCompact() new File(targetTsFileResource.getTsFilePath().concat(".compaction.log"))); sizeTieredCompactionLogger.logFiles(seqResources, CompactionLogger.STR_SOURCE_FILES); new ReadChunkCompactionPerformer(seqResources, targetTsFileResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); sizeTieredCompactionLogger.close(); Path path = new Path(deviceIds[0], measurementSchemas[0].getMeasurementId()); try (TsFileSequenceReader reader = diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java index 2d3207cde4cdd..12509a2e31bd8 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java @@ -22,8 +22,8 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTest; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; @@ -148,7 +148,8 @@ public void testCompactionRecoverWithUncompletedTargetFileAndLog() throws Except new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) .perform(); compactionLogger.close(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); BufferedReader logReader = new BufferedReader(new FileReader(compactionLogFile)); List logs = new ArrayList<>(); String line; @@ -492,7 +493,8 @@ public void testRecoverWithoutAllSourceFilesExisted() throws Exception { new ReadChunkCompactionPerformer( new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) .perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); // delete one source file seqResources.get(0).remove(); compactionLogger.close(); @@ -564,7 +566,8 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExist() throws Excep compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); deleteMap.put( @@ -573,7 +576,7 @@ public void testRecoverWithAllSourcesFileAndCompactonModFileExist() throws Excep CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } - InnerSpaceCompactionUtils.combineModsInCompaction(seqResources, targetResource); + CompactionUtils.combineModsInInnerCompaction(seqResources, targetResource); compactionLogger.close(); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, logFile, true).doCompaction(); @@ -703,7 +706,8 @@ public void testRecoverWithoutAllSourceFilesExistAndModFiles() throws Exception compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); for (int i = 0; i < seqResources.size(); i++) { Map> deleteMap = new HashMap<>(); deleteMap.put( @@ -712,7 +716,7 @@ public void testRecoverWithoutAllSourceFilesExistAndModFiles() throws Exception CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), true); CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); } - InnerSpaceCompactionUtils.combineModsInCompaction(seqResources, targetResource); + CompactionUtils.combineModsInInnerCompaction(seqResources, targetResource); seqResources.get(0).remove(); compactionLogger.close(); @@ -817,7 +821,8 @@ public void testRecoverCompleteTargetFileAndCompactionLog() throws Exception { new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) .perform(); compactionLogger.close(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); tsFileManager.add(targetTsFileResource, true); new CompactionRecoverTask(COMPACTION_TEST_SG, "0", tsFileManager, compactionLogFile, true) .doCompaction(); @@ -914,7 +919,8 @@ public void testCompactionRecoverWithCompletedTargetFileAndLog() throws Exceptio new ReadChunkCompactionPerformer( new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) .perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); compactionLogger.close(); for (TsFileResource resource : new ArrayList<>(seqResources.subList(0, 3))) { deleteFileIfExists(resource.getTsFile()); @@ -1016,7 +1022,8 @@ public void testCompactionRecoverWithCompletedTargetFile() throws Exception { new ArrayList<>(seqResources.subList(0, 3)), targetTsFileResource) .perform(); compactionLogger.close(); - InnerSpaceCompactionUtils.moveTargetFile(targetTsFileResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, COMPACTION_TEST_SG); deleteFileIfExists(compactionLogFile); for (TsFileResource resource : new ArrayList<>(seqResources.subList(0, 3))) { tsFileManager.remove(resource, true); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java index b25168c6575b4..76b37dd3be384 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.engine.compaction.recover; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; @@ -43,6 +43,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.RandomAccessFile; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -126,7 +127,8 @@ public void testCompatibleWithSomeSourceFilesLostWithFilePath() throws Exception TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources, true); new ReadChunkCompactionPerformer(seqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, "root.compactionTest"); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, "root.compactionTest"); // first source file does not exist seqResources.get(0).delete(); @@ -219,7 +221,8 @@ public void testCompatibleWithSomeSourceFilesLostWithFileInfo() throws Exception TsFileResource targetResource = TsFileNameGenerator.getInnerCompactionTargetFileResource(unseqResources, true); new ReadChunkCompactionPerformer(unseqResources, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, "root.compactionTest"); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, "root.compactionTest"); // first source file does not exist unseqResources.get(0).delete(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java index 7a6ae01b80a3b..8f156a3ed81c7 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; @@ -236,7 +236,8 @@ public void testRecoverWithCompleteTargetFileUsingFileInfo() throws Exception { compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); CompactionRecoverTask recoverTask = new CompactionRecoverTask( COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true); @@ -286,7 +287,8 @@ public void testRecoverWithIncompleteTargetFileUsingFileInfo() throws Exception compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); channel.truncate(targetResource.getTsFile().length() - 100); @@ -341,7 +343,8 @@ public void testRecoverWithCompleteTargetFileUsingFilePath() throws Exception { logger.logFiles(Collections.singletonList(targetResource), CompactionLogger.STR_TARGET_FILES); logger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); CompactionRecoverTask recoverTask = new CompactionRecoverTask( COMPACTION_TEST_SG, "0", tsFileManager, new File(logFilePath), true); @@ -391,7 +394,8 @@ public void testRecoverWithIncompleteTargetFileUsingFilePath() throws Exception compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); channel.truncate(targetResource.getTsFile().length() - 100); @@ -449,7 +453,8 @@ public void testRecoverWithCompleteTargetFileUsingFileInfoAndChangingDataDirs() compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); long sizeOfTargetFile = targetResource.getTsFileSize(); FileUtils.moveDirectory( new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"), @@ -545,7 +550,8 @@ public void testRecoverWithIncompleteTargetFileUsingFileInfoAndChangingDataDirs( compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); channel.truncate(targetResource.getTsFile().length() - 100); @@ -638,7 +644,8 @@ public void testRecoverWithCompleteTargetFileUsingFilePathAndChangingDataDirs() compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); long sizeOfTargetFile = targetResource.getTsFileSize(); FileUtils.moveDirectory( new File(TestConstant.BASE_OUTPUT_PATH + File.separator + "data"), @@ -734,7 +741,8 @@ public void testRecoverWithIncompleteTargetFileUsingFilePathAndChangingDataDirs( compactionLogger.logFiles(Collections.singletonList(targetResource), STR_TARGET_FILES); compactionLogger.close(); new ReadChunkCompactionPerformer(sourceFiles, targetResource).perform(); - InnerSpaceCompactionUtils.moveTargetFile(targetResource, COMPACTION_TEST_SG); + CompactionUtils.moveTargetFile( + Collections.singletonList(targetResource), true, COMPACTION_TEST_SG); FileOutputStream targetStream = new FileOutputStream(targetResource.getTsFile(), true); FileChannel channel = targetStream.getChannel(); channel.truncate(targetResource.getTsFile().length() - 100); From e17d66e83262a3a3528a94bf30e7ff1f612f03b7 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Wed, 6 Apr 2022 16:55:12 +0800 Subject: [PATCH 07/22] reconstruct compaction task structure --- .../MergeFileStrategy.java | 2 +- .../AbstractCrossSpaceCompactionTask.java | 208 +++++++++++++--- .../task/RewriteCrossSpaceCompactionTask.java | 188 +-------------- .../AbstractInnerSpaceCompactionTask.java | 218 +++++++++++++++-- .../sizetiered/SizeTieredCompactionTask.java | 222 +----------------- .../task/AbstractCompactionTask.java | 2 + .../CompactionTaskComparatorTest.java | 6 + 7 files changed, 409 insertions(+), 437 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{cross/rewrite/selector => constant}/MergeFileStrategy.java (92%) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/MergeFileStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java similarity index 92% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/MergeFileStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java index 17328a3d12df5..c280964e1833e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/MergeFileStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.cross.rewrite.selector; +package org.apache.iotdb.db.engine.compaction.constant; public enum MergeFileStrategy { MAX_SERIES_NUM, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java index d8b1bbd6a8290..60be0ae84c7c4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java @@ -19,17 +19,42 @@ package org.apache.iotdb.db.engine.compaction.cross; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; +import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.db.query.control.FileReaderManager; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; +import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; + public abstract class AbstractCrossSpaceCompactionTask extends AbstractCompactionTask { - List selectedSequenceFiles; - List selectedUnsequenceFiles; + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); + protected List selectedSequenceFiles; + protected List selectedUnsequenceFiles; + protected TsFileResourceList seqTsFileResourceList; + protected TsFileResourceList unseqTsFileResourceList; + private File logFile; + protected List targetTsfileResourceList; + protected List holdReadLockList = new ArrayList<>(); + protected List holdWriteLockList = new ArrayList<>(); public AbstractCrossSpaceCompactionTask( String fullStorageGroupName, @@ -41,46 +66,126 @@ public AbstractCrossSpaceCompactionTask( super(fullStorageGroupName, timePartition, tsFileManager, currentTaskNum); this.selectedSequenceFiles = selectedSequenceFiles; this.selectedUnsequenceFiles = selectedUnsequenceFiles; + this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); + this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); } @Override - public void setSourceFilesToCompactionCandidate() { - this.selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.COMPACTION_CANDIDATE)); - this.selectedUnsequenceFiles.forEach( - x -> x.setStatus(TsFileResourceStatus.COMPACTION_CANDIDATE)); - } + protected void doCompaction() throws Exception { + try { + if (!tsFileManager.isAllowCompaction()) { + return; + } + long startTime = System.currentTimeMillis(); + targetTsfileResourceList = + TsFileNameGenerator.getCrossCompactionTargetFileResources(selectedSequenceFiles); - public List getSelectedSequenceFiles() { - return selectedSequenceFiles; - } + if (targetTsfileResourceList.isEmpty() + || selectedSequenceFiles.isEmpty() + || selectedUnsequenceFiles.isEmpty()) { + LOGGER.info( + "{} [Compaction] Cross space compaction file list is empty, end it", + fullStorageGroupName); + return; + } - public List getSelectedUnsequenceFiles() { - return selectedUnsequenceFiles; - } + LOGGER.info( + "{} [Compaction] CrossSpaceCompactionTask start. Sequence files : {}, unsequence files : {}", + fullStorageGroupName, + selectedSequenceFiles, + selectedUnsequenceFiles); + logFile = + new File( + selectedSequenceFiles.get(0).getTsFile().getParent() + + File.separator + + targetTsfileResourceList.get(0).getTsFile().getName() + + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); - @Override - public boolean checkValidAndSetMerging() { - for (TsFileResource resource : selectedSequenceFiles) { - if (resource.isCompacting() || !resource.isClosed() || !resource.getTsFile().exists()) { - return false; - } - } + try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) { + // print the path of the temporary file first for priority check during recovery + compactionLogger.logFiles(selectedSequenceFiles, STR_SOURCE_FILES); + compactionLogger.logFiles(selectedUnsequenceFiles, STR_SOURCE_FILES); + compactionLogger.logFiles(targetTsfileResourceList, STR_TARGET_FILES); + // indicates that the cross compaction is complete and the result can be reused during a + // restart recovery + compactionLogger.close(); - for (TsFileResource resource : selectedUnsequenceFiles) { - if (resource.isCompacting() || !resource.isClosed() || !resource.getTsFile().exists()) { - return false; + performCompaction(); + + CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); + CompactionUtils.combineModsInCrossCompaction( + selectedSequenceFiles, selectedUnsequenceFiles, targetTsfileResourceList); + + // update tsfile resource in memory + tsFileManager.replace( + selectedSequenceFiles, + selectedUnsequenceFiles, + targetTsfileResourceList, + timePartition, + true); + + releaseReadAndLockWrite(selectedSequenceFiles); + releaseReadAndLockWrite(selectedUnsequenceFiles); + + deleteOldFiles(selectedSequenceFiles); + deleteOldFiles(selectedUnsequenceFiles); + CompactionUtils.deleteCompactionModsFile(selectedSequenceFiles, selectedUnsequenceFiles); + + if (logFile.exists()) { + FileUtils.delete(logFile); + } + LOGGER.info( + "{} [Compaction] CrossSpaceCompactionTask Costs {} s", + fullStorageGroupName, + (System.currentTimeMillis() - startTime) / 1000); } + } catch (Throwable throwable) { + // catch throwable instead of exception to handle OOM errors + LOGGER.error("Meet errors in cross space compaction, {}", throwable.getMessage()); + CompactionExceptionHandler.handleException( + fullStorageGroupName, + logFile, + targetTsfileResourceList, + selectedSequenceFiles, + selectedUnsequenceFiles, + tsFileManager, + timePartition, + false, + true); + throw throwable; + } finally { + releaseAllLock(); } + } - for (TsFileResource resource : selectedSequenceFiles) { - resource.setStatus(TsFileResourceStatus.COMPACTING); + private void releaseAllLock() { + selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); + for (TsFileResource tsFileResource : holdReadLockList) { + tsFileResource.readUnlock(); + tsFileResource.setStatus(TsFileResourceStatus.CLOSED); } - - for (TsFileResource resource : selectedUnsequenceFiles) { - resource.setStatus(TsFileResourceStatus.COMPACTING); + for (TsFileResource tsFileResource : holdWriteLockList) { + tsFileResource.writeUnlock(); + tsFileResource.setStatus(TsFileResourceStatus.CLOSED); } + holdReadLockList.clear(); + holdWriteLockList.clear(); + } - return true; + public List getSelectedSequenceFiles() { + return selectedSequenceFiles; + } + + @Override + public void setSourceFilesToCompactionCandidate() { + this.selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.COMPACTION_CANDIDATE)); + this.selectedUnsequenceFiles.forEach( + x -> x.setStatus(TsFileResourceStatus.COMPACTION_CANDIDATE)); + } + + public List getSelectedUnsequenceFiles() { + return selectedUnsequenceFiles; } @Override @@ -101,4 +206,49 @@ public void resetCompactionCandidateStatusForAllSourceFiles() { selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); } + + private void deleteOldFiles(List tsFileResourceList) throws IOException { + for (TsFileResource tsFileResource : tsFileResourceList) { + FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFilePath()); + tsFileResource.setStatus(TsFileResourceStatus.DELETED); + tsFileResource.remove(); + LOGGER.info( + "[CrossSpaceCompaction] Delete TsFile :{}.", + tsFileResource.getTsFile().getAbsolutePath()); + } + } + + private void releaseReadAndLockWrite(List tsFileResourceList) { + for (TsFileResource tsFileResource : tsFileResourceList) { + tsFileResource.readUnlock(); + holdReadLockList.remove(tsFileResource); + tsFileResource.writeLock(); + holdWriteLockList.add(tsFileResource); + } + } + + @Override + public boolean checkValidAndSetMerging() { + return addReadLock(selectedSequenceFiles) && addReadLock(selectedUnsequenceFiles); + } + + private boolean addReadLock(List tsFileResourceList) { + for (TsFileResource tsFileResource : tsFileResourceList) { + tsFileResource.readLock(); + holdReadLockList.add(tsFileResource); + if (tsFileResource.isCompacting() + || !tsFileResource.isClosed() + || !tsFileResource.getTsFile().exists() + || tsFileResource.isDeleted()) { + releaseAllLock(); + return false; + } + tsFileResource.setStatus(TsFileResourceStatus.COMPACTING); + } + return true; + } + + public String getStorageGroupName() { + return fullStorageGroupName; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index 8104b1be4d2eb..ca2bcae96cc50 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -19,36 +19,20 @@ package org.apache.iotdb.db.engine.compaction.cross.rewrite.task; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; -import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.exception.metadata.MetadataException; -import org.apache.iotdb.db.query.control.FileReaderManager; -import org.apache.iotdb.tsfile.exception.write.WriteProcessException; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; -import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; - public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { private static final Logger logger = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); @@ -56,13 +40,8 @@ public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactio protected List selectedUnSeqTsFileResourceList; protected TsFileResourceList seqTsFileResourceList; protected TsFileResourceList unseqTsFileResourceList; - private File logFile; private AbstractCompactionPerformer performer; - private List targetTsfileResourceList; - private List holdReadLockList = new ArrayList<>(); - private List holdWriteLockList = new ArrayList<>(); - public RewriteCrossSpaceCompactionTask( String logicalStorageGroupName, String virtualStorageGroupName, @@ -78,171 +57,14 @@ public RewriteCrossSpaceCompactionTask( selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, tsFileManager); - this.selectedSeqTsFileResourceList = selectedSeqTsFileResourceList; - this.selectedUnSeqTsFileResourceList = selectedUnSeqTsFileResourceList; - this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); - this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); - } - - @Override - protected void doCompaction() throws Exception { - try { - executeCompaction(); - } catch (Throwable throwable) { - // catch throwable instead of exception to handle OOM errors - logger.error("Meet errors in cross space compaction, {}", throwable.getMessage()); - CompactionExceptionHandler.handleException( - fullStorageGroupName, - logFile, - targetTsfileResourceList, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList, - tsFileManager, - timePartition, - false, - true); - throw throwable; - } finally { - releaseAllLock(); - } - } - - private void executeCompaction() - throws IOException, StorageEngineException, MetadataException, InterruptedException, - WriteProcessException { - if (!tsFileManager.isAllowCompaction()) { - return; - } - long startTime = System.currentTimeMillis(); - targetTsfileResourceList = - TsFileNameGenerator.getCrossCompactionTargetFileResources(selectedSeqTsFileResourceList); - - if (targetTsfileResourceList.isEmpty() - || selectedSeqTsFileResourceList.isEmpty() - || selectedUnSeqTsFileResourceList.isEmpty()) { - logger.info( - "{} [Compaction] Cross space compaction file list is empty, end it", - fullStorageGroupName); - return; - } - - logger.info( - "{} [Compaction] CrossSpaceCompactionTask start. Sequence files : {}, unsequence files : {}", - fullStorageGroupName, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList); - logFile = - new File( - selectedSeqTsFileResourceList.get(0).getTsFile().getParent() - + File.separator - + targetTsfileResourceList.get(0).getTsFile().getName() - + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); - - try (CompactionLogger compactionLogger = new CompactionLogger(logFile)) { - // print the path of the temporary file first for priority check during recovery - compactionLogger.logFiles(selectedSeqTsFileResourceList, STR_SOURCE_FILES); - compactionLogger.logFiles(selectedUnSeqTsFileResourceList, STR_SOURCE_FILES); - compactionLogger.logFiles(targetTsfileResourceList, STR_TARGET_FILES); - // indicates that the cross compaction is complete and the result can be reused during a - // restart recovery - compactionLogger.close(); - - performer = - new ReadPointCompactionPerformer( - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList, - targetTsfileResourceList); - performer.perform(); - - CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); - CompactionUtils.combineModsInCrossCompaction( - selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList, targetTsfileResourceList); - - // update tsfile resource in memory - tsFileManager.replace( - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList, - targetTsfileResourceList, - timePartition, - true); - - releaseReadAndLockWrite(selectedSeqTsFileResourceList); - releaseReadAndLockWrite(selectedUnSeqTsFileResourceList); - - deleteOldFiles(selectedSeqTsFileResourceList); - deleteOldFiles(selectedUnSeqTsFileResourceList); - CompactionUtils.deleteCompactionModsFile( - selectedSeqTsFileResourceList, selectedUnSeqTsFileResourceList); - - if (logFile.exists()) { - FileUtils.delete(logFile); - } - logger.info( - "{} [Compaction] CrossSpaceCompactionTask Costs {} s", - fullStorageGroupName, - (System.currentTimeMillis() - startTime) / 1000); - } - } - - private boolean addReadLock(List tsFileResourceList) { - for (TsFileResource tsFileResource : tsFileResourceList) { - tsFileResource.readLock(); - holdReadLockList.add(tsFileResource); - if (tsFileResource.isCompacting() - || !tsFileResource.isClosed() - || !tsFileResource.getTsFile().exists() - || tsFileResource.isDeleted()) { - releaseAllLock(); - return false; - } - tsFileResource.setStatus(TsFileResourceStatus.COMPACTING); - } - return true; } @Override - public boolean checkValidAndSetMerging() { - return addReadLock(selectedSeqTsFileResourceList) - && addReadLock(selectedUnSeqTsFileResourceList); - } - - private void releaseReadAndLockWrite(List tsFileResourceList) { - for (TsFileResource tsFileResource : tsFileResourceList) { - tsFileResource.readUnlock(); - holdReadLockList.remove(tsFileResource); - tsFileResource.writeLock(); - holdWriteLockList.add(tsFileResource); - } - } - - private void releaseAllLock() { - selectedSeqTsFileResourceList.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); - selectedUnSeqTsFileResourceList.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); - for (TsFileResource tsFileResource : holdReadLockList) { - tsFileResource.readUnlock(); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); - } - for (TsFileResource tsFileResource : holdWriteLockList) { - tsFileResource.writeUnlock(); - tsFileResource.setStatus(TsFileResourceStatus.CLOSED); - } - holdReadLockList.clear(); - holdWriteLockList.clear(); - } - - private void deleteOldFiles(List tsFileResourceList) throws IOException { - for (TsFileResource tsFileResource : tsFileResourceList) { - FileReaderManager.getInstance().closeFileAndRemoveReader(tsFileResource.getTsFilePath()); - tsFileResource.setStatus(TsFileResourceStatus.DELETED); - tsFileResource.remove(); - logger.info( - "[CrossSpaceCompaction] Delete TsFile :{}.", - tsFileResource.getTsFile().getAbsolutePath()); - } - } - - public String getStorageGroupName() { - return fullStorageGroupName; + protected void performCompaction() throws Exception { + performer = + new ReadPointCompactionPerformer( + selectedSequenceFiles, selectedUnsequenceFiles, targetTsfileResourceList); + performer.perform(); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java index 4f952dc2ca839..7c0560a292486 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java @@ -20,16 +20,24 @@ package org.apache.iotdb.db.engine.compaction.inner; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; +import org.apache.iotdb.db.engine.compaction.CompactionUtils; +import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -38,12 +46,17 @@ public abstract class AbstractInnerSpaceCompactionTask extends AbstractCompactio LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); protected List selectedTsFileResourceList; + protected TsFileResource targetTsFileResource; protected boolean sequence; protected long selectedFileSize; protected int sumOfCompactionCount; protected long maxFileVersion; protected int maxCompactionCount; + protected TsFileResourceList tsFileResourceList; + protected boolean[] isHoldingReadLock; + protected boolean[] isHoldingWriteLock; + public AbstractInnerSpaceCompactionTask( String storageGroupName, long timePartition, @@ -54,9 +67,161 @@ public AbstractInnerSpaceCompactionTask( super(storageGroupName, timePartition, tsFileManager, currentTaskNum); this.selectedTsFileResourceList = selectedTsFileResourceList; this.sequence = sequence; + isHoldingReadLock = new boolean[selectedTsFileResourceList.size()]; + isHoldingWriteLock = new boolean[selectedTsFileResourceList.size()]; + for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { + isHoldingWriteLock[i] = false; + isHoldingReadLock[i] = false; + } + if (sequence) { + tsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); + } else { + tsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); + } collectSelectedFilesInfo(); } + @Override + protected void doCompaction() throws Exception { + if (!tsFileManager.isAllowCompaction()) { + return; + } + long startTime = System.currentTimeMillis(); + // get resource of target file + String dataDirectory = selectedTsFileResourceList.get(0).getTsFile().getParent(); + // Here is tmpTargetFile, which is xxx.target + targetTsFileResource = + TsFileNameGenerator.getInnerCompactionTargetFileResource( + selectedTsFileResourceList, sequence); + LOGGER.info( + "{} [Compaction] starting compaction task with {} files", + fullStorageGroupName, + selectedTsFileResourceList.size()); + File logFile = + new File( + dataDirectory + + File.separator + + targetTsFileResource.getTsFile().getName() + + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); + CompactionLogger compactionLogger = null; + try { + compactionLogger = new CompactionLogger(logFile); + compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES); + compactionLogger.logFiles( + Collections.singletonList(targetTsFileResource), CompactionLogger.STR_TARGET_FILES); + LOGGER.info("{} [SizeTiredCompactionTask] Close the logger", fullStorageGroupName); + compactionLogger.close(); + LOGGER.info( + "{} [Compaction] compaction with {}", fullStorageGroupName, selectedTsFileResourceList); + + // carry out the compaction + performCompaction(); + + CompactionUtils.moveTargetFile( + Collections.singletonList(targetTsFileResource), true, fullStorageGroupName); + + LOGGER.info("{} [SizeTiredCompactionTask] start to rename mods file", fullStorageGroupName); + CompactionUtils.combineModsInInnerCompaction( + selectedTsFileResourceList, targetTsFileResource); + + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException( + String.format("%s [Compaction] abort", fullStorageGroupName)); + } + + // replace the old files with new file, the new is in same position as the old + if (sequence) { + tsFileManager.replace( + selectedTsFileResourceList, + Collections.emptyList(), + Collections.singletonList(targetTsFileResource), + timePartition, + true); + } else { + tsFileManager.replace( + Collections.emptyList(), + selectedTsFileResourceList, + Collections.singletonList(targetTsFileResource), + timePartition, + false); + } + + LOGGER.info( + "{} [Compaction] Compacted target files, try to get the write lock of source files", + fullStorageGroupName); + + // release the read lock of all source files, and get the write lock of them to delete them + for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { + selectedTsFileResourceList.get(i).readUnlock(); + isHoldingReadLock[i] = false; + selectedTsFileResourceList.get(i).writeLock(); + isHoldingWriteLock[i] = true; + } + + if (targetTsFileResource.getTsFile().length() + < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) { + // the file size is smaller than magic string and version number + throw new RuntimeException( + String.format( + "target file %s is smaller than magic string and version number size", + targetTsFileResource)); + } + + LOGGER.info( + "{} [Compaction] compaction finish, start to delete old files", fullStorageGroupName); + // delete the old files + CompactionUtils.deleteTsFilesInDisk(selectedTsFileResourceList, fullStorageGroupName); + CompactionUtils.deleteModificationForSourceFile( + selectedTsFileResourceList, fullStorageGroupName); + + long costTime = System.currentTimeMillis() - startTime; + LOGGER.info( + "{} [SizeTiredCompactionTask] all compaction task finish, target file is {}," + + "time cost is {} s", + fullStorageGroupName, + targetTsFileResource.getTsFile().getName(), + costTime / 1000); + + if (logFile.exists()) { + FileUtils.delete(logFile); + } + } catch (Throwable throwable) { + LOGGER.error( + "{} [Compaction] Throwable is caught during execution of SizeTieredCompaction, {}", + fullStorageGroupName, + throwable); + LOGGER.warn("{} [Compaction] Start to handle exception", fullStorageGroupName); + if (compactionLogger != null) { + compactionLogger.close(); + } + if (isSequence()) { + CompactionExceptionHandler.handleException( + fullStorageGroupName, + logFile, + Collections.singletonList(targetTsFileResource), + selectedTsFileResourceList, + Collections.emptyList(), + tsFileManager, + timePartition, + true, + isSequence()); + } else { + CompactionExceptionHandler.handleException( + fullStorageGroupName, + logFile, + Collections.singletonList(targetTsFileResource), + Collections.emptyList(), + selectedTsFileResourceList, + tsFileManager, + timePartition, + true, + isSequence()); + } + } finally { + releaseFileLocksAndResetMergingStatus(); + } + } + @Override public void setSourceFilesToCompactionCandidate() { this.selectedTsFileResourceList.forEach( @@ -115,20 +280,6 @@ public long getMaxFileVersion() { return maxFileVersion; } - @Override - public boolean checkValidAndSetMerging() { - for (TsFileResource resource : selectedTsFileResourceList) { - if (resource.isCompacting() | !resource.isClosed() || !resource.getTsFile().exists()) { - return false; - } - } - - for (TsFileResource resource : selectedTsFileResourceList) { - resource.setStatus(TsFileResourceStatus.COMPACTING); - } - return true; - } - @Override public String toString() { return new StringBuilder() @@ -148,4 +299,43 @@ public String toString() { public void resetCompactionCandidateStatusForAllSourceFiles() { selectedTsFileResourceList.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); } + + /** + * release the read lock and write lock of files if it is held, and set the merging status of + * selected files to false + */ + protected void releaseFileLocksAndResetMergingStatus() { + for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { + if (isHoldingReadLock[i]) { + selectedTsFileResourceList.get(i).readUnlock(); + } + if (isHoldingWriteLock[i]) { + selectedTsFileResourceList.get(i).writeUnlock(); + } + selectedTsFileResourceList.get(i).setStatus(TsFileResourceStatus.CLOSED); + } + } + + @Override + public boolean checkValidAndSetMerging() { + for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { + TsFileResource resource = selectedTsFileResourceList.get(i); + resource.readLock(); + isHoldingReadLock[i] = true; + if (resource.isCompacting() + || !resource.isClosed() + || !resource.getTsFile().exists() + || resource.isDeleted()) { + // this source file cannot be compacted + // release the lock of locked files, and return + releaseFileLocksAndResetMergingStatus(); + return false; + } + } + + for (TsFileResource resource : selectedTsFileResourceList) { + resource.setStatus(TsFileResourceStatus.COMPACTING); + } + return true; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 78d9e4641e291..98e9a79ae531a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -19,26 +19,17 @@ package org.apache.iotdb.db.engine.compaction.inner.sizetiered; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; -import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus; -import org.apache.iotdb.tsfile.common.conf.TSFileConfig; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -50,9 +41,7 @@ public class SizeTieredCompactionTask extends AbstractInnerSpaceCompactionTask { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - protected TsFileResourceList tsFileResourceList; - protected boolean[] isHoldingReadLock; - protected boolean[] isHoldingWriteLock; + protected AbstractCompactionPerformer performer; public SizeTieredCompactionTask( @@ -70,169 +59,21 @@ public SizeTieredCompactionTask( sequence, selectedTsFileResourceList, tsFileManager); - isHoldingReadLock = new boolean[selectedTsFileResourceList.size()]; - isHoldingWriteLock = new boolean[selectedTsFileResourceList.size()]; - for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { - isHoldingWriteLock[i] = false; - isHoldingReadLock[i] = false; - } - if (sequence) { - tsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); - } else { - tsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); - } } @Override - protected void doCompaction() throws Exception { - if (!tsFileManager.isAllowCompaction()) { - return; - } - long startTime = System.currentTimeMillis(); - // get resource of target file - String dataDirectory = selectedTsFileResourceList.get(0).getTsFile().getParent(); - // Here is tmpTargetFile, which is xxx.target - TsFileResource targetTsFileResource = - TsFileNameGenerator.getInnerCompactionTargetFileResource( - selectedTsFileResourceList, sequence); - LOGGER.info( - "{} [Compaction] starting compaction task with {} files", - fullStorageGroupName, - selectedTsFileResourceList.size()); - File logFile = - new File( - dataDirectory - + File.separator - + targetTsFileResource.getTsFile().getName() - + CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX); - CompactionLogger sizeTieredCompactionLogger = null; - try { - sizeTieredCompactionLogger = new CompactionLogger(logFile); - sizeTieredCompactionLogger.logFiles( - selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES); - sizeTieredCompactionLogger.logFiles( - Collections.singletonList(targetTsFileResource), CompactionLogger.STR_TARGET_FILES); - LOGGER.info("{} [SizeTiredCompactionTask] Close the logger", fullStorageGroupName); - sizeTieredCompactionLogger.close(); - LOGGER.info( - "{} [Compaction] compaction with {}", fullStorageGroupName, selectedTsFileResourceList); - - // carry out the compaction - if (sequence) { - performer = - new ReadChunkCompactionPerformer(selectedTsFileResourceList, targetTsFileResource); - } else { - performer = - new ReadPointCompactionPerformer( - Collections.emptyList(), - selectedTsFileResourceList, - Collections.singletonList(targetTsFileResource)); - } - performer.perform(); - - CompactionUtils.moveTargetFile( - Collections.singletonList(targetTsFileResource), true, fullStorageGroupName); - - LOGGER.info("{} [SizeTiredCompactionTask] start to rename mods file", fullStorageGroupName); - CompactionUtils.combineModsInInnerCompaction( - selectedTsFileResourceList, targetTsFileResource); - - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException( - String.format("%s [Compaction] abort", fullStorageGroupName)); - } - - // replace the old files with new file, the new is in same position as the old - if (sequence) { - tsFileManager.replace( - selectedTsFileResourceList, - Collections.emptyList(), - Collections.singletonList(targetTsFileResource), - timePartition, - true); - } else { - tsFileManager.replace( - Collections.emptyList(), - selectedTsFileResourceList, - Collections.singletonList(targetTsFileResource), - timePartition, - false); - } - - LOGGER.info( - "{} [Compaction] Compacted target files, try to get the write lock of source files", - fullStorageGroupName); - - // release the read lock of all source files, and get the write lock of them to delete them - for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { - selectedTsFileResourceList.get(i).readUnlock(); - isHoldingReadLock[i] = false; - selectedTsFileResourceList.get(i).writeLock(); - isHoldingWriteLock[i] = true; - } - - if (targetTsFileResource.getTsFile().length() - < TSFileConfig.MAGIC_STRING.getBytes().length * 2L + Byte.BYTES) { - // the file size is smaller than magic string and version number - throw new RuntimeException( - String.format( - "target file %s is smaller than magic string and version number size", - targetTsFileResource)); - } - - LOGGER.info( - "{} [Compaction] compaction finish, start to delete old files", fullStorageGroupName); - // delete the old files - CompactionUtils.deleteTsFilesInDisk(selectedTsFileResourceList, fullStorageGroupName); - CompactionUtils.deleteModificationForSourceFile( - selectedTsFileResourceList, fullStorageGroupName); - - long costTime = System.currentTimeMillis() - startTime; - LOGGER.info( - "{} [SizeTiredCompactionTask] all compaction task finish, target file is {}," - + "time cost is {} s", - fullStorageGroupName, - targetTsFileResource.getTsFile().getName(), - costTime / 1000); - - if (logFile.exists()) { - FileUtils.delete(logFile); - } - } catch (Throwable throwable) { - LOGGER.error( - "{} [Compaction] Throwable is caught during execution of SizeTieredCompaction, {}", - fullStorageGroupName, - throwable); - LOGGER.warn("{} [Compaction] Start to handle exception", fullStorageGroupName); - if (sizeTieredCompactionLogger != null) { - sizeTieredCompactionLogger.close(); - } - if (isSequence()) { - CompactionExceptionHandler.handleException( - fullStorageGroupName, - logFile, - Collections.singletonList(targetTsFileResource), - selectedTsFileResourceList, - Collections.emptyList(), - tsFileManager, - timePartition, - true, - isSequence()); - } else { - CompactionExceptionHandler.handleException( - fullStorageGroupName, - logFile, - Collections.singletonList(targetTsFileResource), - Collections.emptyList(), - selectedTsFileResourceList, - tsFileManager, - timePartition, - true, - isSequence()); - } - } finally { - releaseFileLocksAndResetMergingStatus(); + protected void performCompaction() throws Exception { + if (sequence) { + performer = + new ReadChunkCompactionPerformer(selectedTsFileResourceList, targetTsFileResource); + } else { + performer = + new ReadPointCompactionPerformer( + Collections.emptyList(), + selectedTsFileResourceList, + Collections.singletonList(targetTsFileResource)); } + performer.perform(); } @Override @@ -246,43 +87,4 @@ public boolean equalsOtherTask(AbstractCompactionTask other) { } return false; } - - @Override - public boolean checkValidAndSetMerging() { - for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { - TsFileResource resource = selectedTsFileResourceList.get(i); - resource.readLock(); - isHoldingReadLock[i] = true; - if (resource.isCompacting() - || !resource.isClosed() - || !resource.getTsFile().exists() - || resource.isDeleted()) { - // this source file cannot be compacted - // release the lock of locked files, and return - releaseFileLocksAndResetMergingStatus(); - return false; - } - } - - for (TsFileResource resource : selectedTsFileResourceList) { - resource.setStatus(TsFileResourceStatus.COMPACTING); - } - return true; - } - - /** - * release the read lock and write lock of files if it is held, and set the merging status of - * selected files to false - */ - private void releaseFileLocksAndResetMergingStatus() { - for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { - if (isHoldingReadLock[i]) { - selectedTsFileResourceList.get(i).readUnlock(); - } - if (isHoldingWriteLock[i]) { - selectedTsFileResourceList.get(i).writeUnlock(); - } - selectedTsFileResourceList.get(i).setStatus(TsFileResourceStatus.CLOSED); - } - } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java index da46187cc728f..95efdd96ed166 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java @@ -59,6 +59,8 @@ public AbstractCompactionTask( protected abstract void doCompaction() throws Exception; + protected abstract void performCompaction() throws Exception; + @Override public Void call() throws Exception { long startTime = System.currentTimeMillis(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 21852246622b5..345403a43f24e 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -283,6 +283,9 @@ public FakedInnerSpaceCompactionTask( @Override protected void doCompaction() throws Exception {} + @Override + protected void performCompaction() throws Exception {} + @Override public boolean equalsOtherTask(AbstractCompactionTask other) { return false; @@ -315,6 +318,9 @@ public FakeCrossSpaceCompactionTask( @Override protected void doCompaction() throws Exception {} + @Override + protected void performCompaction() throws Exception {} + @Override public boolean equalsOtherTask(AbstractCompactionTask other) { return false; From 69f478930d8ad67a2b5bcec7140bdcb230a7b958 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Wed, 6 Apr 2022 16:58:04 +0800 Subject: [PATCH 08/22] remove useless variable --- .../task/RewriteCrossSpaceCompactionTask.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java index ca2bcae96cc50..093fcf98213e6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java @@ -18,29 +18,17 @@ */ package org.apache.iotdb.db.engine.compaction.cross.rewrite.task; -import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { - private static final Logger logger = - LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - protected List selectedSeqTsFileResourceList; - protected List selectedUnSeqTsFileResourceList; - protected TsFileResourceList seqTsFileResourceList; - protected TsFileResourceList unseqTsFileResourceList; - private AbstractCompactionPerformer performer; public RewriteCrossSpaceCompactionTask( String logicalStorageGroupName, @@ -61,7 +49,7 @@ public RewriteCrossSpaceCompactionTask( @Override protected void performCompaction() throws Exception { - performer = + AbstractCompactionPerformer performer = new ReadPointCompactionPerformer( selectedSequenceFiles, selectedUnsequenceFiles, targetTsfileResourceList); performer.perform(); @@ -71,8 +59,8 @@ protected void performCompaction() throws Exception { public boolean equalsOtherTask(AbstractCompactionTask other) { if (other instanceof RewriteCrossSpaceCompactionTask) { RewriteCrossSpaceCompactionTask otherTask = (RewriteCrossSpaceCompactionTask) other; - return otherTask.selectedSeqTsFileResourceList.equals(selectedSeqTsFileResourceList) - && otherTask.selectedUnSeqTsFileResourceList.equals(selectedUnSeqTsFileResourceList); + return otherTask.selectedSequenceFiles.equals(selectedSequenceFiles) + && otherTask.selectedUnsequenceFiles.equals(selectedUnsequenceFiles); } return false; } From 66bbe6cb3fece0dc65defab19657d87d801f96ed Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Wed, 6 Apr 2022 17:01:37 +0800 Subject: [PATCH 09/22] refactor package structure --- .../db/engine/compaction/CompactionUtils.java | 4 +- .../constant/CrossCompactionStrategy.java | 2 +- ... RewriteCrossSpaceCompactionResource.java} | 8 +-- .../RewriteCrossSpaceCompactionSelector.java | 5 +- .../RewriteCrossSpaceCompactionTask.java | 2 +- .../rewrite/selector/IMergePathSelector.java | 31 ---------- .../rewrite/selector/NaivePathSelector.java | 55 ------------------ .../RewriteCompactionFileSelector.java | 7 ++- .../compaction/CompactionTaskManagerTest.java | 2 +- .../cross/CrossSpaceCompactionTest.java | 14 ++--- .../compaction/cross/MergeUpgradeTest.java | 6 +- .../RewriteCompactionFileSelectorTest.java | 58 ++++++++++--------- .../RewriteCrossSpaceCompactionTest.java | 2 +- .../task/FakedCrossSpaceCompactionTask.java | 2 +- 14 files changed, 59 insertions(+), 139 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/{manage/CrossSpaceCompactionResource.java => RewriteCrossSpaceCompactionResource.java} (97%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/{task => }/RewriteCrossSpaceCompactionTask.java (97%) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/IMergePathSelector.java delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/NaivePathSelector.java diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java index b6618eb3f8b00..92bc4ebed2a26 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.modification.Modification; @@ -228,7 +228,7 @@ public static void deleteModificationForSourceFile( } public static ICrossSpaceMergeFileSelector getCrossSpaceFileSelector( - long budget, CrossSpaceCompactionResource resource) { + long budget, RewriteCrossSpaceCompactionResource resource) { CrossCompactionStrategy strategy = IoTDBDescriptor.getInstance().getConfig().getCrossCompactionStrategy(); switch (strategy) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java index c348ea6562c5a..d554d02be74cf 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionResource.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionResource.java index 7ced2fbcfc940..88949298dd4cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionResource.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.cross.rewrite.manage; +package org.apache.iotdb.db.engine.compaction.cross.rewrite; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -45,7 +45,7 @@ * CrossSpaceMergeResource manages files and caches of readers, writers, MeasurementSchemas and * modifications to avoid unnecessary object creations and file openings. */ -public class CrossSpaceCompactionResource { +public class RewriteCrossSpaceCompactionResource { private List seqFiles; private List unseqFiles = new ArrayList<>(); @@ -61,7 +61,7 @@ public class CrossSpaceCompactionResource { private boolean cacheDeviceMeta = false; - public CrossSpaceCompactionResource( + public RewriteCrossSpaceCompactionResource( List seqFiles, List unseqFiles) { this.seqFiles = seqFiles.stream().filter(this::filterSeqResource).collect(Collectors.toList()); filterUnseqResource(unseqFiles); @@ -88,7 +88,7 @@ private void filterUnseqResource(List unseqResources) { } } - public CrossSpaceCompactionResource( + public RewriteCrossSpaceCompactionResource( Collection seqFiles, List unseqFiles, long ttlLowerBound) { this.ttlLowerBound = ttlLowerBound; this.seqFiles = seqFiles.stream().filter(this::filterSeqResource).collect(Collectors.toList()); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 7150b758e8b1f..ac83392cbca93 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -24,7 +24,6 @@ import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -87,8 +86,8 @@ public void selectAndSubmit() { } long budget = config.getCrossCompactionMemoryBudget(); long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; - CrossSpaceCompactionResource mergeResource = - new CrossSpaceCompactionResource(seqFileList, unSeqFileList, timeLowerBound); + RewriteCrossSpaceCompactionResource mergeResource = + new RewriteCrossSpaceCompactionResource(seqFileList, unSeqFileList, timeLowerBound); ICrossSpaceMergeFileSelector fileSelector = CompactionUtils.getCrossSpaceFileSelector(budget, mergeResource); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java similarity index 97% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java index 093fcf98213e6..daa7183cc2ec7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/task/RewriteCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.cross.rewrite.task; +package org.apache.iotdb.db.engine.compaction.cross.rewrite; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/IMergePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/IMergePathSelector.java deleted file mode 100644 index bf60b193e27ca..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/IMergePathSelector.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.cross.rewrite.selector; - -import org.apache.iotdb.db.metadata.path.PartialPath; - -import java.util.Iterator; -import java.util.List; - -/** - * IMergePathSelector select paths to be merged at a time if all paths of a device cannot be merged - * at the same time. - */ -public interface IMergePathSelector extends Iterator> {} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/NaivePathSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/NaivePathSelector.java deleted file mode 100644 index d3e73c3b72065..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/NaivePathSelector.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.cross.rewrite.selector; - -import org.apache.iotdb.db.metadata.path.PartialPath; - -import java.util.List; -import java.util.NoSuchElementException; - -public class NaivePathSelector implements IMergePathSelector { - - private List paths; - private int idx; - private int maxSeriesNum; - - public NaivePathSelector(List paths, int maxSeriesNum) { - this.paths = paths; - this.maxSeriesNum = maxSeriesNum; - } - - @Override - public boolean hasNext() { - return idx < paths.size(); - } - - @Override - public List next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - List ret = - idx + maxSeriesNum <= paths.size() - ? paths.subList(idx, idx + maxSeriesNum) - : paths.subList(idx, paths.size()); - idx += maxSeriesNum; - return ret; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java index bdf8a683f3025..5859a5ef9a33c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.engine.compaction.cross.rewrite.selector; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.MergeException; import org.apache.iotdb.db.utils.MergeUtils; @@ -46,7 +46,7 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect private static final Logger logger = LoggerFactory.getLogger(RewriteCompactionFileSelector.class); private static final String LOG_FILE_COST = "Memory cost of file {} is {}"; - CrossSpaceCompactionResource resource; + RewriteCrossSpaceCompactionResource resource; long totalCost; private long memoryBudget; @@ -70,7 +70,8 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect private boolean[] seqSelected; private int seqSelectedNum; - public RewriteCompactionFileSelector(CrossSpaceCompactionResource resource, long memoryBudget) { + public RewriteCompactionFileSelector( + RewriteCrossSpaceCompactionResource resource, long memoryBudget) { this.resource = resource; this.memoryBudget = memoryBudget; this.maxCrossCompactionFileNum = diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java index 511cdb144fc60..6042730f3a034 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java index 7a052fd50be6f..c2129c7755f74 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; @@ -416,8 +416,8 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; - CrossSpaceCompactionResource mergeResource = - new CrossSpaceCompactionResource( + RewriteCrossSpaceCompactionResource mergeResource = + new RewriteCrossSpaceCompactionResource( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); ICrossSpaceMergeFileSelector fileSelector = new RewriteCompactionFileSelector(mergeResource, Long.MAX_VALUE); @@ -722,8 +722,8 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; - CrossSpaceCompactionResource mergeResource = - new CrossSpaceCompactionResource( + RewriteCrossSpaceCompactionResource mergeResource = + new RewriteCrossSpaceCompactionResource( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); ICrossSpaceMergeFileSelector fileSelector = new RewriteCompactionFileSelector(mergeResource, Long.MAX_VALUE); @@ -1026,8 +1026,8 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { TsFileResourceList unseqTsFileResourceList = new TsFileResourceList(); unseqTsFileResourceList.addAll(unseqResources); long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; - CrossSpaceCompactionResource mergeResource = - new CrossSpaceCompactionResource( + RewriteCrossSpaceCompactionResource mergeResource = + new RewriteCrossSpaceCompactionResource( seqTsFileResourceList, unseqTsFileResourceList, timeLowerBound); ICrossSpaceMergeFileSelector fileSelector = new RewriteCompactionFileSelector(mergeResource, Long.MAX_VALUE); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java index 2f3bcffc4c485..410822ebc8eb6 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -79,8 +79,8 @@ public void tearDown() { @Test public void testMergeUpgradeSelect() throws MergeException { - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); RewriteCompactionFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); List[] result = mergeFileSelector.select(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java index 5345c6449b1d1..0b5ca8d891dbe 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -52,8 +52,8 @@ public class RewriteCompactionFileSelectorTest extends MergeTest { @Test public void testFullSelection() throws MergeException, IOException { - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); List[] result = mergeFileSelector.select(); @@ -63,7 +63,7 @@ public void testFullSelection() throws MergeException, IOException { assertEquals(unseqResources, unseqSelected); resource.clear(); - resource = new CrossSpaceCompactionResource(seqResources.subList(0, 1), unseqResources); + resource = new RewriteCrossSpaceCompactionResource(seqResources.subList(0, 1), unseqResources); mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); result = mergeFileSelector.select(); seqSelected = result[0]; @@ -72,7 +72,7 @@ public void testFullSelection() throws MergeException, IOException { assertEquals(unseqResources, unseqSelected); resource.clear(); - resource = new CrossSpaceCompactionResource(seqResources, unseqResources.subList(0, 1)); + resource = new RewriteCrossSpaceCompactionResource(seqResources, unseqResources.subList(0, 1)); mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); result = mergeFileSelector.select(); seqSelected = result[0]; @@ -84,8 +84,8 @@ public void testFullSelection() throws MergeException, IOException { @Test public void testNonSelection() throws MergeException, IOException { - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, 1); List[] result = mergeFileSelector.select(); assertEquals(0, result.length); @@ -94,8 +94,8 @@ public void testNonSelection() throws MergeException, IOException { @Test public void testRestrictedSelection() throws MergeException, IOException { - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, 400000); List[] result = mergeFileSelector.select(); @@ -152,8 +152,8 @@ public void testFileOpenSelection() List newUnseqResources = new ArrayList<>(); newUnseqResources.add(largeUnseqTsFileResource); - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, newUnseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, newUnseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); List[] result = mergeFileSelector.select(); @@ -207,8 +207,8 @@ public void testFileOpenSelectionFromCompaction() newUnseqResources.add(largeUnseqTsFileResource); long ttlLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; - CrossSpaceCompactionResource mergeResource = - new CrossSpaceCompactionResource(seqResources, newUnseqResources, ttlLowerBound); + RewriteCrossSpaceCompactionResource mergeResource = + new RewriteCrossSpaceCompactionResource(seqResources, newUnseqResources, ttlLowerBound); assertEquals(5, mergeResource.getSeqFiles().size()); assertEquals(1, mergeResource.getUnseqFiles().size()); mergeResource.clear(); @@ -244,8 +244,8 @@ public void testFileSelectionAboutLastSeqFile() unseqResources.clear(); unseqResources.add(largeUnseqTsFileResource); - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); List[] result = mergeFileSelector.select(); @@ -302,7 +302,8 @@ public void testSelectContinuousUnseqFile() prepareFile(unseqList.get(1), 0, 100, 20); prepareFile(unseqList.get(2), 99, 1, 30); - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); // the budget is enough to select unseq0 and unseq2, but not unseq1 // the first selection should only contain seq0 and unseq0 ICrossSpaceMergeFileSelector mergeFileSelector = @@ -315,7 +316,7 @@ public void testSelectContinuousUnseqFile() resource.clear(); resource = - new CrossSpaceCompactionResource( + new RewriteCrossSpaceCompactionResource( seqList.subList(1, seqList.size()), unseqList.subList(1, unseqList.size())); // the second selection should be empty mergeFileSelector = new RewriteCompactionFileSelector(resource, 29000); @@ -379,7 +380,8 @@ public void testUnseqFilesOverlappedWithOneSeqFile() unseqList.add(fileResource); } - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); Assert.assertEquals(5, resource.getSeqFiles().size()); Assert.assertEquals(10, resource.getUnseqFiles().size()); ICrossSpaceMergeFileSelector mergeFileSelector = @@ -442,7 +444,8 @@ public void testOneUnseqFileOverlappedWithOneSeqFile() unseqList.add(fileResource); } - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); Assert.assertEquals(5, resource.getSeqFiles().size()); Assert.assertEquals(1, resource.getUnseqFiles().size()); ICrossSpaceMergeFileSelector mergeFileSelector = @@ -505,7 +508,8 @@ public void testUnseqFilesOverlapped() throws IOException, WriteProcessException prepareFile(unseqList.get(0), 7, 3, 7); prepareFile(unseqList.get(1), 10, 4, 10); - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); Assert.assertEquals(5, resource.getSeqFiles().size()); Assert.assertEquals(2, resource.getUnseqFiles().size()); ICrossSpaceMergeFileSelector mergeFileSelector = @@ -571,7 +575,8 @@ public void testAllUnseqFilesOverlapped() prepareFile(unseqList.get(2), 14, 3, 14); prepareFile(unseqList.get(3), 17, 2, 17); - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); Assert.assertEquals(5, resource.getSeqFiles().size()); Assert.assertEquals(4, resource.getUnseqFiles().size()); ICrossSpaceMergeFileSelector mergeFileSelector = @@ -640,7 +645,8 @@ public void testAllUnseqFilesOverlappedWithSeqFileOpen() prepareFile(unseqList.get(2), 14, 3, 14); prepareFile(unseqList.get(3), 17, 2, 17); - CrossSpaceCompactionResource resource = new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); Assert.assertEquals(5, resource.getSeqFiles().size()); Assert.assertEquals(4, resource.getUnseqFiles().size()); ICrossSpaceMergeFileSelector mergeFileSelector = @@ -863,8 +869,8 @@ public void testMultiFileOverlapWithOneFile() fileWriter.flushAllChunkGroups(); fileWriter.close(); - CrossSpaceCompactionResource compactionResource = - new CrossSpaceCompactionResource(seqList, unseqList); + RewriteCrossSpaceCompactionResource compactionResource = + new RewriteCrossSpaceCompactionResource(seqList, unseqList); RewriteCompactionFileSelector selector = new RewriteCompactionFileSelector(compactionResource, 500 * 1024 * 1024); List[] result = selector.select(); @@ -877,8 +883,8 @@ public void testMaxFileSelection() throws MergeException, IOException { int oldMaxCrossCompactionCandidateFileNum = IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum(); IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(5); - CrossSpaceCompactionResource resource = - new CrossSpaceCompactionResource(seqResources, unseqResources); + RewriteCrossSpaceCompactionResource resource = + new RewriteCrossSpaceCompactionResource(seqResources, unseqResources); ICrossSpaceMergeFileSelector mergeFileSelector = new RewriteCompactionFileSelector(resource, Long.MAX_VALUE); List[] result = mergeFileSelector.select(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java index 3e8dfa8bb36ae..eed4722bc2176 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java index 315ffc51e3d77..4b588c0cfffe4 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.engine.compaction.task; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.storagegroup.FakedTsFileResource; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; From 9b3388c291bb03afe076298192c6b8c9b4988156 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 7 Apr 2022 09:52:43 +0800 Subject: [PATCH 10/22] fix compilation problem --- .../compaction/task/FakedCrossSpaceCompactionTask.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java index 4b588c0cfffe4..777e16aadb783 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java @@ -48,15 +48,15 @@ public FakedCrossSpaceCompactionTask( @Override protected void doCompaction() { long totalUnseqFileSize = 0; - for (TsFileResource resource : selectedUnSeqTsFileResourceList) { + for (TsFileResource resource : selectedUnsequenceFiles) { totalUnseqFileSize += resource.getTsFileSize(); } - long avgSizeAddToSeqFile = totalUnseqFileSize / selectedSeqTsFileResourceList.size(); - for (TsFileResource resource : selectedSeqTsFileResourceList) { + long avgSizeAddToSeqFile = totalUnseqFileSize / selectedSequenceFiles.size(); + for (TsFileResource resource : selectedSequenceFiles) { ((FakedTsFileResource) resource) .setTsFileSize(resource.getTsFileSize() + avgSizeAddToSeqFile); } - selectedSeqTsFileResourceList.clear(); - selectedUnSeqTsFileResourceList.clear(); + selectedSequenceFiles.clear(); + selectedUnsequenceFiles.clear(); } } From 7cbbc78998aa7c8682d3b95370a67325d2f6cb05 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 7 Apr 2022 10:09:11 +0800 Subject: [PATCH 11/22] refactor CompactionTaskComparator --- .../compaction/CompactionTaskManager.java | 4 +-- .../DefaultCompactionTaskComparatorImpl.java} | 10 +++--- .../comparator/ICompactionTaskComparator.java | 35 +++++++++++++++++++ .../ICrossSpaceMergeFileSelector.java | 2 -- .../RewriteCompactionFileSelector.java | 5 --- .../SizeTieredCompactionSelector.java | 10 +++--- .../CompactionTaskComparatorTest.java | 8 +++-- 7 files changed, 52 insertions(+), 22 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{task/CompactionTaskComparator.java => comparator/DefaultCompactionTaskComparatorImpl.java} (93%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java index 591948175dd17..415a8eba38236 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java @@ -27,9 +27,9 @@ import org.apache.iotdb.commons.service.ServiceType; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; import org.apache.iotdb.db.engine.compaction.constant.CompactionTaskStatus; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.utils.datastructure.FixedPriorityBlockingQueue; import com.google.common.util.concurrent.RateLimiter; @@ -62,7 +62,7 @@ public class CompactionTaskManager implements IService { private WrappedScheduledExecutorService taskExecutionPool; public static volatile AtomicInteger currentTaskNum = new AtomicInteger(0); private FixedPriorityBlockingQueue candidateCompactionTaskQueue = - new FixedPriorityBlockingQueue<>(1024, new CompactionTaskComparator()); + new FixedPriorityBlockingQueue<>(1024, new DefaultCompactionTaskComparatorImpl()); // , it is used to terminate all compaction tasks under the // logicalStorageGroup private Map>> storageGroupTasks = new ConcurrentHashMap<>(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java similarity index 93% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java index 2ca4b016afed3..2c45c7e3273cd 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionTaskComparator.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java @@ -17,19 +17,19 @@ * under the License. */ -package org.apache.iotdb.db.engine.compaction.task; +package org.apache.iotdb.db.engine.compaction.comparator; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import java.util.Comparator; import java.util.List; -public class CompactionTaskComparator implements Comparator { +public class DefaultCompactionTaskComparatorImpl implements ICompactionTaskComparator { private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @Override @@ -55,7 +55,7 @@ public int compare(AbstractCompactionTask o1, AbstractCompactionTask o2) { } } - private int compareInnerSpaceCompactionTask( + public int compareInnerSpaceCompactionTask( AbstractInnerSpaceCompactionTask o1, AbstractInnerSpaceCompactionTask o2) { if (o1.isSequence() ^ o2.isSequence()) { // prioritize sequence file compaction @@ -97,7 +97,7 @@ private int compareInnerSpaceCompactionTask( return 0; } - private int compareCrossSpaceCompactionTask( + public int compareCrossSpaceCompactionTask( AbstractCrossSpaceCompactionTask o1, AbstractCrossSpaceCompactionTask o2) { if (o1.getSelectedSequenceFiles().size() != o2.getSelectedSequenceFiles().size()) { // we prefer the task with fewer sequence files diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java new file mode 100644 index 0000000000000..dcea8908eb33a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.comparator; + +import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; + +import java.util.Comparator; + +public interface ICompactionTaskComparator extends Comparator { + int compare(AbstractCompactionTask task1, AbstractCompactionTask task2); + + int compareInnerSpaceCompactionTask( + AbstractInnerSpaceCompactionTask task1, AbstractInnerSpaceCompactionTask task2); + + int compareCrossSpaceCompactionTask( + AbstractCrossSpaceCompactionTask task1, AbstractCrossSpaceCompactionTask task2); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/ICrossSpaceMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/ICrossSpaceMergeFileSelector.java index b0d36564adc8d..70243dc5c868b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/ICrossSpaceMergeFileSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/ICrossSpaceMergeFileSelector.java @@ -30,6 +30,4 @@ public interface ICrossSpaceMergeFileSelector { List[] select() throws MergeException; - - int getConcurrentMergeNum(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java index 5859a5ef9a33c..f7d1ffbf1d724 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java @@ -392,9 +392,4 @@ private long calculateTightUnseqMemoryCost(TsFileResource unseqFile) throws IOEx long maxCost = unseqFile.getTsFileSize(); return Math.min(multiSeriesCost, maxCost); } - - @Override - public int getConcurrentMergeNum() { - return concurrentMergeNum; - } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 9f3d914f0ce1d..ce9861b5bd7e6 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; +import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -44,10 +44,10 @@ * selector traverses the file list from old to new. If the size of selected files or the number of * select files exceed given threshold, a compaction task will be submitted to task queue in * CompactionTaskManager. In CompactionTaskManager, tasks are ordered by {@link - * CompactionTaskComparator}. To maximize compaction efficiency, selector searches compaction task - * from 0 compaction files(that is, file that never been compacted, named level 0 file) to higher - * level files. If a compaction task is found in some level, selector will not search higher level - * anymore. + * DefaultCompactionTaskComparatorImpl}. To maximize compaction efficiency, selector searches + * compaction task from 0 compaction files(that is, file that never been compacted, named level 0 + * file) to higher level files. If a compaction task is found in some level, selector will not + * search higher level anymore. */ public class SizeTieredCompactionSelector extends AbstractInnerSpaceCompactionSelector { private static final Logger LOGGER = diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 345403a43f24e..308ea31c930b8 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -19,11 +19,11 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.CompactionTaskComparator; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -47,7 +47,7 @@ public class CompactionTaskComparatorTest { private final Logger LOGGER = LoggerFactory.getLogger(CompactionTaskComparatorTest.class); private final AtomicInteger taskNum = new AtomicInteger(0); private FixedPriorityBlockingQueue compactionTaskQueue = - new FixedPriorityBlockingQueue<>(1024, new CompactionTaskComparator()); + new FixedPriorityBlockingQueue<>(1024, new DefaultCompactionTaskComparatorImpl()); private TsFileManager tsFileManager = new TsFileManager("fakeSg", "0", "/"); @Before @@ -128,7 +128,9 @@ public void testFileCompactCountCompare() throws InterruptedException { @Test public void testPriorityQueueSizeLimit() { MinMaxPriorityQueue limitQueue = - MinMaxPriorityQueue.orderedBy(new CompactionTaskComparator()).maximumSize(50).create(); + MinMaxPriorityQueue.orderedBy(new DefaultCompactionTaskComparatorImpl()) + .maximumSize(50) + .create(); AbstractCompactionTask[] compactionTasks = new AbstractCompactionTask[100]; for (int i = 0; i < 100; ++i) { List resources = new ArrayList<>(); From 91fdbb9be5467ec1dd95726e0190c8e413f1e7c8 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 7 Apr 2022 11:04:54 +0800 Subject: [PATCH 12/22] format the code --- .../inner/sizetiered/SizeTieredCompactionTask.java | 6 ------ .../compaction/inner/InnerCompactionSchedulerTest.java | 9 +++------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java index 98e9a79ae531a..e5c156a1ac3c1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.engine.compaction.inner.sizetiered; -import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; @@ -27,9 +26,6 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -39,8 +35,6 @@ * SizeTieredCompactionSelector} into one file. */ public class SizeTieredCompactionTask extends AbstractInnerSpaceCompactionTask { - private static final Logger LOGGER = - LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); protected AbstractCompactionPerformer performer; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java index c549d5a8879fc..d2f38befdc6bc 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java @@ -92,8 +92,7 @@ public void testFileSelector1() throws IOException, MetadataException, WriteProc TsFileManager tsFileManager = new TsFileManager("testSG", "0", "tmp"); tsFileManager.addAll(seqResources, true); - CompactionScheduler.tryToSubmitInnerSpaceCompactionTask( - "testSG", "0", 0L, tsFileManager, true); + CompactionScheduler.tryToSubmitInnerSpaceCompactionTask("testSG", "0", 0L, tsFileManager, true); CompactionTaskManager.getInstance().submitTaskFromTaskQueue(); try { Thread.sleep(5000); @@ -114,8 +113,7 @@ public void testFileSelector2() throws IOException, MetadataException, WriteProc seqResources.get(0).setStatus(TsFileResourceStatus.COMPACTING); TsFileManager tsFileManager = new TsFileManager("testSG", "0", "tmp"); tsFileManager.addAll(seqResources, true); - CompactionScheduler.tryToSubmitInnerSpaceCompactionTask( - "testSG", "0", 0L, tsFileManager, true); + CompactionScheduler.tryToSubmitInnerSpaceCompactionTask("testSG", "0", 0L, tsFileManager, true); CompactionTaskManager.getInstance().submitTaskFromTaskQueue(); long waitingTime = 0; @@ -145,8 +143,7 @@ public void testFileSelectorWithUnclosedFile() seqResources.get(3).setStatus(TsFileResourceStatus.UNCLOSED); TsFileManager tsFileManager = new TsFileManager("testSG", "0", "tmp"); tsFileManager.addAll(seqResources, true); - CompactionScheduler.tryToSubmitInnerSpaceCompactionTask( - "testSG", "0", 0L, tsFileManager, true); + CompactionScheduler.tryToSubmitInnerSpaceCompactionTask("testSG", "0", 0L, tsFileManager, true); CompactionTaskManager.getInstance().submitTaskFromTaskQueue(); long waitingTime = 0; From 871f3d889a2326f3ccc991d82d437a94020c5605 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 7 Apr 2022 11:46:31 +0800 Subject: [PATCH 13/22] temp submit --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++-- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +- .../compaction/CompactionScheduler.java | 60 ++++++++++++------- ...a => InnerSequenceCompactionStrategy.java} | 17 ++---- .../InnerUnsequenceCompactionStrategy.java | 48 +++++++++++++++ .../AbstractCrossSpaceCompactionSelector.java | 7 ++- .../RewriteCrossSpaceCompactionSelector.java | 33 +++++----- ...InnerSequenceSpaceCompactionSelector.java} | 17 ++---- ...nnerUnsequenceSpaceCompactionSelector.java | 29 +++++++++ .../SizeTieredCompactionSelector.java | 21 +++++-- ...Selector.java => ICompactionSelector.java} | 6 +- .../utils/CompactionConfigRestorer.java | 4 +- 12 files changed, 179 insertions(+), 77 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/{InnerCompactionStrategy.java => InnerSequenceCompactionStrategy.java} (85%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/{AbstractInnerSpaceCompactionSelector.java => AbstractInnerSequenceSpaceCompactionSelector.java} (79%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/task/{AbstractCompactionSelector.java => ICompactionSelector.java} (92%) diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 5ff59c8ed939a..0b278c8c0a826 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; @@ -379,8 +379,8 @@ public class IoTDBConfig { * The strategy of inner space compaction task. There are just one inner space compaction strategy * SIZE_TIRED_COMPACTION: */ - private InnerCompactionStrategy innerCompactionStrategy = - InnerCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerSequenceCompactionStrategy innerCompactionStrategy = + InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; /** * The strategy of cross space compaction task. There are just one cross space compaction strategy @@ -2411,11 +2411,11 @@ public void setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompaction) { this.enableCrossSpaceCompaction = enableCrossSpaceCompaction; } - public InnerCompactionStrategy getInnerCompactionStrategy() { + public InnerSequenceCompactionStrategy getInnerCompactionStrategy() { return innerCompactionStrategy; } - public void setInnerCompactionStrategy(InnerCompactionStrategy innerCompactionStrategy) { + public void setInnerCompactionStrategy(InnerSequenceCompactionStrategy innerCompactionStrategy) { this.innerCompactionStrategy = innerCompactionStrategy; } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index d071e953e9ff1..92544fe98b4a8 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; import org.apache.iotdb.db.exception.BadNodeUrlFormatException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.utils.DatetimeUtils; @@ -365,7 +365,7 @@ private void loadProps() { "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString()))); conf.setInnerCompactionStrategy( - InnerCompactionStrategy.getInnerCompactionStrategy( + InnerSequenceCompactionStrategy.getInnerSequenceCompactionStrategy( properties.getProperty( "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString()))); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index 59a69207f945f..aea67027b11f5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -19,31 +19,39 @@ package org.apache.iotdb.db.engine.compaction; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + /** * CompactionScheduler schedules and submits the compaction task periodically, and it counts the * total number of running compaction task. There are three compaction strategy: BALANCE, * INNER_CROSS, CROSS_INNER. Difference strategies will lead to different compaction preferences. * For different types of compaction task(e.g. InnerSpaceCompaction), CompactionScheduler will call - * the corresponding {@link org.apache.iotdb.db.engine.compaction.task.AbstractCompactionSelector - * selector} according to the compaction machanism of the task(e.g. LevelCompaction, - * SizeTiredCompaction), and the selection and submission process is carried out in the {@link - * AbstractCompactionSelector#selectAndSubmit() selectAndSubmit()} in selector. + * the corresponding {@link ICompactionSelector selector} according to the compaction machanism of + * the task(e.g. LevelCompaction, SizeTiredCompaction), and the selection and submission process is + * carried out in the {@link ICompactionSelector#select() selectAndSubmit()} in selector. */ public class CompactionScheduler { - + private static final Logger LOGGER = + LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); public static void scheduleCompaction(TsFileManager tsFileManager, long timePartition) { if (!tsFileManager.isAllowCompaction()) { return; } + try { tryToSubmitCrossSpaceCompactionTask( tsFileManager.getStorageGroupName(), tsFileManager.getVirtualStorageGroup(), @@ -62,6 +70,9 @@ public static void scheduleCompaction(TsFileManager tsFileManager, long timePart timePartition, tsFileManager, false); + } catch (InterruptedException e) { + LOGGER.error("Exception occurs when selecting compaction tasks", e); + } } public static void tryToSubmitInnerSpaceCompactionTask( @@ -69,22 +80,27 @@ public static void tryToSubmitInnerSpaceCompactionTask( String virtualStorageGroupName, long timePartition, TsFileManager tsFileManager, - boolean sequence) { + boolean sequence) + throws InterruptedException { if ((!config.isEnableSeqSpaceCompaction() && sequence) || (!config.isEnableUnseqSpaceCompaction() && !sequence)) { return; } - AbstractInnerSpaceCompactionSelector innerSpaceCompactionSelector = - config - .getInnerCompactionStrategy() - .getCompactionSelector( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - sequence); - innerSpaceCompactionSelector.selectAndSubmit(); + List taskList = null; + if (sequence) { + AbstractInnerSequenceSpaceCompactionSelector innerSpaceCompactionSelector = + config + .getInnerCompactionStrategy() + .getCompactionSelector( + logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager); + taskList = innerSpaceCompactionSelector.select(); + } else { + + } + for (AbstractCompactionTask task : taskList) { + CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); + } } private static void tryToSubmitCrossSpaceCompactionTask( @@ -92,7 +108,8 @@ private static void tryToSubmitCrossSpaceCompactionTask( String virtualStorageGroupName, String storageGroupDir, long timePartition, - TsFileManager tsFileManager) { + TsFileManager tsFileManager) + throws InterruptedException { if (!config.isEnableCrossSpaceCompaction()) { return; } @@ -105,6 +122,9 @@ private static void tryToSubmitCrossSpaceCompactionTask( storageGroupDir, timePartition, tsFileManager); - crossSpaceCompactionSelector.selectAndSubmit(); + List taskList = crossSpaceCompactionSelector.select(); + for (AbstractCompactionTask task : taskList) { + CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java similarity index 85% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java index 5e766e0c041f7..f90b96f49305a 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.engine.compaction.constant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; @@ -29,10 +29,10 @@ import java.util.List; -public enum InnerCompactionStrategy { +public enum InnerSequenceCompactionStrategy { SIZE_TIERED_COMPACTION; - public static InnerCompactionStrategy getInnerCompactionStrategy(String name) { + public static InnerSequenceCompactionStrategy getInnerSequenceCompactionStrategy(String name) { if ("SIZE_TIERED_COMPACTION".equalsIgnoreCase(name)) { return SIZE_TIERED_COMPACTION; } @@ -60,21 +60,16 @@ public AbstractInnerSpaceCompactionTask getCompactionTask( } } - public AbstractInnerSpaceCompactionSelector getCompactionSelector( + public AbstractInnerSequenceSpaceCompactionSelector getCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, - TsFileManager tsFileManager, - boolean sequence) { + TsFileManager tsFileManager) { switch (this) { case SIZE_TIERED_COMPACTION: default: return new SizeTieredCompactionSelector( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - sequence); + logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, true); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java new file mode 100644 index 0000000000000..d77f9b3176666 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.constant; + +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileManager; + +public enum InnerUnsequenceCompactionStrategy { + SIZE_TIERED_COMPACTION; + + public static InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStrategy( + String name) { + if (SIZE_TIERED_COMPACTION.toString().equalsIgnoreCase(name)) { + return SIZE_TIERED_COMPACTION; + } + throw new RuntimeException("Illegal Compaction Strategy " + name); + } + + public AbstractInnerSequenceSpaceCompactionSelector getCompactionSelector( + String logicalStorageGroupName, + String virtualStorageGroupName, + long timePartition, + TsFileManager tsFileManager) { + switch (this) { + case SIZE_TIERED_COMPACTION: + default: + return new SizeTieredCompactionSelector( + logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, false); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java index 743001161db80..d53da2b99be5c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java @@ -18,13 +18,14 @@ */ package org.apache.iotdb.db.engine.compaction.cross; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionSelector; +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; -public abstract class AbstractCrossSpaceCompactionSelector extends AbstractCompactionSelector { +public abstract class AbstractCrossSpaceCompactionSelector implements ICompactionSelector { protected String logicalStorageGroupName; protected String virtualGroupId; protected String storageGroupDir; @@ -49,5 +50,5 @@ public AbstractCrossSpaceCompactionSelector( } @Override - public abstract void selectAndSubmit(); + public abstract List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index ac83392cbca93..56cc0141a2032 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -66,10 +67,10 @@ public RewriteCrossSpaceCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public void selectAndSubmit() { + public List select() { if ((CompactionTaskManager.currentTaskNum.get() >= config.getConcurrentCompactionThread()) || (!config.isEnableCrossSpaceCompaction())) { - return; + return Collections.emptyList(); } Iterator seqIterator = sequenceFileList.iterator(); Iterator unSeqIterator = unsequenceFileList.iterator(); @@ -82,7 +83,7 @@ public void selectAndSubmit() { unSeqFileList.add(unSeqIterator.next()); } if (seqFileList.isEmpty() || unSeqFileList.isEmpty()) { - return; + return Collections.emptyList(); } long budget = config.getCrossCompactionMemoryBudget(); long timeLowerBound = System.currentTimeMillis() - Long.MAX_VALUE; @@ -103,7 +104,7 @@ public void selectAndSubmit() { logicalStorageGroupName, budget); } - return; + return Collections.emptyList(); } LOGGER.info( "select files for cross compaction, sequence files: {}, unsequence files {}", @@ -111,27 +112,25 @@ public void selectAndSubmit() { mergeFiles[1]); if (mergeFiles[0].size() > 0 && mergeFiles[1].size() > 0) { - AbstractCompactionTask compactionTask = - IoTDBDescriptor.getInstance() - .getConfig() - .getCrossCompactionStrategy() - .getCompactionTask( - logicalStorageGroupName, - virtualGroupId, - timePartition, - tsFileManager, - mergeFiles[0], - mergeFiles[1]); - CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); LOGGER.info( "{} [Compaction] submit a task with {} sequence file and {} unseq files", logicalStorageGroupName + "-" + virtualGroupId, mergeResource.getSeqFiles().size(), mergeResource.getUnseqFiles().size()); + return Collections.singletonList( + new RewriteCrossSpaceCompactionTask( + logicalStorageGroupName, + virtualGroupId, + timePartition, + tsFileManager, + mergeFiles[0], + mergeFiles[1], + CompactionTaskManager.currentTaskNum)); } - } catch (MergeException | IOException | InterruptedException e) { + } catch (MergeException | IOException e) { LOGGER.error("{} cannot select file for cross space compaction", logicalStorageGroupName, e); } + return Collections.emptyList(); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java similarity index 79% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java index ffed6030ed695..2bcf6f31355f2 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java @@ -18,21 +18,22 @@ */ package org.apache.iotdb.db.engine.compaction.inner; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionSelector; +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; -public abstract class AbstractInnerSpaceCompactionSelector extends AbstractCompactionSelector { +public abstract class AbstractInnerSequenceSpaceCompactionSelector implements ICompactionSelector { protected String logicalStorageGroupName; protected String virtualStorageGroupName; protected long timePartition; protected List tsFileResources; - protected boolean sequence; protected TsFileManager tsFileManager; + protected boolean sequence; - public AbstractInnerSpaceCompactionSelector( + public AbstractInnerSequenceSpaceCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, @@ -43,14 +44,8 @@ public AbstractInnerSpaceCompactionSelector( this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.sequence = sequence; - if (sequence) { - tsFileResources = tsFileManager.getSequenceListByTimePartition(timePartition).getArrayList(); - } else { - tsFileResources = - tsFileManager.getUnsequenceListByTimePartition(timePartition).getArrayList(); - } } @Override - public abstract void selectAndSubmit(); + public abstract List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java new file mode 100644 index 0000000000000..c50c33ff99b4a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.inner; + +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; + +import java.util.List; + +public abstract class AbstractInnerUnsequenceSpaceCompactionSelector + implements ICompactionSelector { + public abstract List select(); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 814179862bdf2..6181f8a8449cb 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -36,7 +36,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; +import java.util.LinkedList; import java.util.List; import java.util.PriorityQueue; @@ -50,7 +52,7 @@ * file) to higher level files. If a compaction task is found in some level, selector will not * search higher level anymore. */ -public class SizeTieredCompactionSelector extends AbstractInnerSpaceCompactionSelector { +public class SizeTieredCompactionSelector extends AbstractInnerSequenceSpaceCompactionSelector { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); @@ -73,7 +75,7 @@ public SizeTieredCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public void selectAndSubmit() { + public List select() { PriorityQueue, Long>> taskPriorityQueue = new PriorityQueue<>(new SizeTieredCompactionTaskComparator()); try { @@ -83,12 +85,23 @@ public void selectAndSubmit() { break; } } + List taskList = new LinkedList<>(); while (taskPriorityQueue.size() > 0) { - createAndSubmitTask(taskPriorityQueue.poll().left); + taskList.add( + new SizeTieredCompactionTask( + logicalStorageGroupName, + virtualStorageGroupName, + timePartition, + tsFileManager, + taskPriorityQueue.poll().left, + sequence, + CompactionTaskManager.currentTaskNum)); } + return taskList; } catch (Exception e) { LOGGER.error("Exception occurs while selecting files", e); } + return Collections.emptyList(); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java similarity index 92% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionSelector.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java index 96cd85a5bc2f4..ed75918dae5f8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java @@ -18,12 +18,14 @@ */ package org.apache.iotdb.db.engine.compaction.task; +import java.util.List; + /** * AbstractCompactionSelector is the base class of all CompactionSelector. It runs the file * selection process, if there still threads availabe for compaction task, it will submit a * compaction task to {@link org.apache.iotdb.db.engine.compaction.CompactionTaskManager} and * increase the global compaction task count. */ -public abstract class AbstractCompactionSelector { - public abstract void selectAndSubmit(); +public interface ICompactionSelector { + List select(); } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java index 62853b5d9dcce..6874b3cece2e6 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java @@ -23,14 +23,14 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; public class CompactionConfigRestorer { private boolean enableSeqSpaceCompaction = true; private boolean enableUnseqSpaceCompaction = false; private boolean enableCrossSpaceCompaction = true; private CrossCompactionStrategy crossStrategy = CrossCompactionStrategy.REWRITE_COMPACTION; - private InnerCompactionStrategy innerStrategy = InnerCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerSequenceCompactionStrategy innerStrategy = InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; private CompactionPriority priority = CompactionPriority.BALANCE; private long targetFileSize = 1073741824L; private long targetChunkSize = 1048576L; From 5408c55f881b6194e2019292488d6d654b7ee1bc Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Thu, 7 Apr 2022 17:06:10 +0800 Subject: [PATCH 14/22] refactor compaction task selector --- .../resources/conf/iotdb-engine.properties | 8 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 24 +++- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +- .../compaction/CompactionMetricsManager.java | 12 +- .../compaction/CompactionScheduler.java | 46 ++++--- .../DefaultCompactionTaskComparatorImpl.java | 25 ++-- .../comparator/ICompactionTaskComparator.java | 8 +- .../constant/CrossCompactionStrategy.java | 9 +- .../InnerSequenceCompactionStrategy.java | 27 ---- .../InnerUnsequenceCompactionStrategy.java | 4 +- ...ask.java => CrossSpaceCompactionTask.java} | 36 ++++-- .../RewriteCrossSpaceCompactionSelector.java | 5 +- .../RewriteCrossSpaceCompactionTask.java | 67 ---------- ...tInnerSequenceSpaceCompactionSelector.java | 27 +--- ...nnerUnsequenceSpaceCompactionSelector.java | 5 +- ...ask.java => InnerSpaceCompactionTask.java} | 46 +++++-- .../SizeTieredCompactionSelector.java | 48 ++++---- .../sizetiered/SizeTieredCompactionTask.java | 84 ------------- .../AbstractCompactionPerformer.java | 10 ++ .../ReadChunkCompactionPerformer.java | 22 ++++ .../ReadPointCompactionPerformer.java | 13 ++ .../task/AbstractCompactionTask.java | 4 +- .../CompactionTaskComparatorTest.java | 30 ++--- .../compaction/CompactionTaskManagerTest.java | 115 +++++++++++++----- .../RewriteCrossSpaceCompactionTest.java | 38 +++--- .../inner/InnerCompactionSchedulerTest.java | 8 +- .../task/FakedCrossSpaceCompactionTask.java | 62 ---------- .../FakedCrossSpaceCompactionTaskFactory.java | 47 ------- .../task/FakedInnerSpaceCompactionTask.java | 97 --------------- .../utils/CompactionConfigRestorer.java | 5 +- .../StorageGroupProcessorTest.java | 8 +- 31 files changed, 369 insertions(+), 583 deletions(-) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/{AbstractCrossSpaceCompactionTask.java => CrossSpaceCompactionTask.java} (88%) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/{AbstractInnerSpaceCompactionTask.java => InnerSpaceCompactionTask.java} (88%) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java delete mode 100644 server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java delete mode 100644 server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java delete mode 100644 server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTask.java diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index d6598a5d1437c..edf385449b77f 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -427,9 +427,13 @@ timestamp_precision=ms # Options: rewrite_compaction # cross_compaction_strategy=rewrite_compaction -# the strategy of inner space compaction task +# the strategy of inner sequence space compaction task # Options: size_tiered_compaction -# inner_compaction_strategy=size_tiered_compaction +# inner_seq_compaction_strategy=size_tiered_compaction + +# the strategy of inner unsequence space compaction task +# Options: size_tiered_compaction +# inner_unseq_compaction_strategy=size_tiered_compaction # The priority of compaction execution # INNER_CROSS: prioritize inner space compaction, reduce the number of files first diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 0b278c8c0a826..3e0685b97b238 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionStrategy; import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; @@ -379,9 +380,12 @@ public class IoTDBConfig { * The strategy of inner space compaction task. There are just one inner space compaction strategy * SIZE_TIRED_COMPACTION: */ - private InnerSequenceCompactionStrategy innerCompactionStrategy = + private InnerSequenceCompactionStrategy innerSequenceCompactionStrategy = InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerUnsequenceCompactionStrategy innerUnsequenceCompactionStrategy = + InnerUnsequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + /** * The strategy of cross space compaction task. There are just one cross space compaction strategy * SIZE_TIRED_COMPACTION: @@ -2411,12 +2415,22 @@ public void setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompaction) { this.enableCrossSpaceCompaction = enableCrossSpaceCompaction; } - public InnerSequenceCompactionStrategy getInnerCompactionStrategy() { - return innerCompactionStrategy; + public InnerSequenceCompactionStrategy getInnerSequenceCompactionStrategy() { + return innerSequenceCompactionStrategy; + } + + public void setInnerSequenceCompactionStrategy( + InnerSequenceCompactionStrategy innerSequenceCompactionStrategy) { + this.innerSequenceCompactionStrategy = innerSequenceCompactionStrategy; + } + + public InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStrategy() { + return innerUnsequenceCompactionStrategy; } - public void setInnerCompactionStrategy(InnerSequenceCompactionStrategy innerCompactionStrategy) { - this.innerCompactionStrategy = innerCompactionStrategy; + public void setInnerUnsequenceCompactionStrategy( + InnerUnsequenceCompactionStrategy innerUnsequenceCompactionStrategy) { + this.innerUnsequenceCompactionStrategy = innerUnsequenceCompactionStrategy; } public CrossCompactionStrategy getCrossCompactionStrategy() { diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 92544fe98b4a8..f2cc2d2d6c78c 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionStrategy; import org.apache.iotdb.db.exception.BadNodeUrlFormatException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.utils.DatetimeUtils; @@ -364,10 +365,17 @@ private void loadProps() { properties.getProperty( "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString()))); - conf.setInnerCompactionStrategy( + conf.setInnerSequenceCompactionStrategy( InnerSequenceCompactionStrategy.getInnerSequenceCompactionStrategy( properties.getProperty( - "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString()))); + "inner_seq_compaction_strategy", + conf.getInnerSequenceCompactionStrategy().toString()))); + + conf.setInnerUnsequenceCompactionStrategy( + InnerUnsequenceCompactionStrategy.getInnerUnsequenceCompactionStrategy( + properties.getProperty( + "inner_unseq_compaction_strategy", + conf.getInnerUnsequenceCompactionStrategy().toString()))); conf.setCompactionPriority( CompactionPriority.valueOf( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionMetricsManager.java index 4c6b195ca58c4..f852dd9d61d25 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionMetricsManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionMetricsManager.java @@ -21,8 +21,8 @@ import org.apache.iotdb.db.engine.compaction.constant.CompactionTaskStatus; import org.apache.iotdb.db.engine.compaction.constant.CompactionType; import org.apache.iotdb.db.engine.compaction.constant.ProcessChunkType; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.service.metrics.Metric; import org.apache.iotdb.db.service.metrics.MetricsService; @@ -89,10 +89,10 @@ public static void recordTaskInfo( } String taskType = "unknown"; boolean isInnerTask = false; - if (task instanceof AbstractInnerSpaceCompactionTask) { + if (task instanceof InnerSpaceCompactionTask) { isInnerTask = true; taskType = "inner"; - } else if (task instanceof AbstractCrossSpaceCompactionTask) { + } else if (task instanceof CrossSpaceCompactionTask) { taskType = "cross"; } @@ -154,9 +154,7 @@ public static void recordTaskInfo( Tag.NAME.toString(), "inner_compaction_count", Tag.TYPE.toString(), - ((AbstractInnerSpaceCompactionTask) task).isSequence() - ? "sequence" - : "unsequence"); + ((InnerSpaceCompactionTask) task).isSequence() ? "sequence" : "unsequence"); } else { MetricsService.getInstance() .getMetricManager() diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index aea67027b11f5..48c4e07eaecdc 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -52,24 +53,24 @@ public static void scheduleCompaction(TsFileManager tsFileManager, long timePart return; } try { - tryToSubmitCrossSpaceCompactionTask( - tsFileManager.getStorageGroupName(), - tsFileManager.getVirtualStorageGroup(), - tsFileManager.getStorageGroupDir(), - timePartition, - tsFileManager); - tryToSubmitInnerSpaceCompactionTask( - tsFileManager.getStorageGroupName(), - tsFileManager.getVirtualStorageGroup(), - timePartition, - tsFileManager, - true); - tryToSubmitInnerSpaceCompactionTask( - tsFileManager.getStorageGroupName(), - tsFileManager.getVirtualStorageGroup(), - timePartition, - tsFileManager, - false); + tryToSubmitCrossSpaceCompactionTask( + tsFileManager.getStorageGroupName(), + tsFileManager.getVirtualStorageGroup(), + tsFileManager.getStorageGroupDir(), + timePartition, + tsFileManager); + tryToSubmitInnerSpaceCompactionTask( + tsFileManager.getStorageGroupName(), + tsFileManager.getVirtualStorageGroup(), + timePartition, + tsFileManager, + true); + tryToSubmitInnerSpaceCompactionTask( + tsFileManager.getStorageGroupName(), + tsFileManager.getVirtualStorageGroup(), + timePartition, + tsFileManager, + false); } catch (InterruptedException e) { LOGGER.error("Exception occurs when selecting compaction tasks", e); } @@ -91,12 +92,17 @@ public static void tryToSubmitInnerSpaceCompactionTask( if (sequence) { AbstractInnerSequenceSpaceCompactionSelector innerSpaceCompactionSelector = config - .getInnerCompactionStrategy() + .getInnerSequenceCompactionStrategy() .getCompactionSelector( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager); taskList = innerSpaceCompactionSelector.select(); } else { - + AbstractInnerUnsequenceSpaceCompactionSelector innerSpaceCompactionSelector = + config + .getInnerUnsequenceCompactionStrategy() + .getCompactionSelector( + logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager); + taskList = innerSpaceCompactionSelector.select(); } for (AbstractCompactionTask task : taskList) { CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java index 2c45c7e3273cd..04a213b7e684b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/DefaultCompactionTaskComparatorImpl.java @@ -22,8 +22,8 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -34,29 +34,28 @@ public class DefaultCompactionTaskComparatorImpl implements ICompactionTaskCompa @Override public int compare(AbstractCompactionTask o1, AbstractCompactionTask o2) { - if ((((o1 instanceof AbstractInnerSpaceCompactionTask) - && (o2 instanceof AbstractCrossSpaceCompactionTask)) - || ((o2 instanceof AbstractInnerSpaceCompactionTask) - && (o1 instanceof AbstractCrossSpaceCompactionTask)))) { + if ((((o1 instanceof InnerSpaceCompactionTask) && (o2 instanceof CrossSpaceCompactionTask)) + || ((o2 instanceof InnerSpaceCompactionTask) + && (o1 instanceof CrossSpaceCompactionTask)))) { if (config.getCompactionPriority() == CompactionPriority.BALANCE) { return 0; } else if (config.getCompactionPriority() == CompactionPriority.INNER_CROSS) { - return o1 instanceof AbstractInnerSpaceCompactionTask ? -1 : 1; + return o1 instanceof InnerSpaceCompactionTask ? -1 : 1; } else { - return o1 instanceof AbstractCrossSpaceCompactionTask ? -1 : 1; + return o1 instanceof CrossSpaceCompactionTask ? -1 : 1; } } - if (o1 instanceof AbstractInnerSpaceCompactionTask) { + if (o1 instanceof InnerSpaceCompactionTask) { return compareInnerSpaceCompactionTask( - (AbstractInnerSpaceCompactionTask) o1, (AbstractInnerSpaceCompactionTask) o2); + (InnerSpaceCompactionTask) o1, (InnerSpaceCompactionTask) o2); } else { return compareCrossSpaceCompactionTask( - (AbstractCrossSpaceCompactionTask) o1, (AbstractCrossSpaceCompactionTask) o2); + (CrossSpaceCompactionTask) o1, (CrossSpaceCompactionTask) o2); } } public int compareInnerSpaceCompactionTask( - AbstractInnerSpaceCompactionTask o1, AbstractInnerSpaceCompactionTask o2) { + InnerSpaceCompactionTask o1, InnerSpaceCompactionTask o2) { if (o1.isSequence() ^ o2.isSequence()) { // prioritize sequence file compaction return o1.isSequence() ? -1 : 1; @@ -98,7 +97,7 @@ public int compareInnerSpaceCompactionTask( } public int compareCrossSpaceCompactionTask( - AbstractCrossSpaceCompactionTask o1, AbstractCrossSpaceCompactionTask o2) { + CrossSpaceCompactionTask o1, CrossSpaceCompactionTask o2) { if (o1.getSelectedSequenceFiles().size() != o2.getSelectedSequenceFiles().size()) { // we prefer the task with fewer sequence files // because this type of tasks consume fewer memory during execution diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java index dcea8908eb33a..e8a2dae539197 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/comparator/ICompactionTaskComparator.java @@ -18,8 +18,8 @@ */ package org.apache.iotdb.db.engine.compaction.comparator; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import java.util.Comparator; @@ -28,8 +28,8 @@ public interface ICompactionTaskComparator extends Comparator selectedTsFileResourceList, - boolean sequence) { - switch (this) { - case SIZE_TIERED_COMPACTION: - default: - return new SizeTieredCompactionTask( - logicalStorageGroupName, - virtualStorageGroup, - timePartition, - tsFileManager, - selectedTsFileResourceList, - sequence, - CompactionTaskManager.currentTaskNum); - } - } - public AbstractInnerSequenceSpaceCompactionSelector getCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java index d77f9b3176666..270b47200e9ad 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.engine.compaction.constant; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -33,7 +33,7 @@ public static InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStra throw new RuntimeException("Illegal Compaction Strategy " + name); } - public AbstractInnerSequenceSpaceCompactionSelector getCompactionSelector( + public AbstractInnerUnsequenceSpaceCompactionSelector getCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java similarity index 88% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java index fc6df3ddd72be..4011ce688fcc7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -44,7 +45,7 @@ import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; -public abstract class AbstractCrossSpaceCompactionTask extends AbstractCompactionTask { +public class CrossSpaceCompactionTask extends AbstractCompactionTask { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); protected List selectedSequenceFiles; @@ -56,18 +57,25 @@ public abstract class AbstractCrossSpaceCompactionTask extends AbstractCompactio protected List holdReadLockList = new ArrayList<>(); protected List holdWriteLockList = new ArrayList<>(); - public AbstractCrossSpaceCompactionTask( - String fullStorageGroupName, + public CrossSpaceCompactionTask( + String logicalStorageGroup, + String virutalStorageGroupName, long timePartition, - AtomicInteger currentTaskNum, + TsFileManager tsFileManager, List selectedSequenceFiles, List selectedUnsequenceFiles, - TsFileManager tsFileManager) { - super(fullStorageGroupName, timePartition, tsFileManager, currentTaskNum); + AbstractCompactionPerformer performer, + AtomicInteger currentTaskNum) { + super( + logicalStorageGroup + "-" + virutalStorageGroupName, + timePartition, + tsFileManager, + currentTaskNum); this.selectedSequenceFiles = selectedSequenceFiles; this.selectedUnsequenceFiles = selectedUnsequenceFiles; this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition); + this.performer = performer; } @Override @@ -110,7 +118,10 @@ protected void doCompaction() throws Exception { // restart recovery compactionLogger.close(); - performCompaction(); + performer.setSeqFiles(selectedSequenceFiles); + performer.setUnseqFiles(unseqTsFileResourceList); + performer.setTargetFiles(targetTsfileResourceList); + performer.perform(); CompactionUtils.moveTargetFile(targetTsfileResourceList, false, fullStorageGroupName); CompactionUtils.combineModsInCrossCompaction( @@ -158,6 +169,17 @@ protected void doCompaction() throws Exception { } } + @Override + public boolean equalsOtherTask(AbstractCompactionTask otherTask) { + if (!(otherTask instanceof CrossSpaceCompactionTask)) { + return false; + } + CrossSpaceCompactionTask otherCrossCompactionTask = (CrossSpaceCompactionTask) otherTask; + return this.selectedSequenceFiles.equals(otherCrossCompactionTask.selectedSequenceFiles) + && this.selectedUnsequenceFiles.equals(otherCrossCompactionTask.selectedUnsequenceFiles) + && this.performer.getClass().isInstance(otherCrossCompactionTask.performer); + } + private void releaseAllLock() { selectedSequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); selectedUnsequenceFiles.forEach(x -> x.setStatus(TsFileResourceStatus.CLOSED)); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 56cc0141a2032..ba78e14f6b642 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -24,7 +24,9 @@ import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -118,13 +120,14 @@ public List select() { mergeResource.getSeqFiles().size(), mergeResource.getUnseqFiles().size()); return Collections.singletonList( - new RewriteCrossSpaceCompactionTask( + new CrossSpaceCompactionTask( logicalStorageGroupName, virtualGroupId, timePartition, tsFileManager, mergeFiles[0], mergeFiles[1], + new ReadPointCompactionPerformer(), CompactionTaskManager.currentTaskNum)); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java deleted file mode 100644 index daa7183cc2ec7..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionTask.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.cross.rewrite; - -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -public class RewriteCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { - - public RewriteCrossSpaceCompactionTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartitionId, - TsFileManager tsFileManager, - List selectedSeqTsFileResourceList, - List selectedUnSeqTsFileResourceList, - AtomicInteger currentTaskNum) { - super( - logicalStorageGroupName + "-" + virtualStorageGroupName, - timePartitionId, - currentTaskNum, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList, - tsFileManager); - } - - @Override - protected void performCompaction() throws Exception { - AbstractCompactionPerformer performer = - new ReadPointCompactionPerformer( - selectedSequenceFiles, selectedUnsequenceFiles, targetTsfileResourceList); - performer.perform(); - } - - @Override - public boolean equalsOtherTask(AbstractCompactionTask other) { - if (other instanceof RewriteCrossSpaceCompactionTask) { - RewriteCrossSpaceCompactionTask otherTask = (RewriteCrossSpaceCompactionTask) other; - return otherTask.selectedSequenceFiles.equals(selectedSequenceFiles) - && otherTask.selectedUnsequenceFiles.equals(selectedUnsequenceFiles); - } - return false; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java index 2bcf6f31355f2..88933dec56f2e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java @@ -20,32 +20,9 @@ import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; -public abstract class AbstractInnerSequenceSpaceCompactionSelector implements ICompactionSelector { - protected String logicalStorageGroupName; - protected String virtualStorageGroupName; - protected long timePartition; - protected List tsFileResources; - protected TsFileManager tsFileManager; - protected boolean sequence; - - public AbstractInnerSequenceSpaceCompactionSelector( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartition, - TsFileManager tsFileManager, - boolean sequence) { - this.logicalStorageGroupName = logicalStorageGroupName; - this.virtualStorageGroupName = virtualStorageGroupName; - this.timePartition = timePartition; - this.tsFileManager = tsFileManager; - this.sequence = sequence; - } - - @Override - public abstract List select(); +public interface AbstractInnerSequenceSpaceCompactionSelector extends ICompactionSelector { + List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java index c50c33ff99b4a..b82d75800ef32 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java @@ -23,7 +23,6 @@ import java.util.List; -public abstract class AbstractInnerUnsequenceSpaceCompactionSelector - implements ICompactionSelector { - public abstract List select(); +public interface AbstractInnerUnsequenceSpaceCompactionSelector extends ICompactionSelector { + List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java similarity index 88% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java index 7f87af2da8601..a9878b8c6a884 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -41,7 +42,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -public abstract class AbstractInnerSpaceCompactionTask extends AbstractCompactionTask { +public class InnerSpaceCompactionTask extends AbstractCompactionTask { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); @@ -57,16 +58,23 @@ public abstract class AbstractInnerSpaceCompactionTask extends AbstractCompactio protected boolean[] isHoldingReadLock; protected boolean[] isHoldingWriteLock; - public AbstractInnerSpaceCompactionTask( - String storageGroupName, + public InnerSpaceCompactionTask( + String logicalStorageGroupName, + String virtualStorageGroupName, long timePartition, - AtomicInteger currentTaskNum, - boolean sequence, + TsFileManager tsFileManager, List selectedTsFileResourceList, - TsFileManager tsFileManager) { - super(storageGroupName, timePartition, tsFileManager, currentTaskNum); + boolean sequence, + AbstractCompactionPerformer performer, + AtomicInteger currentTaskNum) { + super( + logicalStorageGroupName + "-" + virtualStorageGroupName, + timePartition, + tsFileManager, + currentTaskNum); this.selectedTsFileResourceList = selectedTsFileResourceList; this.sequence = sequence; + this.performer = performer; isHoldingReadLock = new boolean[selectedTsFileResourceList.size()]; isHoldingWriteLock = new boolean[selectedTsFileResourceList.size()]; for (int i = 0; i < selectedTsFileResourceList.size(); ++i) { @@ -109,18 +117,24 @@ protected void doCompaction() throws Exception { compactionLogger.logFiles(selectedTsFileResourceList, CompactionLogger.STR_SOURCE_FILES); compactionLogger.logFiles( Collections.singletonList(targetTsFileResource), CompactionLogger.STR_TARGET_FILES); - LOGGER.info("{} [SizeTiredCompactionTask] Close the logger", fullStorageGroupName); + LOGGER.info("{} [InnerSpaceCompactionTask] Close the logger", fullStorageGroupName); compactionLogger.close(); LOGGER.info( "{} [Compaction] compaction with {}", fullStorageGroupName, selectedTsFileResourceList); // carry out the compaction - performCompaction(); + if (sequence) { + performer.setSeqFiles(selectedTsFileResourceList); + } else { + performer.setUnseqFiles(selectedTsFileResourceList); + } + performer.setTargetFiles(Collections.singletonList(targetTsFileResource)); + performer.perform(); CompactionUtils.moveTargetFile( Collections.singletonList(targetTsFileResource), true, fullStorageGroupName); - LOGGER.info("{} [SizeTiredCompactionTask] start to rename mods file", fullStorageGroupName); + LOGGER.info("{} [InnerSpaceCompactionTask] start to rename mods file", fullStorageGroupName); CompactionUtils.combineModsInInnerCompaction( selectedTsFileResourceList, targetTsFileResource); @@ -176,7 +190,7 @@ protected void doCompaction() throws Exception { long costTime = System.currentTimeMillis() - startTime; LOGGER.info( - "{} [SizeTiredCompactionTask] all compaction task finish, target file is {}," + "{} [InnerSpaceCompactionTask] all compaction task finish, target file is {}," + "time cost is {} s", fullStorageGroupName, targetTsFileResource.getTsFile().getName(), @@ -222,6 +236,16 @@ protected void doCompaction() throws Exception { } } + @Override + public boolean equalsOtherTask(AbstractCompactionTask otherTask) { + if (!(otherTask instanceof InnerSpaceCompactionTask)) { + return false; + } + InnerSpaceCompactionTask task = (InnerSpaceCompactionTask) otherTask; + return this.selectedTsFileResourceList.equals(task.selectedTsFileResourceList) + && this.performer.getClass().isInstance(task.performer); + } + @Override public void setSourceFilesToCompactionCandidate() { this.selectedTsFileResourceList.forEach( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 6181f8a8449cb..f199402c27846 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -22,8 +22,10 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -47,15 +49,23 @@ * selector traverses the file list from old to new. If the size of selected files or the number of * select files exceed given threshold, a compaction task will be submitted to task queue in * CompactionTaskManager. In CompactionTaskManager, tasks are ordered by {@link - * DefaultCompactionTaskComparatorImpl}. To maximize compaction efficiency, selector searches - * compaction task from 0 compaction files(that is, file that never been compacted, named level 0 - * file) to higher level files. If a compaction task is found in some level, selector will not - * search higher level anymore. + * org.apache.iotdb.db.engine.compaction.comparator.ICompactionTaskComparator}. To maximize + * compaction efficiency, selector searches compaction task from 0 compaction files(that is, file + * that never been compacted, named level 0 file) to higher level files. If a compaction task is + * found in some level, selector will not search higher level anymore. */ -public class SizeTieredCompactionSelector extends AbstractInnerSequenceSpaceCompactionSelector { +public class SizeTieredCompactionSelector + implements AbstractInnerSequenceSpaceCompactionSelector, + AbstractInnerUnsequenceSpaceCompactionSelector { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + protected String logicalStorageGroupName; + protected String virtualStorageGroupName; + protected long timePartition; + protected List tsFileResources; + protected TsFileManager tsFileManager; + protected boolean sequence; public SizeTieredCompactionSelector( String logicalStorageGroupName, @@ -63,7 +73,11 @@ public SizeTieredCompactionSelector( long timePartition, TsFileManager tsFileManager, boolean sequence) { - super(logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, sequence); + this.logicalStorageGroupName = logicalStorageGroupName; + this.virtualStorageGroupName = virtualStorageGroupName; + this.timePartition = timePartition; + this.tsFileManager = tsFileManager; + this.sequence = sequence; } /** @@ -87,14 +101,16 @@ public List select() { } List taskList = new LinkedList<>(); while (taskPriorityQueue.size() > 0) { + List resources = taskPriorityQueue.poll().left; taskList.add( - new SizeTieredCompactionTask( + new InnerSpaceCompactionTask( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, - taskPriorityQueue.poll().left, + resources, sequence, + new ReadChunkCompactionPerformer(), CompactionTaskManager.currentTaskNum)); } return taskList; @@ -170,20 +186,6 @@ private int searchMaxFileLevel() throws IOException { return maxLevel; } - private void createAndSubmitTask(List selectedFileList) - throws InterruptedException { - AbstractCompactionTask compactionTask = - new SizeTieredCompactionTask( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - selectedFileList, - sequence, - CompactionTaskManager.currentTaskNum); - CompactionTaskManager.getInstance().addTaskToWaitingQueue(compactionTask); - } - private class SizeTieredCompactionTaskComparator implements Comparator, Long>> { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java deleted file mode 100644 index e5c156a1ac3c1..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionTask.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.inner.sizetiered; - -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * SizeTiredCompactionTask compact several inner space files selected by {@link - * SizeTieredCompactionSelector} into one file. - */ -public class SizeTieredCompactionTask extends AbstractInnerSpaceCompactionTask { - - protected AbstractCompactionPerformer performer; - - public SizeTieredCompactionTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartition, - TsFileManager tsFileManager, - List selectedTsFileResourceList, - boolean sequence, - AtomicInteger currentTaskNum) { - super( - logicalStorageGroupName + "-" + virtualStorageGroupName, - timePartition, - currentTaskNum, - sequence, - selectedTsFileResourceList, - tsFileManager); - } - - @Override - protected void performCompaction() throws Exception { - if (sequence) { - performer = - new ReadChunkCompactionPerformer(selectedTsFileResourceList, targetTsFileResource); - } else { - performer = - new ReadPointCompactionPerformer( - Collections.emptyList(), - selectedTsFileResourceList, - Collections.singletonList(targetTsFileResource)); - } - performer.perform(); - } - - @Override - public boolean equalsOtherTask(AbstractCompactionTask other) { - if (other instanceof SizeTieredCompactionTask) { - SizeTieredCompactionTask otherSizeTieredTask = (SizeTieredCompactionTask) other; - if (!selectedTsFileResourceList.equals(otherSizeTieredTask.selectedTsFileResourceList)) { - return false; - } - return true; - } - return false; - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java index 5dbfa741fd1d4..441856cfd2046 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java @@ -37,4 +37,14 @@ public abstract class AbstractCompactionPerformer { public abstract void perform() throws IOException, MetadataException, StorageEngineException, InterruptedException; + + public abstract void setTargetFiles(List targetFiles); + + public void setSeqFiles(List seqFiles) { + this.seqFiles = seqFiles; + } + + public void setUnseqFiles(List unseqFiles) { + this.unseqFiles = unseqFiles; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java index 67268a95160a5..9ca2a16bcc680 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java @@ -54,6 +54,12 @@ public ReadChunkCompactionPerformer(List sourceFiles, TsFileReso this.targetResource = targetFile; } + public ReadChunkCompactionPerformer(List sourceFiles) { + this.seqFiles = sourceFiles; + } + + public ReadChunkCompactionPerformer() {} + @Override public void perform() throws IOException, MetadataException, InterruptedException, StorageEngineException { @@ -81,6 +87,22 @@ public void perform() } } + @Override + public void setTargetFiles(List targetFiles) { + if (targetFiles.size() != 1) { + throw new RuntimeException( + String.format( + "Current performer only supports for one target file while getting %d target files", + targetFiles.size())); + } + this.targetResource = targetFiles.get(0); + } + + @Override + public void setUnseqFiles(List unseqFiles) { + throw new RuntimeException("The performer of this class cannot compact with unseq files!"); + } + private void compactAlignedSeries( String device, TsFileResource targetResource, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java index 1ad1af53c5ad6..703d487d25648 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java @@ -69,6 +69,14 @@ public ReadPointCompactionPerformer( this.targetFiles = targetFiles; } + public ReadPointCompactionPerformer( + List seqFiles, List unseqFiles) { + this.seqFiles = seqFiles; + this.unseqFiles = unseqFiles; + } + + public ReadPointCompactionPerformer() {} + @Override public void perform() throws IOException, MetadataException, StorageEngineException, InterruptedException { @@ -107,6 +115,11 @@ public void perform() } } + @Override + public void setTargetFiles(List targetFiles) { + this.targetFiles = targetFiles; + } + private void compactAlignedSeries( String device, MultiTsFileDeviceIterator deviceIterator, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java index 792d9c5a87fdf..e8aeb7f6d3f15 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; +import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.slf4j.Logger; @@ -45,6 +46,7 @@ public abstract class AbstractCompactionTask implements Callable { protected long timeCost = 0L; protected volatile boolean ran = false; protected volatile boolean finished = false; + protected AbstractCompactionPerformer performer; public AbstractCompactionTask( String fullStorageGroupName, @@ -61,8 +63,6 @@ public AbstractCompactionTask( protected abstract void doCompaction() throws Exception; - protected abstract void performCompaction() throws Exception; - @Override public Void call() throws Exception { ran = true; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 308ea31c930b8..4629b45628e16 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -21,8 +21,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.comparator.DefaultCompactionTaskComparatorImpl; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -264,7 +266,7 @@ public void testComparationOfCrossSpaceTask() throws InterruptedException { } } - private static class FakedInnerSpaceCompactionTask extends AbstractInnerSpaceCompactionTask { + private static class FakedInnerSpaceCompactionTask extends InnerSpaceCompactionTask { public FakedInnerSpaceCompactionTask( String storageGroupName, @@ -275,19 +277,18 @@ public FakedInnerSpaceCompactionTask( List selectedTsFileResourceList) { super( storageGroupName, + "0", timePartition, - currentTaskNum, - sequence, + tsFileManager, selectedTsFileResourceList, - tsFileManager); + sequence, + new ReadChunkCompactionPerformer(), + currentTaskNum); } @Override protected void doCompaction() throws Exception {} - @Override - protected void performCompaction() throws Exception {} - @Override public boolean equalsOtherTask(AbstractCompactionTask other) { return false; @@ -299,7 +300,7 @@ public boolean checkValidAndSetMerging() { } } - private static class FakeCrossSpaceCompactionTask extends AbstractCrossSpaceCompactionTask { + private static class FakeCrossSpaceCompactionTask extends CrossSpaceCompactionTask { public FakeCrossSpaceCompactionTask( String fullStorageGroupName, @@ -310,19 +311,18 @@ public FakeCrossSpaceCompactionTask( List selectedUnsequenceFiles) { super( fullStorageGroupName, + "0", timePartition, - currentTaskNum, + tsFileManager, selectedSequenceFiles, selectedUnsequenceFiles, - tsFileManager); + new ReadPointCompactionPerformer(), + currentTaskNum); } @Override protected void doCompaction() throws Exception {} - @Override - protected void performCompaction() throws Exception {} - @Override public boolean equalsOtherTask(AbstractCompactionTask other) { return false; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java index 6042730f3a034..46f598c6982b0 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java @@ -19,9 +19,11 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.db.constant.TestConstant; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionTest; -import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -68,12 +70,26 @@ public void testRepeatedSubmitBeforeExecution() throws Exception { TsFileManager tsFileManager = new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); - SizeTieredCompactionTask task1 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); - SizeTieredCompactionTask task2 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); + InnerSpaceCompactionTask task1 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); + InnerSpaceCompactionTask task2 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); seqResources.get(0).readLock(); CompactionTaskManager manager = CompactionTaskManager.getInstance(); try { @@ -119,12 +135,26 @@ public void testRepeatedSubmitWhenExecuting() throws Exception { TsFileManager tsFileManager = new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); - SizeTieredCompactionTask task1 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); - SizeTieredCompactionTask task2 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); + InnerSpaceCompactionTask task1 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); + InnerSpaceCompactionTask task2 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); seqResources.get(0).readLock(); try { CompactionTaskManager manager = CompactionTaskManager.getInstance(); @@ -171,12 +201,26 @@ public void testRepeatedSubmitAfterExecution() throws Exception { TsFileManager tsFileManager = new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); - SizeTieredCompactionTask task1 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); - SizeTieredCompactionTask task2 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); + InnerSpaceCompactionTask task1 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); + InnerSpaceCompactionTask task2 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); CompactionTaskManager manager = CompactionTaskManager.getInstance(); manager.addTaskToWaitingQueue(task1); manager.submitTaskFromTaskQueue(); @@ -209,9 +253,16 @@ public void testRemoveSelfFromRunningList() throws Exception { TsFileManager tsFileManager = new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); - SizeTieredCompactionTask task1 = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); + InnerSpaceCompactionTask task1 = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); CompactionTaskManager manager = CompactionTaskManager.getInstance(); manager.restart(); seqResources.get(0).readLock(); @@ -248,9 +299,16 @@ public void testSizeTieredCompactionStatus() throws Exception { TsFileManager tsFileManager = new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); - SizeTieredCompactionTask task = - new SizeTieredCompactionTask( - "root.compactionTest", "0", 0, tsFileManager, seqResources, true, new AtomicInteger(0)); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + "root.compactionTest", + "0", + 0, + tsFileManager, + seqResources, + true, + new ReadChunkCompactionPerformer(seqResources), + new AtomicInteger(0)); CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); for (TsFileResource resource : seqResources) { @@ -270,14 +328,15 @@ public void testRewriteCrossCompactionFileStatus() throws Exception { new TsFileManager("root.compactionTest", "0", tempSGDir.getAbsolutePath()); tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); - RewriteCrossSpaceCompactionTask task = - new RewriteCrossSpaceCompactionTask( + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( COMPACTION_TEST_SG, "0", 0, tsFileManager, seqResources, unseqResources, + new ReadPointCompactionPerformer(), new AtomicInteger(0)); for (TsFileResource resource : seqResources) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java index eed4722bc2176..1a65e09a7d30b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -219,16 +219,17 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInTimeseries() thro new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath()); tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); - RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask = - new RewriteCrossSpaceCompactionTask( + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( COMPACTION_TEST_SG, "0", 0, tsFileManager, seqResources, unseqResources, + new ReadPointCompactionPerformer(), new AtomicInteger(0)); - rewriteCrossSpaceCompactionTask.call(); + task.call(); for (TsFileResource resource : seqResources) { resource.resetModFile(); @@ -456,16 +457,17 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() t new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath()); tsFileManager.addAll(seqResources, true); tsFileManager.addAll(unseqResources, false); - RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask = - new RewriteCrossSpaceCompactionTask( + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( COMPACTION_TEST_SG, "0", 0, tsFileManager, seqResources, unseqResources, + new ReadPointCompactionPerformer(), new AtomicInteger(0)); - rewriteCrossSpaceCompactionTask.call(); + task.call(); for (TsFileResource resource : seqResources) { Assert.assertFalse(resource.getModFile().exists()); @@ -603,17 +605,18 @@ public void testOneDeletionDuringCompaction() throws Exception { 0, null); - RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask = - new RewriteCrossSpaceCompactionTask( + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( COMPACTION_TEST_SG, "0", 0, vsgp.getTsFileResourceManager(), seqResources, unseqResources, + new ReadPointCompactionPerformer(), new AtomicInteger(0)); - rewriteCrossSpaceCompactionTask.setSourceFilesToCompactionCandidate(); - rewriteCrossSpaceCompactionTask.checkValidAndSetMerging(); + task.setSourceFilesToCompactionCandidate(); + task.checkValidAndSetMerging(); // delete data in source file during compaction vsgp.delete( new PartialPath( @@ -646,7 +649,7 @@ public void testOneDeletionDuringCompaction() throws Exception { Assert.assertTrue(resource.getModFile().exists()); Assert.assertEquals(2, resource.getModFile().getModifications().size()); } - rewriteCrossSpaceCompactionTask.call(); + task.call(); for (TsFileResource resource : seqResources) { Assert.assertFalse(resource.getTsFile().exists()); Assert.assertFalse(resource.getModFile().exists()); @@ -713,17 +716,18 @@ public void testSeveralDeletionsDuringCompaction() throws Exception { 0, null); - RewriteCrossSpaceCompactionTask rewriteCrossSpaceCompactionTask = - new RewriteCrossSpaceCompactionTask( + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( COMPACTION_TEST_SG, "0", 0, vsgp.getTsFileResourceManager(), seqResources, unseqResources, + new ReadPointCompactionPerformer(), new AtomicInteger(0)); - rewriteCrossSpaceCompactionTask.setSourceFilesToCompactionCandidate(); - rewriteCrossSpaceCompactionTask.checkValidAndSetMerging(); + task.setSourceFilesToCompactionCandidate(); + task.checkValidAndSetMerging(); // delete data in source file during compaction vsgp.delete( new PartialPath( @@ -768,7 +772,7 @@ public void testSeveralDeletionsDuringCompaction() throws Exception { Assert.assertTrue(resource.getModFile().exists()); Assert.assertEquals(3, resource.getModFile().getModifications().size()); } - rewriteCrossSpaceCompactionTask.call(); + task.call(); for (TsFileResource resource : seqResources) { Assert.assertFalse(resource.getTsFile().exists()); Assert.assertFalse(resource.getModFile().exists()); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java index d2f38befdc6bc..2a6d2e26ea1da 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java @@ -77,7 +77,8 @@ public void tearDown() throws IOException, StorageEngineException { } @Test - public void testFileSelector1() throws IOException, MetadataException, WriteProcessException { + public void testFileSelector1() + throws IOException, MetadataException, WriteProcessException, InterruptedException { IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(true); IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(true); IoTDBDescriptor.getInstance().getConfig().setConcurrentCompactionThread(50); @@ -103,7 +104,8 @@ public void testFileSelector1() throws IOException, MetadataException, WriteProc } @Test - public void testFileSelector2() throws IOException, MetadataException, WriteProcessException { + public void testFileSelector2() + throws IOException, MetadataException, WriteProcessException, InterruptedException { IoTDBDescriptor.getInstance().getConfig().setConcurrentCompactionThread(50); IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(50); TsFileResourceList tsFileResources = new TsFileResourceList(); @@ -134,7 +136,7 @@ public void testFileSelector2() throws IOException, MetadataException, WriteProc @Test public void testFileSelectorWithUnclosedFile() - throws IOException, MetadataException, WriteProcessException { + throws IOException, MetadataException, WriteProcessException, InterruptedException { IoTDBDescriptor.getInstance().getConfig().setConcurrentCompactionThread(50); IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(50); TsFileResourceList tsFileResources = new TsFileResourceList(); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java deleted file mode 100644 index 777e16aadb783..0000000000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTask.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.task; - -import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.FakedTsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.List; - -public class FakedCrossSpaceCompactionTask extends RewriteCrossSpaceCompactionTask { - public FakedCrossSpaceCompactionTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartitionId, - TsFileManager tsFileManager, - List selectedSeqTsFileResourceList, - List selectedUnSeqTsFileResourceList) { - super( - logicalStorageGroupName, - virtualStorageGroupName, - timePartitionId, - tsFileManager, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList, - CompactionTaskManager.currentTaskNum); - } - - @Override - protected void doCompaction() { - long totalUnseqFileSize = 0; - for (TsFileResource resource : selectedUnsequenceFiles) { - totalUnseqFileSize += resource.getTsFileSize(); - } - long avgSizeAddToSeqFile = totalUnseqFileSize / selectedSequenceFiles.size(); - for (TsFileResource resource : selectedSequenceFiles) { - ((FakedTsFileResource) resource) - .setTsFileSize(resource.getTsFileSize() + avgSizeAddToSeqFile); - } - selectedSequenceFiles.clear(); - selectedUnsequenceFiles.clear(); - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java deleted file mode 100644 index 1fb4b2e3fc4bf..0000000000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedCrossSpaceCompactionTaskFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.engine.compaction.task; - -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.List; - -public class FakedCrossSpaceCompactionTaskFactory { - public AbstractCompactionTask createTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartitionId, - TsFileManager tsFileManager, - List selectedSeqTsFileResourceList, - List selectedUnSeqTsFileResourceList) { - return IoTDBDescriptor.getInstance() - .getConfig() - .getCrossCompactionStrategy() - .getCompactionTask( - logicalStorageGroupName, - virtualStorageGroupName, - timePartitionId, - tsFileManager, - selectedSeqTsFileResourceList, - selectedUnSeqTsFileResourceList); - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTask.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTask.java deleted file mode 100644 index 8eafa19e75c4a..0000000000000 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/task/FakedInnerSpaceCompactionTask.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.task; - -import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; -import org.apache.iotdb.db.engine.storagegroup.FakedTsFileResource; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -public class FakedInnerSpaceCompactionTask extends SizeTieredCompactionTask { - - public FakedInnerSpaceCompactionTask( - String logicalStorageGroupName, - String virtualStorageGroupName, - long timePartition, - TsFileManager tsFileManager, - List selectedTsFileResourceList, - boolean sequence, - AtomicInteger currentTaskNum) { - super( - logicalStorageGroupName, - virtualStorageGroupName, - timePartition, - tsFileManager, - selectedTsFileResourceList, - sequence, - currentTaskNum); - } - - @Override - protected void doCompaction() throws IOException { - try { - TsFileNameGenerator.TsFileName name = - TsFileNameGenerator.getTsFileName( - selectedTsFileResourceList.get(0).getTsFile().getName()); - String newName = - TsFileNameGenerator.generateNewTsFileName( - name.getTime(), - name.getVersion(), - name.getInnerCompactionCnt() + 1, - name.getCrossCompactionCnt()); - FakedTsFileResource targetTsFileResource = new FakedTsFileResource(0, newName); - long targetFileSize = 0; - for (TsFileResource resource : selectedTsFileResourceList) { - targetFileSize += resource.getTsFileSize(); - } - targetTsFileResource.setTsFileSize(targetFileSize); - this.tsFileResourceList.insertBefore(selectedTsFileResourceList.get(0), targetTsFileResource); - for (TsFileResource tsFileResource : selectedTsFileResourceList) { - this.tsFileResourceList.remove(tsFileResource); - } - } finally { - CompactionTaskManager.getInstance().removeRunningTaskFromList(this); - } - } - - @Override - public boolean equalsOtherTask(AbstractCompactionTask otherTask) { - if (otherTask instanceof FakedInnerSpaceCompactionTask) { - FakedInnerSpaceCompactionTask fakedOtherTask = (FakedInnerSpaceCompactionTask) otherTask; - return this.selectedTsFileResourceList.equals(fakedOtherTask.selectedTsFileResourceList); - } - return false; - } - - @Override - public boolean checkValidAndSetMerging() { - for (TsFileResource resource : selectedTsFileResourceList) { - if (resource.isCompacting() || !resource.isClosed()) { - return false; - } - } - return true; - } -} diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java index 6874b3cece2e6..a678cc12f91ff 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java @@ -30,7 +30,8 @@ public class CompactionConfigRestorer { private boolean enableUnseqSpaceCompaction = false; private boolean enableCrossSpaceCompaction = true; private CrossCompactionStrategy crossStrategy = CrossCompactionStrategy.REWRITE_COMPACTION; - private InnerSequenceCompactionStrategy innerStrategy = InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerSequenceCompactionStrategy innerStrategy = + InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; private CompactionPriority priority = CompactionPriority.BALANCE; private long targetFileSize = 1073741824L; private long targetChunkSize = 1048576L; @@ -52,7 +53,7 @@ public void restoreCompactionConfig() { config.setEnableUnseqSpaceCompaction(enableUnseqSpaceCompaction); config.setEnableCrossSpaceCompaction(enableCrossSpaceCompaction); config.setCrossCompactionStrategy(crossStrategy); - config.setInnerCompactionStrategy(innerStrategy); + config.setInnerSequenceCompactionStrategy(innerStrategy); config.setCompactionPriority(priority); config.setTargetCompactionFileSize(targetFileSize); config.setTargetChunkSize(targetChunkSize); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java index 837356e933164..74999fb2f74e5 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java @@ -25,8 +25,9 @@ import org.apache.iotdb.db.engine.MetadataManagerHelper; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; +import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.flush.FlushManager; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; @@ -721,14 +722,15 @@ public void testDeleteStorageGroupWhenCompacting() throws Exception { processor.asyncCloseAllWorkingTsFileProcessors(); } processor.syncCloseAllWorkingTsFileProcessors(); - SizeTieredCompactionTask task = - new SizeTieredCompactionTask( + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( storageGroup, "0", 0, processor.getTsFileManager(), processor.getSequenceFileList(), true, + new ReadChunkCompactionPerformer(processor.getSequenceFileList()), new AtomicInteger(0)); CompactionTaskManager.getInstance().submitTask(task); Thread.sleep(20); From db80cc467e7f1d0f114b21d4eaf28c72e0daf7f6 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Thu, 7 Apr 2022 17:41:21 +0800 Subject: [PATCH 15/22] try to fix ci --- .../inner/sizetiered/SizeTieredCompactionSelector.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index f199402c27846..e6ad4982ed61e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -78,6 +78,11 @@ public SizeTieredCompactionSelector( this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.sequence = sequence; + if (sequence) { + this.tsFileResources = tsFileManager.getSequenceListByTimePartition(timePartition); + } else { + this.tsFileResources = tsFileManager.getUnsequenceListByTimePartition(timePartition); + } } /** From dd659eb95075eff2cc20456cf2d6584edb5eaf62 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Thu, 7 Apr 2022 19:51:24 +0800 Subject: [PATCH 16/22] fix ci --- .../db/engine/compaction/cross/CrossSpaceCompactionTask.java | 2 +- .../inner/sizetiered/SizeTieredCompactionSelector.java | 3 ++- .../compaction/performer/AbstractCompactionPerformer.java | 5 +++-- .../compaction/performer/ReadPointCompactionPerformer.java | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java index 4011ce688fcc7..429688b02bda4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java @@ -119,7 +119,7 @@ protected void doCompaction() throws Exception { compactionLogger.close(); performer.setSeqFiles(selectedSequenceFiles); - performer.setUnseqFiles(unseqTsFileResourceList); + performer.setUnseqFiles(selectedUnsequenceFiles); performer.setTargetFiles(targetTsfileResourceList); performer.perform(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index e6ad4982ed61e..5bfe07fd35ae0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -115,7 +116,7 @@ public List select() { tsFileManager, resources, sequence, - new ReadChunkCompactionPerformer(), + sequence ? new ReadChunkCompactionPerformer() : new ReadPointCompactionPerformer(), CompactionTaskManager.currentTaskNum)); } return taskList; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java index 441856cfd2046..1c2672625b692 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException; import java.io.IOException; +import java.util.Collections; import java.util.List; /** @@ -32,8 +33,8 @@ * all kinds of Performer can be used for all kinds of compaction tasks! */ public abstract class AbstractCompactionPerformer { - protected List seqFiles; - protected List unseqFiles; + protected List seqFiles = Collections.emptyList(); + protected List unseqFiles = Collections.emptyList(); public abstract void perform() throws IOException, MetadataException, StorageEngineException, InterruptedException; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java index 703d487d25648..1de635c68ac92 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java @@ -58,7 +58,7 @@ public class ReadPointCompactionPerformer extends AbstractCompactionPerformer { private Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - private List targetFiles; + private List targetFiles = Collections.emptyList(); public ReadPointCompactionPerformer( List seqFiles, From 1eeda27093d1d76ed5b7c3e1be183a0593e08e8f Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 8 Apr 2022 17:44:25 +0800 Subject: [PATCH 17/22] refactor CompactionPerformer --- .../compaction/CompactionScheduler.java | 12 ++--- .../compaction/CompactionTaskManager.java | 2 +- .../constant/CrossCompactionStrategy.java | 6 +-- .../InnerSequenceCompactionStrategy.java | 4 +- .../InnerUnsequenceCompactionStrategy.java | 4 +- .../AbstractCrossSpaceCompactionSelector.java | 54 ------------------- .../cross/CrossSpaceCompactionTask.java | 7 ++- .../compaction/cross/ICrossSpaceSelector.java | 29 ++++++++++ .../RewriteCrossSpaceCompactionSelector.java | 24 +++++---- ...ector.java => IInnerSeqSpaceSelector.java} | 2 +- ...tor.java => IInnerUnseqSpaceSelector.java} | 2 +- .../inner/InnerSpaceCompactionTask.java | 10 ++-- .../SizeTieredCompactionSelector.java | 11 ++-- ...rformer.java => ICompactionPerformer.java} | 18 +++---- .../performer/ICrossCompactionPerformer.java | 27 ++++++++++ .../ISeqCompactionPerformer.java} | 12 ++--- .../performer/IUnseqCompactionPerformer.java | 27 ++++++++++ .../ReadChunkCompactionPerformer.java | 16 +++--- .../ReadPointCompactionPerformer.java | 20 ++++++- .../task/AbstractCompactionTask.java | 6 +-- .../CompactionTaskComparatorTest.java | 4 +- .../compaction/CompactionTaskManagerTest.java | 4 +- .../ReadPointCompactionPerformerTest.java | 2 +- .../CrossSpaceCompactionExceptionTest.java | 2 +- ...ewriteCrossSpaceCompactionRecoverTest.java | 2 +- .../RewriteCrossSpaceCompactionTest.java | 2 +- .../inner/InnerSeqCompactionTest.java | 2 +- .../InnerSpaceCompactionExceptionTest.java | 2 +- .../inner/InnerUnseqCompactionTest.java | 2 +- ...adChunkCompactionPerformerAlignedTest.java | 2 +- ...ChunkCompactionPerformerNoAlignedTest.java | 2 +- .../ReadChunkCompactionPerformerOldTest.java | 2 +- .../SizeTieredCompactionRecoverTest.java | 2 +- ...TieredCompactionRecoverCompatibleTest.java | 2 +- .../SizeTieredCompactionRecoverTest.java | 2 +- .../StorageGroupProcessorTest.java | 2 +- 36 files changed, 187 insertions(+), 142 deletions(-) delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/{AbstractInnerSequenceSpaceCompactionSelector.java => IInnerSeqSpaceSelector.java} (92%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/{AbstractInnerUnsequenceSpaceCompactionSelector.java => IInnerUnseqSpaceSelector.java} (91%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/{AbstractCompactionPerformer.java => ICompactionPerformer.java} (75%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICrossCompactionPerformer.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/{constant/MergeFileStrategy.java => performer/ISeqCompactionPerformer.java} (75%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/IUnseqCompactionPerformer.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/{ => impl}/ReadChunkCompactionPerformer.java (94%) rename server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/{ => impl}/ReadPointCompactionPerformer.java (94%) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index 48c4e07eaecdc..961320d1ae47e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -22,9 +22,9 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -90,14 +90,14 @@ public static void tryToSubmitInnerSpaceCompactionTask( List taskList = null; if (sequence) { - AbstractInnerSequenceSpaceCompactionSelector innerSpaceCompactionSelector = + IInnerSeqSpaceSelector innerSpaceCompactionSelector = config .getInnerSequenceCompactionStrategy() .getCompactionSelector( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager); taskList = innerSpaceCompactionSelector.select(); } else { - AbstractInnerUnsequenceSpaceCompactionSelector innerSpaceCompactionSelector = + IInnerUnseqSpaceSelector innerSpaceCompactionSelector = config .getInnerUnsequenceCompactionStrategy() .getCompactionSelector( @@ -119,7 +119,7 @@ private static void tryToSubmitCrossSpaceCompactionTask( if (!config.isEnableCrossSpaceCompaction()) { return; } - AbstractCrossSpaceCompactionSelector crossSpaceCompactionSelector = + ICrossSpaceSelector crossSpaceCompactionSelector = config .getCrossCompactionStrategy() .getCompactionSelector( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java index fc1201fca7ca4..3fc594f4a12f4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java @@ -280,7 +280,7 @@ public static void mergeRateLimiterAcquire(RateLimiter limiter, long bytesLength } } - public synchronized void removeRunningTaskFromList(AbstractCompactionTask task) { + public synchronized void removeRunningTaskFuture(AbstractCompactionTask task) { String storageGroupName = task.getFullStorageGroupName(); if (storageGroupTasks.containsKey(storageGroupName)) { storageGroupTasks.get(storageGroupName).remove(task); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java index 2d614a94c2475..f64b10034ed83 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java @@ -19,10 +19,10 @@ package org.apache.iotdb.db.engine.compaction.constant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -60,7 +60,7 @@ public CrossSpaceCompactionTask getCompactionTask( } } - public AbstractCrossSpaceCompactionSelector getCompactionSelector( + public ICrossSpaceSelector getCompactionSelector( String logicalStorageGroupName, String virtualGroupId, String storageGroupDir, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java index 396fb799ad6fc..e3824d3255776 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.engine.compaction.constant; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -33,7 +33,7 @@ public static InnerSequenceCompactionStrategy getInnerSequenceCompactionStrategy throw new RuntimeException("Illegal Compaction Strategy " + name); } - public AbstractInnerSequenceSpaceCompactionSelector getCompactionSelector( + public IInnerSeqSpaceSelector getCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java index 270b47200e9ad..1e9338a171ff8 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.engine.compaction.constant; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector; import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -33,7 +33,7 @@ public static InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStra throw new RuntimeException("Illegal Compaction Strategy " + name); } - public AbstractInnerUnsequenceSpaceCompactionSelector getCompactionSelector( + public IInnerUnseqSpaceSelector getCompactionSelector( String logicalStorageGroupName, String virtualStorageGroupName, long timePartition, diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java deleted file mode 100644 index d53da2b99be5c..0000000000000 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/AbstractCrossSpaceCompactionSelector.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.engine.compaction.cross; - -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; -import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; -import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -import org.apache.iotdb.db.engine.storagegroup.TsFileResource; - -import java.util.List; - -public abstract class AbstractCrossSpaceCompactionSelector implements ICompactionSelector { - protected String logicalStorageGroupName; - protected String virtualGroupId; - protected String storageGroupDir; - protected long timePartition; - protected TsFileManager tsFileManager; - protected List sequenceFileList; - protected List unsequenceFileList; - - public AbstractCrossSpaceCompactionSelector( - String logicalStorageGroupName, - String virtualGroupId, - String storageGroupDir, - long timePartition, - TsFileManager tsFileManager) { - this.logicalStorageGroupName = logicalStorageGroupName; - this.virtualGroupId = virtualGroupId; - this.storageGroupDir = storageGroupDir; - this.timePartition = timePartition; - this.sequenceFileList = tsFileManager.getSequenceListByTimePartition(timePartition); - this.unsequenceFileList = tsFileManager.getUnsequenceListByTimePartition(timePartition); - this.tsFileManager = tsFileManager; - } - - @Override - public abstract List select(); -} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java index 429688b02bda4..1cae381a0eec9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ICrossCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -64,7 +64,7 @@ public CrossSpaceCompactionTask( TsFileManager tsFileManager, List selectedSequenceFiles, List selectedUnsequenceFiles, - AbstractCompactionPerformer performer, + ICrossCompactionPerformer performer, AtomicInteger currentTaskNum) { super( logicalStorageGroup + "-" + virutalStorageGroupName, @@ -118,8 +118,7 @@ protected void doCompaction() throws Exception { // restart recovery compactionLogger.close(); - performer.setSeqFiles(selectedSequenceFiles); - performer.setUnseqFiles(selectedUnsequenceFiles); + performer.setSourceFiles(selectedSequenceFiles, selectedUnsequenceFiles); performer.setTargetFiles(targetTsfileResourceList); performer.perform(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java new file mode 100644 index 0000000000000..ab56c49f7b52b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.cross; + +import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; + +import java.util.List; + +public interface ICrossSpaceSelector extends ICompactionSelector { + @Override + List select(); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index ba78e14f6b642..0d5b7263f3f79 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -23,10 +23,10 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.cross.AbstractCrossSpaceCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -41,10 +41,17 @@ import java.util.Iterator; import java.util.List; -public class RewriteCrossSpaceCompactionSelector extends AbstractCrossSpaceCompactionSelector { +public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + protected String logicalStorageGroupName; + protected String virtualGroupId; + protected String storageGroupDir; + protected long timePartition; + protected TsFileManager tsFileManager; + protected List sequenceFileList; + protected List unsequenceFileList; public RewriteCrossSpaceCompactionSelector( String logicalStorageGroupName, @@ -52,12 +59,11 @@ public RewriteCrossSpaceCompactionSelector( String storageGroupDir, long timePartition, TsFileManager tsFileManager) { - super( - logicalStorageGroupName, - virtualStorageGroupId, - storageGroupDir, - timePartition, - tsFileManager); + this.storageGroupDir = storageGroupDir; + this.logicalStorageGroupName = logicalStorageGroupName; + this.virtualGroupId = virtualStorageGroupId; + this.timePartition = timePartition; + this.tsFileManager = tsFileManager; } /** diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java similarity index 92% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java index 88933dec56f2e..d8b98f9853304 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerSequenceSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java @@ -23,6 +23,6 @@ import java.util.List; -public interface AbstractInnerSequenceSpaceCompactionSelector extends ICompactionSelector { +public interface IInnerSeqSpaceSelector extends ICompactionSelector { List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java similarity index 91% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java index b82d75800ef32..adfd2d7d0f864 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/AbstractInnerUnsequenceSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java @@ -23,6 +23,6 @@ import java.util.List; -public interface AbstractInnerUnsequenceSpaceCompactionSelector extends ICompactionSelector { +public interface IInnerUnseqSpaceSelector extends ICompactionSelector { List select(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java index a9878b8c6a884..c2152cc85e050 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ICompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -65,7 +65,7 @@ public InnerSpaceCompactionTask( TsFileManager tsFileManager, List selectedTsFileResourceList, boolean sequence, - AbstractCompactionPerformer performer, + ICompactionPerformer performer, AtomicInteger currentTaskNum) { super( logicalStorageGroupName + "-" + virtualStorageGroupName, @@ -123,11 +123,7 @@ protected void doCompaction() throws Exception { "{} [Compaction] compaction with {}", fullStorageGroupName, selectedTsFileResourceList); // carry out the compaction - if (sequence) { - performer.setSeqFiles(selectedTsFileResourceList); - } else { - performer.setUnseqFiles(selectedTsFileResourceList); - } + performer.setSourceFiles(selectedTsFileResourceList); performer.setTargetFiles(Collections.singletonList(targetTsFileResource)); performer.perform(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 5bfe07fd35ae0..6ecd97188cb5c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -22,11 +22,11 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSequenceSpaceCompactionSelector; -import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerUnsequenceSpaceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector; +import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; @@ -56,8 +56,7 @@ * found in some level, selector will not search higher level anymore. */ public class SizeTieredCompactionSelector - implements AbstractInnerSequenceSpaceCompactionSelector, - AbstractInnerUnsequenceSpaceCompactionSelector { + implements IInnerSeqSpaceSelector, IInnerUnseqSpaceSelector { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICompactionPerformer.java similarity index 75% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICompactionPerformer.java index 1c2672625b692..93baeb0d55e5e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/AbstractCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICompactionPerformer.java @@ -23,7 +23,6 @@ import org.apache.iotdb.db.exception.metadata.MetadataException; import java.io.IOException; -import java.util.Collections; import java.util.List; /** @@ -32,20 +31,19 @@ * from tsfile, and some may using query tools to read data point by point from tsfile. Notice, not * all kinds of Performer can be used for all kinds of compaction tasks! */ -public abstract class AbstractCompactionPerformer { - protected List seqFiles = Collections.emptyList(); - protected List unseqFiles = Collections.emptyList(); +public interface ICompactionPerformer { - public abstract void perform() + void perform() throws IOException, MetadataException, StorageEngineException, InterruptedException; - public abstract void setTargetFiles(List targetFiles); + void setTargetFiles(List targetFiles); - public void setSeqFiles(List seqFiles) { - this.seqFiles = seqFiles; + default void setSourceFiles(List files) { + throw new RuntimeException("Cannot set single type of source files to this kind of performer"); } - public void setUnseqFiles(List unseqFiles) { - this.unseqFiles = unseqFiles; + default void setSourceFiles(List seqFiles, List unseqFiles) { + throw new RuntimeException( + "Cannot set both seq files and unseq files to this kind of performer"); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICrossCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICrossCompactionPerformer.java new file mode 100644 index 0000000000000..d96e576c267f3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ICrossCompactionPerformer.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.performer; + +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + +import java.util.List; + +public interface ICrossCompactionPerformer extends ICompactionPerformer { + void setSourceFiles(List seqFiles, List unseqFiles); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ISeqCompactionPerformer.java similarity index 75% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ISeqCompactionPerformer.java index c280964e1833e..8835bb544fca0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/MergeFileStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ISeqCompactionPerformer.java @@ -16,12 +16,12 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.iotdb.db.engine.compaction.performer; -package org.apache.iotdb.db.engine.compaction.constant; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; -public enum MergeFileStrategy { - MAX_SERIES_NUM, - MAX_FILE_NUM, - // TODO: HOW? - TRADE_OFF, +import java.util.List; + +public interface ISeqCompactionPerformer extends ICompactionPerformer { + void setSourceFiles(List seqFiles); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/IUnseqCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/IUnseqCompactionPerformer.java new file mode 100644 index 0000000000000..3c48d41df0711 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/IUnseqCompactionPerformer.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.performer; + +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + +import java.util.List; + +public interface IUnseqCompactionPerformer extends ICompactionPerformer { + void setSourceFiles(List unseqFiles); +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java similarity index 94% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java index 9ca2a16bcc680..28cfa869636aa 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadChunkCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java @@ -16,13 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.performer; +package org.apache.iotdb.db.engine.compaction.performer.impl; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.inner.utils.AlignedSeriesCompactionExecutor; import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; import org.apache.iotdb.db.engine.compaction.inner.utils.SingleSeriesCompactionExecutor; +import org.apache.iotdb.db.engine.compaction.performer.ISeqCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -44,10 +45,11 @@ import java.util.LinkedList; import java.util.List; -public class ReadChunkCompactionPerformer extends AbstractCompactionPerformer { +public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); private TsFileResource targetResource; + private List seqFiles; public ReadChunkCompactionPerformer(List sourceFiles, TsFileResource targetFile) { this.seqFiles = sourceFiles; @@ -98,11 +100,6 @@ public void setTargetFiles(List targetFiles) { this.targetResource = targetFiles.get(0); } - @Override - public void setUnseqFiles(List unseqFiles) { - throw new RuntimeException("The performer of this class cannot compact with unseq files!"); - } - private void compactAlignedSeries( String device, TsFileResource targetResource, @@ -161,4 +158,9 @@ private void compactNotAlignedSeries( compactionExecutorOfCurrentTimeSeries.execute(); } } + + @Override + public void setSourceFiles(List seqFiles) { + this.seqFiles = seqFiles; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java similarity index 94% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java index 1de635c68ac92..2ea8c9db1c81b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/ReadPointCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.iotdb.db.engine.compaction.performer; +package org.apache.iotdb.db.engine.compaction.performer.impl; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.inner.utils.MultiTsFileDeviceIterator; +import org.apache.iotdb.db.engine.compaction.performer.ICrossCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.IUnseqCompactionPerformer; import org.apache.iotdb.db.engine.compaction.writer.AbstractCompactionWriter; import org.apache.iotdb.db.engine.compaction.writer.CrossSpaceCompactionWriter; import org.apache.iotdb.db.engine.compaction.writer.InnerSpaceCompactionWriter; @@ -55,8 +57,11 @@ import java.util.Set; import java.util.stream.Collectors; -public class ReadPointCompactionPerformer extends AbstractCompactionPerformer { +public class ReadPointCompactionPerformer + implements ICrossCompactionPerformer, IUnseqCompactionPerformer { private Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); + private List seqFiles; + private List unseqFiles; private List targetFiles = Collections.emptyList(); @@ -307,4 +312,15 @@ private void checkThreadInterrupted() throws InterruptedException { "[Compaction] compaction for target file %s abort", targetFiles.toString())); } } + + @Override + public void setSourceFiles(List seqFiles, List unseqFiles) { + this.seqFiles = seqFiles; + this.unseqFiles = unseqFiles; + } + + @Override + public void setSourceFiles(List unseqFiles) { + this.unseqFiles = unseqFiles; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java index e8aeb7f6d3f15..2cc19c6e548d9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/AbstractCompactionTask.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; -import org.apache.iotdb.db.engine.compaction.performer.AbstractCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.ICompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.slf4j.Logger; @@ -46,7 +46,7 @@ public abstract class AbstractCompactionTask implements Callable { protected long timeCost = 0L; protected volatile boolean ran = false; protected volatile boolean finished = false; - protected AbstractCompactionPerformer performer; + protected ICompactionPerformer performer; public AbstractCompactionTask( String fullStorageGroupName, @@ -74,7 +74,7 @@ public Void call() throws Exception { LOGGER.error(e.getMessage(), e); } finally { this.currentTaskNum.decrementAndGet(); - CompactionTaskManager.getInstance().removeRunningTaskFromList(this); + CompactionTaskManager.getInstance().removeRunningTaskFuture(this); timeCost = System.currentTimeMillis() - startTime; finished = true; } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 4629b45628e16..67c2c8f0450bd 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -23,8 +23,8 @@ import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java index 46f598c6982b0..c5866109481af 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java @@ -22,8 +22,8 @@ import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionTest; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java index fb163b33c03e4..a6711346b0343 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java index cb79feb73fcdb..e77babf24b6d8 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.ModificationFile; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java index b5f4a7f152e18..240e497af3fbb 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.ModificationFile; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java index 1a65e09a7d30b..6688dc3479cda 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java index c1a79b8d12199..a327b5e33e3ba 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSeqCompactionTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java index 207d9483b4d77..1b4ad64e6cc11 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java index de9d62233b0cd..5b36608ece5ca 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerUnseqCompactionTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.performer.ReadPointCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionClearUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java index c1e85647fe3e9..819a816942aff 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; import org.apache.iotdb.db.engine.compaction.TestUtilsForAlignedSeries; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java index 71523083b6e5d..ebc9522ed0760 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerNoAlignedTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.cache.ChunkCache; import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java index 40a3b8000e37a..74699ec670ef6 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerOldTest.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.MetadataException; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java index 12509a2e31bd8..ae6104830eae1 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.inner.AbstractInnerSpaceCompactionTest; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java index 76b37dd3be384..165958f1d3625 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java @@ -22,7 +22,7 @@ import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java index 8f156a3ed81c7..a81daa26bd54b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.db.constant.TestConstant; import org.apache.iotdb.db.engine.compaction.CompactionUtils; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.CompactionRecoverTask; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java index 74999fb2f74e5..6191ab05d5aea 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; -import org.apache.iotdb.db.engine.compaction.performer.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.engine.flush.FlushManager; import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy; From 21ecc309d53d47b8a1ae505e6da13483bcec8805 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Fri, 8 Apr 2022 21:58:39 +0800 Subject: [PATCH 18/22] fix ci --- .../cross/rewrite/RewriteCrossSpaceCompactionSelector.java | 2 ++ .../performer/impl/ReadPointCompactionPerformer.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 0d5b7263f3f79..8aa346220c846 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -64,6 +64,8 @@ public RewriteCrossSpaceCompactionSelector( this.virtualGroupId = virtualStorageGroupId; this.timePartition = timePartition; this.tsFileManager = tsFileManager; + this.sequenceFileList = tsFileManager.getSequenceListByTimePartition(timePartition); + this.unsequenceFileList = tsFileManager.getUnsequenceListByTimePartition(timePartition); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java index 2ea8c9db1c81b..30fb72143f6b2 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadPointCompactionPerformer.java @@ -60,8 +60,8 @@ public class ReadPointCompactionPerformer implements ICrossCompactionPerformer, IUnseqCompactionPerformer { private Logger LOGGER = LoggerFactory.getLogger(IoTDBConstant.COMPACTION_LOGGER_NAME); - private List seqFiles; - private List unseqFiles; + private List seqFiles = Collections.emptyList(); + private List unseqFiles = Collections.emptyList(); private List targetFiles = Collections.emptyList(); From 23f3b79fc559c51a00bd1022807775d812a80c1d Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Mon, 11 Apr 2022 20:57:41 +0800 Subject: [PATCH 19/22] refactor ICompactionSelector interface --- .../compaction/CompactionScheduler.java | 29 ++++++++++--------- .../constant/CrossCompactionStrategy.java | 3 +- .../compaction/cross/ICrossSpaceSelector.java | 5 ++-- .../RewriteCrossSpaceCompactionSelector.java | 10 ++----- .../inner/IInnerSeqSpaceSelector.java | 3 +- .../inner/IInnerUnseqSpaceSelector.java | 3 +- .../SizeTieredCompactionSelector.java | 7 +---- .../compaction/task/ICompactionSelector.java | 13 ++++++++- 8 files changed, 38 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index 34a4950487a99..553e04a689cb7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -23,8 +23,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; -import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector; -import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector; import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; @@ -41,7 +39,8 @@ * For different types of compaction task(e.g. InnerSpaceCompaction), CompactionScheduler will call * the corresponding {@link ICompactionSelector selector} according to the compaction machanism of * the task(e.g. LevelCompaction, SizeTiredCompaction), and the selection and submission process is - * carried out in the {@link ICompactionSelector#select() selectAndSubmit()} in selector. + * carried out in the {@link ICompactionSelector#selectInnerSpaceTask(List)} () and {@link + * ICompactionSelector#selectCrossSpaceTask(List, List)}} in selector. */ public class CompactionScheduler { private static final Logger LOGGER = @@ -88,22 +87,25 @@ public static void tryToSubmitInnerSpaceCompactionTask( return; } - List taskList = null; + ICompactionSelector innerSpaceCompactionSelector = null; if (sequence) { - IInnerSeqSpaceSelector innerSpaceCompactionSelector = + innerSpaceCompactionSelector = config .getInnerSequenceCompactionStrategy() .getCompactionSelector( logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); - taskList = innerSpaceCompactionSelector.select(); } else { - IInnerUnseqSpaceSelector innerSpaceCompactionSelector = + innerSpaceCompactionSelector = config .getInnerUnsequenceCompactionStrategy() .getCompactionSelector( logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); - taskList = innerSpaceCompactionSelector.select(); } + List taskList = + innerSpaceCompactionSelector.selectInnerSpaceTask( + sequence + ? tsFileManager.getSequenceListByTimePartition(timePartition) + : tsFileManager.getUnsequenceListByTimePartition(timePartition)); for (AbstractCompactionTask task : taskList) { CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); } @@ -123,12 +125,11 @@ private static void tryToSubmitCrossSpaceCompactionTask( config .getCrossCompactionStrategy() .getCompactionSelector( - logicalStorageGroupName, - dataRegionId, - storageGroupDir, - timePartition, - tsFileManager); - List taskList = crossSpaceCompactionSelector.select(); + logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); + List taskList = + crossSpaceCompactionSelector.selectCrossSpaceTask( + tsFileManager.getSequenceListByTimePartition(timePartition), + tsFileManager.getUnsequenceListByTimePartition(timePartition)); for (AbstractCompactionTask task : taskList) { CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java index f64b10034ed83..337b549483362 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java @@ -63,14 +63,13 @@ public CrossSpaceCompactionTask getCompactionTask( public ICrossSpaceSelector getCompactionSelector( String logicalStorageGroupName, String virtualGroupId, - String storageGroupDir, long timePartition, TsFileManager tsFileManager) { switch (this) { case REWRITE_COMPACTION: default: return new RewriteCrossSpaceCompactionSelector( - logicalStorageGroupName, virtualGroupId, storageGroupDir, timePartition, tsFileManager); + logicalStorageGroupName, virtualGroupId, timePartition, tsFileManager); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java index ab56c49f7b52b..7f1a9e982bd82 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java @@ -20,10 +20,11 @@ import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; public interface ICrossSpaceSelector extends ICompactionSelector { - @Override - List select(); + List selectCrossSpaceTask( + List seqFiles, List unseqFiles); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 4838e5ad06188..3b1baa8878151 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -47,25 +47,18 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); protected String logicalStorageGroupName; protected String dataRegionId; - protected String storageGroupDir; protected long timePartition; protected TsFileManager tsFileManager; - protected List sequenceFileList; - protected List unsequenceFileList; public RewriteCrossSpaceCompactionSelector( String logicalStorageGroupName, String dataRegionId, - String storageGroupDir, long timePartition, TsFileManager tsFileManager) { - this.storageGroupDir = storageGroupDir; this.logicalStorageGroupName = logicalStorageGroupName; this.dataRegionId = dataRegionId; this.timePartition = timePartition; this.tsFileManager = tsFileManager; - this.sequenceFileList = tsFileManager.getSequenceListByTimePartition(timePartition); - this.unsequenceFileList = tsFileManager.getUnsequenceListByTimePartition(timePartition); } /** @@ -77,7 +70,8 @@ public RewriteCrossSpaceCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public List select() { + public List selectCrossSpaceTask( + List sequenceFileList, List unsequenceFileList) { if ((CompactionTaskManager.currentTaskNum.get() >= config.getConcurrentCompactionThread()) || (!config.isEnableCrossSpaceCompaction())) { return Collections.emptyList(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java index d8b98f9853304..7163e467def8d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java @@ -20,9 +20,10 @@ import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; public interface IInnerSeqSpaceSelector extends ICompactionSelector { - List select(); + List selectInnerSpaceTask(List resources); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java index adfd2d7d0f864..96338cc87f471 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java @@ -20,9 +20,10 @@ import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; public interface IInnerUnseqSpaceSelector extends ICompactionSelector { - List select(); + List selectInnerSpaceTask(List resources); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 37fd9422334a6..3d7cbdf902f43 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -78,11 +78,6 @@ public SizeTieredCompactionSelector( this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.sequence = sequence; - if (sequence) { - this.tsFileResources = tsFileManager.getSequenceListByTimePartition(timePartition); - } else { - this.tsFileResources = tsFileManager.getUnsequenceListByTimePartition(timePartition); - } } /** @@ -94,7 +89,7 @@ public SizeTieredCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public List select() { + public List selectInnerSpaceTask(List tsFileResources) { PriorityQueue, Long>> taskPriorityQueue = new PriorityQueue<>(new SizeTieredCompactionTaskComparator()); try { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java index ed75918dae5f8..a3af4209915c9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.engine.compaction.task; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; + import java.util.List; /** @@ -27,5 +29,14 @@ * increase the global compaction task count. */ public interface ICompactionSelector { - List select(); + default List selectInnerSpaceTask(List resources) { + throw new RuntimeException("This kind of selector cannot be used to select inner space task"); + } + + default List selectCrossSpaceTask( + List seqFiles, List unseqFiles) { + { + throw new RuntimeException("This kind of selector cannot be used to select cross space task"); + } + } } From 1127c1bd5ec804f7e4300a09f2b9ab648b9057bb Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Mon, 11 Apr 2022 22:56:34 +0800 Subject: [PATCH 20/22] Separate compaction selector and performer in the configuration --- .../resources/conf/iotdb-engine.properties | 30 +++++-- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 86 +++++++++++++------ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 44 +++++++--- .../compaction/CompactionScheduler.java | 53 +++++++++--- .../db/engine/compaction/CompactionUtils.java | 8 +- .../constant/CrossCompactionPerformer.java | 41 +++++++++ ...tegy.java => CrossCompactionSelector.java} | 16 ++-- .../constant/InnerSeqCompactionPerformer.java | 41 +++++++++ ...a => InnerSequenceCompactionSelector.java} | 10 +-- .../InnerUnseqCompactionPerformer.java | 41 +++++++++ ...=> InnerUnsequenceCompactionSelector.java} | 12 +-- .../cross/CrossSpaceCompactionTask.java | 8 +- .../compaction/cross/ICrossSpaceSelector.java | 4 +- .../RewriteCrossSpaceCompactionSelector.java | 17 +--- .../inner/IInnerSeqSpaceSelector.java | 3 +- .../inner/IInnerUnseqSpaceSelector.java | 3 +- .../inner/InnerSpaceCompactionTask.java | 7 +- .../SizeTieredCompactionSelector.java | 19 +--- .../compaction/task/ICompactionSelector.java | 8 +- .../CompactionTaskComparatorTest.java | 4 - .../compaction/CompactionTaskManagerTest.java | 18 ---- .../cross/CrossSpaceCompactionTest.java | 12 +-- .../RewriteCrossSpaceCompactionTest.java | 8 -- .../utils/CompactionConfigRestorer.java | 14 +-- .../StorageGroupProcessorTest.java | 2 - 25 files changed, 334 insertions(+), 175 deletions(-) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionPerformer.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/{CrossCompactionStrategy.java => CrossCompactionSelector.java} (87%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSeqCompactionPerformer.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/{InnerSequenceCompactionStrategy.java => InnerSequenceCompactionSelector.java} (88%) create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnseqCompactionPerformer.java rename server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/{InnerUnsequenceCompactionStrategy.java => InnerUnsequenceCompactionSelector.java} (85%) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index b442aa3e5eca3..ec7652668e6a7 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -453,17 +453,29 @@ timestamp_precision=ms # Datatype: boolean # enable_cross_space_compaction=true -# the strategy of cross space compaction task -# Options: rewrite_compaction -# cross_compaction_strategy=rewrite_compaction +# the selector of cross space compaction task +# Options: rewrite +# cross_selector=rewrite -# the strategy of inner sequence space compaction task -# Options: size_tiered_compaction -# inner_seq_compaction_strategy=size_tiered_compaction +# the compaction performer of cross space compaction task +# Options: read_point +# cross_performer=read_point -# the strategy of inner unsequence space compaction task -# Options: size_tiered_compaction -# inner_unseq_compaction_strategy=size_tiered_compaction +# the selector of inner sequence space compaction task +# Options: size_tiered +# inner_seq_selector=size_tiered + +# the performer of inner sequence space compaction task +# Options: read_chunk +# inner_seq_performer=read_chunk + +# the selector of inner unsequence space compaction task +# Options: size_tiered +# inner_unseq_selector=size_tiered + +# the performer of inner unsequence space compaction task +# Options: read_point +# inner_seq_performer=read_point # The priority of compaction execution # INNER_CROSS: prioritize inner space compaction, reduce the number of files first diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index cfcdd4d421ac1..edee34186307e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -21,9 +21,12 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionSelector; +import org.apache.iotdb.db.engine.compaction.constant.InnerSeqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnseqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.timeindex.TimeIndexLevel; import org.apache.iotdb.db.exception.LoadConfigurationException; import org.apache.iotdb.db.metadata.LocalSchemaProcessor; @@ -363,10 +366,10 @@ public class IoTDBConfig { /** When average series point number reaches this, flush the memtable to disk */ private int avgSeriesPointNumberThreshold = 10000; - /** Enable inner space copaction for sequence files */ + /** Enable inner space compaction for sequence files */ private boolean enableSeqSpaceCompaction = true; - /** Enable inner space copaction for unsequence files */ + /** Enable inner space compaction for unsequence files */ private boolean enableUnseqSpaceCompaction = true; /** Compact the unsequence files into the overlapped sequence files */ @@ -376,18 +379,25 @@ public class IoTDBConfig { * The strategy of inner space compaction task. There are just one inner space compaction strategy * SIZE_TIRED_COMPACTION: */ - private InnerSequenceCompactionStrategy innerSequenceCompactionStrategy = - InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerSequenceCompactionSelector innerSequenceCompactionSelector = + InnerSequenceCompactionSelector.SIZE_TIERED; - private InnerUnsequenceCompactionStrategy innerUnsequenceCompactionStrategy = - InnerUnsequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + private InnerSeqCompactionPerformer innerSeqCompactionPerformer = + InnerSeqCompactionPerformer.READ_CHUNK; + + private InnerUnsequenceCompactionSelector innerUnsequenceCompactionSelector = + InnerUnsequenceCompactionSelector.SIZE_TIERED; + + private InnerUnseqCompactionPerformer innerUnseqCompactionPerformer = + InnerUnseqCompactionPerformer.READ_POINT; /** * The strategy of cross space compaction task. There are just one cross space compaction strategy * SIZE_TIRED_COMPACTION: */ - private CrossCompactionStrategy crossCompactionStrategy = - CrossCompactionStrategy.REWRITE_COMPACTION; + private CrossCompactionSelector crossCompactionSelector = CrossCompactionSelector.REWRITE; + + private CrossCompactionPerformer crossCompactionPerformer = CrossCompactionPerformer.READ_POINT; /** * The priority of compaction task execution. There are three priority strategy INNER_CROSS: @@ -2459,30 +2469,56 @@ public void setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompaction) { this.enableCrossSpaceCompaction = enableCrossSpaceCompaction; } - public InnerSequenceCompactionStrategy getInnerSequenceCompactionStrategy() { - return innerSequenceCompactionStrategy; + public InnerSequenceCompactionSelector getInnerSequenceCompactionSelector() { + return innerSequenceCompactionSelector; + } + + public void setInnerSequenceCompactionSelector( + InnerSequenceCompactionSelector innerSequenceCompactionSelector) { + this.innerSequenceCompactionSelector = innerSequenceCompactionSelector; + } + + public InnerUnsequenceCompactionSelector getInnerUnsequenceCompactionSelector() { + return innerUnsequenceCompactionSelector; + } + + public void setInnerUnsequenceCompactionSelector( + InnerUnsequenceCompactionSelector innerUnsequenceCompactionSelector) { + this.innerUnsequenceCompactionSelector = innerUnsequenceCompactionSelector; + } + + public InnerSeqCompactionPerformer getInnerSeqCompactionPerformer() { + return innerSeqCompactionPerformer; + } + + public void setInnerSeqCompactionPerformer( + InnerSeqCompactionPerformer innerSeqCompactionPerformer) { + this.innerSeqCompactionPerformer = innerSeqCompactionPerformer; + } + + public InnerUnseqCompactionPerformer getInnerUnseqCompactionPerformer() { + return innerUnseqCompactionPerformer; } - public void setInnerSequenceCompactionStrategy( - InnerSequenceCompactionStrategy innerSequenceCompactionStrategy) { - this.innerSequenceCompactionStrategy = innerSequenceCompactionStrategy; + public void setInnerUnseqCompactionPerformer( + InnerUnseqCompactionPerformer innerUnseqCompactionPerformer) { + this.innerUnseqCompactionPerformer = innerUnseqCompactionPerformer; } - public InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStrategy() { - return innerUnsequenceCompactionStrategy; + public CrossCompactionSelector getCrossCompactionSelector() { + return crossCompactionSelector; } - public void setInnerUnsequenceCompactionStrategy( - InnerUnsequenceCompactionStrategy innerUnsequenceCompactionStrategy) { - this.innerUnsequenceCompactionStrategy = innerUnsequenceCompactionStrategy; + public void setCrossCompactionSelector(CrossCompactionSelector crossCompactionSelector) { + this.crossCompactionSelector = crossCompactionSelector; } - public CrossCompactionStrategy getCrossCompactionStrategy() { - return crossCompactionStrategy; + public CrossCompactionPerformer getCrossCompactionPerformer() { + return crossCompactionPerformer; } - public void setCrossCompactionStrategy(CrossCompactionStrategy crossCompactionStrategy) { - this.crossCompactionStrategy = crossCompactionStrategy; + public void setCrossCompactionPerformer(CrossCompactionPerformer crossCompactionPerformer) { + this.crossCompactionPerformer = crossCompactionPerformer; } public CompactionPriority getCompactionPriority() { diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index de6906e03d59e..e48f56989c1e3 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -22,9 +22,12 @@ import org.apache.iotdb.db.conf.directories.DirectoryManager; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionSelector; +import org.apache.iotdb.db.engine.compaction.constant.InnerSeqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionSelector; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnseqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.constant.InnerUnsequenceCompactionSelector; import org.apache.iotdb.db.exception.BadNodeUrlFormatException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.qp.utils.DatetimeUtils; @@ -360,22 +363,35 @@ private void loadProps() { "enable_unseq_space_compaction", Boolean.toString(conf.isEnableUnseqSpaceCompaction())))); - conf.setCrossCompactionStrategy( - CrossCompactionStrategy.getCrossCompactionStrategy( + conf.setCrossCompactionSelector( + CrossCompactionSelector.getCrossCompactionStrategy( properties.getProperty( - "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString()))); + "cross_selector", conf.getCrossCompactionSelector().toString()))); - conf.setInnerSequenceCompactionStrategy( - InnerSequenceCompactionStrategy.getInnerSequenceCompactionStrategy( + conf.setInnerSequenceCompactionSelector( + InnerSequenceCompactionSelector.getInnerSequenceCompactionStrategy( properties.getProperty( - "inner_seq_compaction_strategy", - conf.getInnerSequenceCompactionStrategy().toString()))); + "inner_seq_selector", conf.getInnerSequenceCompactionSelector().toString()))); - conf.setInnerUnsequenceCompactionStrategy( - InnerUnsequenceCompactionStrategy.getInnerUnsequenceCompactionStrategy( + conf.setInnerUnsequenceCompactionSelector( + InnerUnsequenceCompactionSelector.getInnerUnsequenceCompactionStrategy( properties.getProperty( - "inner_unseq_compaction_strategy", - conf.getInnerUnsequenceCompactionStrategy().toString()))); + "inner_unseq_selector", conf.getInnerUnsequenceCompactionSelector().toString()))); + + conf.setInnerSeqCompactionPerformer( + InnerSeqCompactionPerformer.getInnerSeqCompactionPerformer( + properties.getProperty( + "inner_seq_performer", conf.getInnerUnseqCompactionPerformer().toString()))); + + conf.setInnerUnseqCompactionPerformer( + InnerUnseqCompactionPerformer.getInnerUnseqCompactionPerformer( + properties.getProperty( + "inner_unseq_performer", conf.getInnerUnseqCompactionPerformer().toString()))); + + conf.setCrossCompactionPerformer( + CrossCompactionPerformer.getCrossCompactionPerformer( + properties.getProperty( + "cross_performer", conf.getCrossCompactionPerformer().toString()))); conf.setCompactionPriority( CompactionPriority.valueOf( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java index 553e04a689cb7..02eca44554cfa 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java @@ -22,10 +22,14 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; +import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; +import org.apache.iotdb.db.engine.compaction.performer.ICompactionPerformer; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; +import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,23 +95,41 @@ public static void tryToSubmitInnerSpaceCompactionTask( if (sequence) { innerSpaceCompactionSelector = config - .getInnerSequenceCompactionStrategy() + .getInnerSequenceCompactionSelector() .getCompactionSelector( logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); } else { innerSpaceCompactionSelector = config - .getInnerUnsequenceCompactionStrategy() + .getInnerUnsequenceCompactionSelector() .getCompactionSelector( logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); } - List taskList = + List> taskList = innerSpaceCompactionSelector.selectInnerSpaceTask( sequence ? tsFileManager.getSequenceListByTimePartition(timePartition) : tsFileManager.getUnsequenceListByTimePartition(timePartition)); - for (AbstractCompactionTask task : taskList) { - CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); + for (List task : taskList) { + ICompactionPerformer performer = + sequence + ? IoTDBDescriptor.getInstance() + .getConfig() + .getInnerSeqCompactionPerformer() + .getCompactionPerformer() + : IoTDBDescriptor.getInstance() + .getConfig() + .getInnerUnseqCompactionPerformer() + .getCompactionPerformer(); + CompactionTaskManager.getInstance() + .addTaskToWaitingQueue( + new InnerSpaceCompactionTask( + timePartition, + tsFileManager, + task, + sequence, + performer, + CompactionTaskManager.currentTaskNum)); } } @@ -123,15 +145,26 @@ private static void tryToSubmitCrossSpaceCompactionTask( } ICrossSpaceSelector crossSpaceCompactionSelector = config - .getCrossCompactionStrategy() + .getCrossCompactionSelector() .getCompactionSelector( logicalStorageGroupName, dataRegionId, timePartition, tsFileManager); - List taskList = + List, List>> taskList = crossSpaceCompactionSelector.selectCrossSpaceTask( tsFileManager.getSequenceListByTimePartition(timePartition), tsFileManager.getUnsequenceListByTimePartition(timePartition)); - for (AbstractCompactionTask task : taskList) { - CompactionTaskManager.getInstance().addTaskToWaitingQueue(task); + for (Pair, List> selectedFilesPair : taskList) { + CompactionTaskManager.getInstance() + .addTaskToWaitingQueue( + new CrossSpaceCompactionTask( + timePartition, + tsFileManager, + selectedFilesPair.left, + selectedFilesPair.right, + IoTDBDescriptor.getInstance() + .getConfig() + .getCrossCompactionPerformer() + .getCompactionPerformer(), + CompactionTaskManager.currentTaskNum)); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java index f34b5100378d4..ac3d9bcfaec98 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.RewriteCrossSpaceCompactionResource; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.RewriteCompactionFileSelector; @@ -231,10 +231,10 @@ public static void deleteModificationForSourceFile( public static ICrossSpaceMergeFileSelector getCrossSpaceFileSelector( long budget, RewriteCrossSpaceCompactionResource resource) { - CrossCompactionStrategy strategy = - IoTDBDescriptor.getInstance().getConfig().getCrossCompactionStrategy(); + CrossCompactionSelector strategy = + IoTDBDescriptor.getInstance().getConfig().getCrossCompactionSelector(); switch (strategy) { - case REWRITE_COMPACTION: + case REWRITE: return new RewriteCompactionFileSelector(resource, budget); default: throw new UnsupportedOperationException("Unknown CrossSpaceFileStrategy " + strategy); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionPerformer.java new file mode 100644 index 0000000000000..9dd13cc995a8b --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionPerformer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.constant; + +import org.apache.iotdb.db.engine.compaction.performer.ICrossCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; + +public enum CrossCompactionPerformer { + READ_POINT; + + public static CrossCompactionPerformer getCrossCompactionPerformer(String name) { + if (READ_POINT.toString().equalsIgnoreCase(name)) { + return READ_POINT; + } + throw new RuntimeException("Illegal compaction performer for cross compaction " + name); + } + + public ICrossCompactionPerformer getCompactionPerformer() { + switch (this) { + case READ_POINT: + default: + return new ReadPointCompactionPerformer(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionSelector.java similarity index 87% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionSelector.java index 337b549483362..caf0810530ff7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/CrossCompactionSelector.java @@ -28,29 +28,25 @@ import java.util.List; -public enum CrossCompactionStrategy { - REWRITE_COMPACTION; +public enum CrossCompactionSelector { + REWRITE; - public static CrossCompactionStrategy getCrossCompactionStrategy(String name) { + public static CrossCompactionSelector getCrossCompactionStrategy(String name) { if ("REWRITE_COMPACTION".equalsIgnoreCase(name)) { - return REWRITE_COMPACTION; + return REWRITE; } throw new RuntimeException("Illegal Cross Compaction Strategy " + name); } public CrossSpaceCompactionTask getCompactionTask( - String logicalStorageGroupName, - String virtualStorageGroupName, long timePartitionId, TsFileManager tsFileManager, List selectedSeqTsFileResourceList, List selectedUnSeqTsFileResourceList) { switch (this) { - case REWRITE_COMPACTION: + case REWRITE: default: return new CrossSpaceCompactionTask( - logicalStorageGroupName, - virtualStorageGroupName, timePartitionId, tsFileManager, selectedSeqTsFileResourceList, @@ -66,7 +62,7 @@ public ICrossSpaceSelector getCompactionSelector( long timePartition, TsFileManager tsFileManager) { switch (this) { - case REWRITE_COMPACTION: + case REWRITE: default: return new RewriteCrossSpaceCompactionSelector( logicalStorageGroupName, virtualGroupId, timePartition, tsFileManager); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSeqCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSeqCompactionPerformer.java new file mode 100644 index 0000000000000..57a731978293f --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSeqCompactionPerformer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.constant; + +import org.apache.iotdb.db.engine.compaction.performer.ISeqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; + +public enum InnerSeqCompactionPerformer { + READ_CHUNK; + + public static InnerSeqCompactionPerformer getInnerSeqCompactionPerformer(String name) { + if (READ_CHUNK.toString().equalsIgnoreCase(name)) { + return READ_CHUNK; + } + throw new RuntimeException("Illegal compaction performer for seq inner compaction " + name); + } + + public ISeqCompactionPerformer getCompactionPerformer() { + switch (this) { + case READ_CHUNK: + default: + return new ReadChunkCompactionPerformer(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java similarity index 88% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java index e3824d3255776..f9fe30aaf6fc7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerSequenceCompactionSelector.java @@ -23,12 +23,12 @@ import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -public enum InnerSequenceCompactionStrategy { - SIZE_TIERED_COMPACTION; +public enum InnerSequenceCompactionSelector { + SIZE_TIERED; - public static InnerSequenceCompactionStrategy getInnerSequenceCompactionStrategy(String name) { + public static InnerSequenceCompactionSelector getInnerSequenceCompactionStrategy(String name) { if ("SIZE_TIERED_COMPACTION".equalsIgnoreCase(name)) { - return SIZE_TIERED_COMPACTION; + return SIZE_TIERED; } throw new RuntimeException("Illegal Compaction Strategy " + name); } @@ -39,7 +39,7 @@ public IInnerSeqSpaceSelector getCompactionSelector( long timePartition, TsFileManager tsFileManager) { switch (this) { - case SIZE_TIERED_COMPACTION: + case SIZE_TIERED: default: return new SizeTieredCompactionSelector( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, true); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnseqCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnseqCompactionPerformer.java new file mode 100644 index 0000000000000..75ef19ff9f92f --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnseqCompactionPerformer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.compaction.constant; + +import org.apache.iotdb.db.engine.compaction.performer.IUnseqCompactionPerformer; +import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; + +public enum InnerUnseqCompactionPerformer { + READ_POINT; + + public static InnerUnseqCompactionPerformer getInnerUnseqCompactionPerformer(String name) { + if (READ_POINT.toString().equalsIgnoreCase(name)) { + return READ_POINT; + } + throw new RuntimeException("Illegal compaction performer for unseq inner compaction " + name); + } + + public IUnseqCompactionPerformer getCompactionPerformer() { + switch (this) { + case READ_POINT: + default: + return new ReadPointCompactionPerformer(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java similarity index 85% rename from server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java rename to server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java index 1e9338a171ff8..f325a9b19a1b7 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionStrategy.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/constant/InnerUnsequenceCompactionSelector.java @@ -22,13 +22,13 @@ import org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; -public enum InnerUnsequenceCompactionStrategy { - SIZE_TIERED_COMPACTION; +public enum InnerUnsequenceCompactionSelector { + SIZE_TIERED; - public static InnerUnsequenceCompactionStrategy getInnerUnsequenceCompactionStrategy( + public static InnerUnsequenceCompactionSelector getInnerUnsequenceCompactionStrategy( String name) { - if (SIZE_TIERED_COMPACTION.toString().equalsIgnoreCase(name)) { - return SIZE_TIERED_COMPACTION; + if (SIZE_TIERED.toString().equalsIgnoreCase(name)) { + return SIZE_TIERED; } throw new RuntimeException("Illegal Compaction Strategy " + name); } @@ -39,7 +39,7 @@ public IInnerUnseqSpaceSelector getCompactionSelector( long timePartition, TsFileManager tsFileManager) { switch (this) { - case SIZE_TIERED_COMPACTION: + case SIZE_TIERED: default: return new SizeTieredCompactionSelector( logicalStorageGroupName, virtualStorageGroupName, timePartition, tsFileManager, false); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java index f8f76467ea0e7..c4e1866f3ee95 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java @@ -58,15 +58,17 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { protected List holdWriteLockList = new ArrayList<>(); public CrossSpaceCompactionTask( - String logicalStorageGroup, - String dataRegionId, long timePartition, TsFileManager tsFileManager, List selectedSequenceFiles, List selectedUnsequenceFiles, ICrossCompactionPerformer performer, AtomicInteger currentTaskNum) { - super(logicalStorageGroup + "-" + dataRegionId, timePartition, tsFileManager, currentTaskNum); + super( + tsFileManager.getStorageGroupName() + "-" + tsFileManager.getDataRegion(), + timePartition, + tsFileManager, + currentTaskNum); this.selectedSequenceFiles = selectedSequenceFiles; this.selectedUnsequenceFiles = selectedUnsequenceFiles; this.seqTsFileResourceList = tsFileManager.getSequenceListByTimePartition(timePartition); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java index 7f1a9e982bd82..061e8d2f26663 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/ICrossSpaceSelector.java @@ -18,13 +18,13 @@ */ package org.apache.iotdb.db.engine.compaction.cross; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.utils.Pair; import java.util.List; public interface ICrossSpaceSelector extends ICompactionSelector { - List selectCrossSpaceTask( + List, List>> selectCrossSpaceTask( List seqFiles, List unseqFiles); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java index 3b1baa8878151..3950afc75ce9f 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java @@ -23,14 +23,12 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.CompactionUtils; -import org.apache.iotdb.db.engine.compaction.cross.CrossSpaceCompactionTask; import org.apache.iotdb.db.engine.compaction.cross.ICrossSpaceSelector; import org.apache.iotdb.db.engine.compaction.cross.rewrite.selector.ICrossSpaceMergeFileSelector; -import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.exception.MergeException; +import org.apache.iotdb.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,7 +68,7 @@ public RewriteCrossSpaceCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public List selectCrossSpaceTask( + public List selectCrossSpaceTask( List sequenceFileList, List unsequenceFileList) { if ((CompactionTaskManager.currentTaskNum.get() >= config.getConcurrentCompactionThread()) || (!config.isEnableCrossSpaceCompaction())) { @@ -121,16 +119,7 @@ public List selectCrossSpaceTask( logicalStorageGroupName + "-" + dataRegionId, mergeResource.getSeqFiles().size(), mergeResource.getUnseqFiles().size()); - return Collections.singletonList( - new CrossSpaceCompactionTask( - logicalStorageGroupName, - dataRegionId, - timePartition, - tsFileManager, - mergeFiles[0], - mergeFiles[1], - new ReadPointCompactionPerformer(), - CompactionTaskManager.currentTaskNum)); + return Collections.singletonList(new Pair<>(mergeFiles[0], mergeFiles[1])); } } catch (MergeException | IOException e) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java index 7163e467def8d..7a6b326a8e4a2 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerSeqSpaceSelector.java @@ -18,12 +18,11 @@ */ package org.apache.iotdb.db.engine.compaction.inner; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; public interface IInnerSeqSpaceSelector extends ICompactionSelector { - List selectInnerSpaceTask(List resources); + List> selectInnerSpaceTask(List resources); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java index 96338cc87f471..c7f02f5f65469 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/IInnerUnseqSpaceSelector.java @@ -18,12 +18,11 @@ */ package org.apache.iotdb.db.engine.compaction.inner; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.compaction.task.ICompactionSelector; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import java.util.List; public interface IInnerUnseqSpaceSelector extends ICompactionSelector { - List selectInnerSpaceTask(List resources); + List> selectInnerSpaceTask(List resources); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java index eef5d947ea509..a42ea6d2a1dee 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java @@ -59,8 +59,6 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { protected boolean[] isHoldingWriteLock; public InnerSpaceCompactionTask( - String logicalStorageGroupName, - String dataRegionId, long timePartition, TsFileManager tsFileManager, List selectedTsFileResourceList, @@ -68,7 +66,10 @@ public InnerSpaceCompactionTask( ICompactionPerformer performer, AtomicInteger currentTaskNum) { super( - logicalStorageGroupName + "-" + dataRegionId, timePartition, tsFileManager, currentTaskNum); + tsFileManager.getStorageGroupName() + "-" + tsFileManager.getDataRegion(), + timePartition, + tsFileManager, + currentTaskNum); this.selectedTsFileResourceList = selectedTsFileResourceList; this.sequence = sequence; this.performer = performer; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index 3d7cbdf902f43..e0a996e73725b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -24,10 +24,6 @@ import org.apache.iotdb.db.engine.compaction.CompactionTaskManager; import org.apache.iotdb.db.engine.compaction.inner.IInnerSeqSpaceSelector; import org.apache.iotdb.db.engine.compaction.inner.IInnerUnseqSpaceSelector; -import org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionTask; -import org.apache.iotdb.db.engine.compaction.performer.impl.ReadChunkCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; -import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask; import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -89,7 +85,7 @@ public SizeTieredCompactionSelector( * @return Returns whether the file was found and submits the merge task */ @Override - public List selectInnerSpaceTask(List tsFileResources) { + public List> selectInnerSpaceTask(List tsFileResources) { PriorityQueue, Long>> taskPriorityQueue = new PriorityQueue<>(new SizeTieredCompactionTaskComparator()); try { @@ -99,19 +95,10 @@ public List selectInnerSpaceTask(List ts break; } } - List taskList = new LinkedList<>(); + List> taskList = new LinkedList<>(); while (taskPriorityQueue.size() > 0) { List resources = taskPriorityQueue.poll().left; - taskList.add( - new InnerSpaceCompactionTask( - logicalStorageGroupName, - dataRegionId, - timePartition, - tsFileManager, - resources, - sequence, - sequence ? new ReadChunkCompactionPerformer() : new ReadPointCompactionPerformer(), - CompactionTaskManager.currentTaskNum)); + taskList.add(resources); } return taskList; } catch (Exception e) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java index a3af4209915c9..a97fb8a9f7d00 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.compaction.task; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; +import org.apache.iotdb.tsfile.utils.Pair; import java.util.List; @@ -29,11 +30,14 @@ * increase the global compaction task count. */ public interface ICompactionSelector { - default List selectInnerSpaceTask(List resources) { + /* + * This method should be implemented by all SequenceSpaceInnerSelector and UnsequenceSpaceInnerSelector. It takes the list of tsfile in a time partition as input, and returns a list of list. Each list in the returned list is the source files of one compaction tasks. + */ + default List> selectInnerSpaceTask(List resources) { throw new RuntimeException("This kind of selector cannot be used to select inner space task"); } - default List selectCrossSpaceTask( + default List, List>> selectCrossSpaceTask( List seqFiles, List unseqFiles) { { throw new RuntimeException("This kind of selector cannot be used to select cross space task"); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java index 67c2c8f0450bd..0475026486ba7 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskComparatorTest.java @@ -276,8 +276,6 @@ public FakedInnerSpaceCompactionTask( boolean sequence, List selectedTsFileResourceList) { super( - storageGroupName, - "0", timePartition, tsFileManager, selectedTsFileResourceList, @@ -310,8 +308,6 @@ public FakeCrossSpaceCompactionTask( List selectedSequenceFiles, List selectedUnsequenceFiles) { super( - fullStorageGroupName, - "0", timePartition, tsFileManager, selectedSequenceFiles, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java index c5866109481af..977ebd1a43daf 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java @@ -72,8 +72,6 @@ public void testRepeatedSubmitBeforeExecution() throws Exception { tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task1 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -82,8 +80,6 @@ public void testRepeatedSubmitBeforeExecution() throws Exception { new AtomicInteger(0)); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -137,8 +133,6 @@ public void testRepeatedSubmitWhenExecuting() throws Exception { tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task1 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -147,8 +141,6 @@ public void testRepeatedSubmitWhenExecuting() throws Exception { new AtomicInteger(0)); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -203,8 +195,6 @@ public void testRepeatedSubmitAfterExecution() throws Exception { tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task1 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -213,8 +203,6 @@ public void testRepeatedSubmitAfterExecution() throws Exception { new AtomicInteger(0)); InnerSpaceCompactionTask task2 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -255,8 +243,6 @@ public void testRemoveSelfFromRunningList() throws Exception { tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task1 = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -301,8 +287,6 @@ public void testSizeTieredCompactionStatus() throws Exception { tsFileManager.addAll(seqResources, true); InnerSpaceCompactionTask task = new InnerSpaceCompactionTask( - "root.compactionTest", - "0", 0, tsFileManager, seqResources, @@ -330,8 +314,6 @@ public void testRewriteCrossCompactionFileStatus() throws Exception { tsFileManager.addAll(unseqResources, false); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( - COMPACTION_TEST_SG, - "0", 0, tsFileManager, seqResources, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java index 6c6c367d6bbf9..7dae07dfcd2da 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java @@ -431,10 +431,8 @@ public void testOneSeqFileAndSixUnseqFile() throws Exception { AbstractCompactionTask compactionTask = IoTDBDescriptor.getInstance() .getConfig() - .getCrossCompactionStrategy() + .getCrossCompactionSelector() .getCompactionTask( - COMPACTION_TEST_SG, - "0", 0, new TsFileManager( "root.compactionTest", @@ -736,10 +734,8 @@ public void testFiveSeqFileAndOneUnseqFileWithSomeDeviceNotInSeqFiles() throws E AbstractCompactionTask compactionTask = IoTDBDescriptor.getInstance() .getConfig() - .getCrossCompactionStrategy() + .getCrossCompactionSelector() .getCompactionTask( - COMPACTION_TEST_SG, - "0", 0, new TsFileManager( "root.compactionTest", @@ -1040,10 +1036,8 @@ public void testFiveSeqFileAndOneUnseqFile() throws Exception { AbstractCompactionTask compactionTask = IoTDBDescriptor.getInstance() .getConfig() - .getCrossCompactionStrategy() + .getCrossCompactionSelector() .getCompactionTask( - COMPACTION_TEST_SG, - "0", 0, new TsFileManager( "root.compactionTest", diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java index 219c061bb0d83..55cb5bf4298fe 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java @@ -221,8 +221,6 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInTimeseries() thro tsFileManager.addAll(unseqResources, false); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( - COMPACTION_TEST_SG, - "0", 0, tsFileManager, seqResources, @@ -459,8 +457,6 @@ public void testAlignedCrossSpaceCompactionWithAllDataDeletedInOneTargetFile() t tsFileManager.addAll(unseqResources, false); CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( - COMPACTION_TEST_SG, - "0", 0, tsFileManager, seqResources, @@ -607,8 +603,6 @@ public void testOneDeletionDuringCompaction() throws Exception { CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( - COMPACTION_TEST_SG, - "0", 0, vsgp.getTsFileResourceManager(), seqResources, @@ -718,8 +712,6 @@ public void testSeveralDeletionsDuringCompaction() throws Exception { CrossSpaceCompactionTask task = new CrossSpaceCompactionTask( - COMPACTION_TEST_SG, - "0", 0, vsgp.getTsFileResourceManager(), seqResources, diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java index a678cc12f91ff..8820d198d776b 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java @@ -22,16 +22,16 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.compaction.constant.CompactionPriority; -import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionStrategy; -import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionStrategy; +import org.apache.iotdb.db.engine.compaction.constant.CrossCompactionSelector; +import org.apache.iotdb.db.engine.compaction.constant.InnerSequenceCompactionSelector; public class CompactionConfigRestorer { private boolean enableSeqSpaceCompaction = true; private boolean enableUnseqSpaceCompaction = false; private boolean enableCrossSpaceCompaction = true; - private CrossCompactionStrategy crossStrategy = CrossCompactionStrategy.REWRITE_COMPACTION; - private InnerSequenceCompactionStrategy innerStrategy = - InnerSequenceCompactionStrategy.SIZE_TIERED_COMPACTION; + private CrossCompactionSelector crossStrategy = CrossCompactionSelector.REWRITE; + private InnerSequenceCompactionSelector innerStrategy = + InnerSequenceCompactionSelector.SIZE_TIERED; private CompactionPriority priority = CompactionPriority.BALANCE; private long targetFileSize = 1073741824L; private long targetChunkSize = 1048576L; @@ -52,8 +52,8 @@ public void restoreCompactionConfig() { config.setEnableSeqSpaceCompaction(enableSeqSpaceCompaction); config.setEnableUnseqSpaceCompaction(enableUnseqSpaceCompaction); config.setEnableCrossSpaceCompaction(enableCrossSpaceCompaction); - config.setCrossCompactionStrategy(crossStrategy); - config.setInnerSequenceCompactionStrategy(innerStrategy); + config.setCrossCompactionSelector(crossStrategy); + config.setInnerSequenceCompactionSelector(innerStrategy); config.setCompactionPriority(priority); config.setTargetCompactionFileSize(targetFileSize); config.setTargetChunkSize(targetChunkSize); diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java index 4c2732822eb1a..9c1d7bfee992d 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java @@ -724,8 +724,6 @@ public void testDeleteStorageGroupWhenCompacting() throws Exception { processor.syncCloseAllWorkingTsFileProcessors(); InnerSpaceCompactionTask task = new InnerSpaceCompactionTask( - storageGroup, - "0", 0, processor.getTsFileManager(), processor.getSequenceFileList(), From d9b9ef9b45c74da24009dfe583a2bd284e5de7c2 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Mon, 11 Apr 2022 23:09:41 +0800 Subject: [PATCH 21/22] add some comment --- .../db/engine/compaction/task/ICompactionSelector.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java index a97fb8a9f7d00..34acc5fa732c5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java @@ -31,12 +31,20 @@ */ public interface ICompactionSelector { /* - * This method should be implemented by all SequenceSpaceInnerSelector and UnsequenceSpaceInnerSelector. It takes the list of tsfile in a time partition as input, and returns a list of list. Each list in the returned list is the source files of one compaction tasks. + * This method should be implemented by all SequenceSpaceInnerSelector and UnsequenceSpaceInnerSelector. + * It takes the list of tsfile in a time partition as input, and returns a list of list. Each list in + * the returned list is the source files of one compaction tasks. */ default List> selectInnerSpaceTask(List resources) { throw new RuntimeException("This kind of selector cannot be used to select inner space task"); } + /* + * This method should be implemented by all CrossSpaceSelector. It takes the list of sequence files and + * list of unsequence files as input, and returns a list of pair of list and list. Each pair in the returned + * list contains two list: the left one is the selected sequence files, the right one is the selected + * unsequence files. Each pair is corresponding to a cross space compaction task. + */ default List, List>> selectCrossSpaceTask( List seqFiles, List unseqFiles) { { From ee091a5725dbea00fc77ad43a7307a12c47398e2 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Tue, 12 Apr 2022 09:39:27 +0800 Subject: [PATCH 22/22] fix ci --- .../inner/sizetiered/SizeTieredCompactionSelector.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java index e0a996e73725b..dc413a13701bc 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java @@ -86,6 +86,7 @@ public SizeTieredCompactionSelector( */ @Override public List> selectInnerSpaceTask(List tsFileResources) { + this.tsFileResources = tsFileResources; PriorityQueue, Long>> taskPriorityQueue = new PriorityQueue<>(new SizeTieredCompactionTaskComparator()); try {