From b705bcaceb45c050c5d10b7fc308ae104a60bf69 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Mon, 2 Feb 2026 14:40:21 +0800 Subject: [PATCH 1/8] [feature] Enrich catalog interface with more methods --- include/paimon/catalog/catalog.h | 235 +++++++++++++++++- include/paimon/catalog/database.h | 50 ++++ .../paimon/core/io/data_file_meta.h | 0 .../paimon/core/manifest/file_entry.h | 0 .../paimon/core/manifest/manifest_entry.h | 0 .../paimon/core/manifest/manifest_file_meta.h | 0 .../paimon/core/partition/partition_info.h | 0 .../core/partition/partition_statistics.h | 0 .../paimon/core/schema/table_schema.h | 0 {src => include}/paimon/core/snapshot.h | 0 .../paimon/core/stats/simple_stats.h | 0 include/paimon/core/table/table.h | 55 ++++ include/paimon/core/view/view.h | 52 ++++ 13 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 include/paimon/catalog/database.h rename {src => include}/paimon/core/io/data_file_meta.h (100%) rename {src => include}/paimon/core/manifest/file_entry.h (100%) rename {src => include}/paimon/core/manifest/manifest_entry.h (100%) rename {src => include}/paimon/core/manifest/manifest_file_meta.h (100%) rename {src => include}/paimon/core/partition/partition_info.h (100%) rename {src => include}/paimon/core/partition/partition_statistics.h (100%) rename {src => include}/paimon/core/schema/table_schema.h (100%) rename {src => include}/paimon/core/snapshot.h (100%) rename {src => include}/paimon/core/stats/simple_stats.h (100%) create mode 100644 include/paimon/core/table/table.h create mode 100644 include/paimon/core/view/view.h diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 523ee0e8..28eef25e 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -21,9 +21,13 @@ #include #include +#include "paimon/catalog/database.h" #include "paimon/catalog/identifier.h" +#include "paimon/core/partition/partition_statistics.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/table.h" +#include "paimon/core/view/view.h" #include "paimon/result.h" -#include "paimon/schema/schema.h" #include "paimon/status.h" #include "paimon/type_fwd.h" #include "paimon/visibility.h" @@ -32,7 +36,6 @@ struct ArrowSchema; namespace paimon { class Identifier; - /// This interface is responsible for reading and writing metadata such as database/table from a /// paimon catalog. class PAIMON_EXPORT Catalog { @@ -99,6 +102,234 @@ class PAIMON_EXPORT Catalog { /// status. virtual Result> ListTables(const std::string& db_name) const = 0; + /// Drops a database. + /// + /// @param name Name of the database to be dropped. + /// @param ignore_if_not_exists If true, no action is taken if the database does not exist. + /// @param cascade If true, drops all tables and functions in the database before dropping the + /// database. + /// @return A status indicating success or failure. + virtual Status DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { + return Status::NotImplemented("DropDatabase not implemented"); + } + + /// Alters a database. + /// + /// @param name Name of the database to alter. + /// @param changes Properties to be changed. + /// @param ignore_if_not_exists If true, no action is taken if the database does not exist. + /// @return A status indicating success or failure. + virtual Status AlterDatabase(const std::string& name, + const std::map& changes, + bool ignore_if_not_exists) { + return Status::NotImplemented("AlterDatabase not implemented"); + } + + /// Gets a database. + /// + /// @param name Name of the database to get. + /// @return A result containing the database information, or an error status. + virtual Result> GetDatabase(const std::string& name) const { + return Status::NotImplemented("GetDatabase not implemented"); + } + + /// Gets a table. + /// + /// @param identifier Identifier of the table to get. + /// @return A result containing the table, or an error status. + virtual Result> GetTable(const Identifier& identifier) const { + return Status::NotImplemented("GetTable not implemented"); + } + + /// Gets a table by ID. + /// + /// @param table_id ID of the table to get. + /// @return A result containing the table, or an error status. + virtual Result> GetTableById(const std::string& table_id) const { + return Status::NotImplemented("GetTableById not implemented"); + } + + /// Drops a table. + /// + /// @param identifier Identifier of the table to drop. + /// @param ignore_if_not_exists If true, no action is taken if the table does not exist. + /// @return A status indicating success or failure. + virtual Status DropTable(const Identifier& identifier, bool ignore_if_not_exists) { + return Status::NotImplemented("DropTable not implemented"); + } + + /// Renames a table. + /// + /// @param from_table Current identifier of the table. + /// @param to_table New identifier for the table. + /// @param ignore_if_not_exists If true, no action is taken if the table does not exist. + /// @return A status indicating success or failure. + virtual Status RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) { + return Status::NotImplemented("RenameTable not implemented"); + } + + /// TODO(liangzi): Support Alter table + + /// Invalidates cached table metadata. + /// + /// @param identifier Identifier of the table to invalidate. + virtual void InvalidateTable(const Identifier& identifier) {} + + /// Marks partitions as done. + /// + /// @param identifier Identifier of the table. + /// @param partitions List of partition specifications. + /// @return A status indicating success or failure. + virtual Status MarkDonePartitions( + const Identifier& identifier, + const std::vector>& partitions) { + return Status::NotImplemented("MarkDonePartitions not implemented"); + } + + /// Lists all partitions of a table. + /// + /// @param identifier Identifier of the table. + /// @return A result containing a list of partitions, or an error status. + virtual Result>> ListPartitions( + const Identifier& identifier) const { + return Status::NotImplemented("ListPartitions not implemented"); + } + + /// Creates partitions. + /// + /// @param identifier Identifier of the table. + /// @param partitions List of partition specifications to create. + /// @return A status indicating success or failure. + virtual Status CreatePartitions( + const Identifier& identifier, + const std::vector>& partitions) { + return Status::NotImplemented("CreatePartitions not implemented"); + } + + /// Drops partitions. + /// + /// @param identifier Identifier of the table. + /// @param partitions List of partition specifications to drop. + /// @return A status indicating success or failure. + virtual Status DropPartitions( + const Identifier& identifier, + const std::vector>& partitions) { + return Status::NotImplemented("DropPartitions not implemented"); + } + + /// Alters partitions. + /// + /// @param identifier Identifier of the table. + /// @param partitions List of partition statistics to alter. + /// @return A status indicating success or failure. + virtual Status AlterPartitions(const Identifier& identifier, + const std::vector& partitions) { + return Status::NotImplemented("AlterPartitions not implemented"); + } + + /// Gets a view. + /// + /// @param identifier Identifier of the view to get. + /// @return A result containing the view, or an error status. + virtual Result> GetView(const Identifier& identifier) const { + return Status::NotImplemented("GetView not implemented"); + } + + /// Drops a view. + /// + /// @param identifier Identifier of the view to drop. + /// @param ignore_if_not_exists If true, no action is taken if the view does not exist. + /// @return A status indicating success or failure. + virtual Status DropView(const Identifier& identifier, bool ignore_if_not_exists) { + return Status::NotImplemented("DropView not implemented"); + } + + /// Creates a view. + /// + /// @param identifier Identifier of the view to create. + /// @param view The view definition. + /// @param ignore_if_exists If true, no action is taken if the view already exists. + /// @return A status indicating success or failure. + virtual Status CreateView(const Identifier& identifier, const View& view, + bool ignore_if_exists) { + return Status::NotImplemented("CreateView not implemented"); + } + + /// Lists all views in a database. + /// + /// @param database_name Name of the database. + /// @return A result containing a list of view names, or an error status. + virtual Result> ListViews(const std::string& database_name) const { + return Status::NotImplemented("ListViews not implemented"); + } + + /// Renames a view. + /// + /// @param from_view Current identifier of the view. + /// @param to_view New identifier for the view. + /// @param ignore_if_not_exists If true, no action is taken if the view does not exist. + /// @return A status indicating success or failure. + virtual Status RenameView(const Identifier& from_view, const Identifier& to_view, + bool ignore_if_not_exists) { + return Status::NotImplemented("RenameView not implemented"); + } + + /// TODO(liangzi): Support Function/Snapshot/Tag/Branch/Authorizes api + + /// Repairs the entire catalog. + /// + /// @return A status indicating success or failure. + virtual Status RepairCatalog() { + return Status::NotImplemented("RepairCatalog not implemented"); + } + + /// Repairs a database. + /// + /// @param database_name Name of the database to repair. + /// @return A status indicating success or failure. + virtual Status RepairDatabase(const std::string& database_name) { + return Status::NotImplemented("RepairDatabase not implemented"); + } + + /// Repairs a table. + /// + /// @param identifier Identifier of the table to repair. + /// @return A status indicating success or failure. + virtual Status RepairTable(const Identifier& identifier) { + return Status::NotImplemented("RepairTable not implemented"); + } + + /// Registers a table. + /// + /// @param identifier Identifier of the table to register. + /// @param path Path of the table. + /// @return A status indicating success or failure. + virtual Status RegisterTable(const Identifier& identifier, const std::string& path) { + return Status::NotImplemented("RegisterTable not implemented"); + } + + /// Checks if list objects paged is supported. + /// + /// @return True if supported, false otherwise. + virtual bool SupportsListObjectsPaged() const { + return false; + } + + /// Checks if list by pattern is supported. + /// + /// @return True if supported, false otherwise. + virtual bool SupportsListByPattern() const { + return false; + } + + /// Checks if list table by type is supported. + /// + /// @return True if supported, false otherwise. + virtual bool SupportsListTableByType() const { + return false; + } + /// Checks whether a database with the specified name exists in the catalog. /// /// @param db_name The name of the database to check for existence. diff --git a/include/paimon/catalog/database.h b/include/paimon/catalog/database.h new file mode 100644 index 00000000..e242a9d5 --- /dev/null +++ b/include/paimon/catalog/database.h @@ -0,0 +1,50 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/type_fwd.h" +#include "paimon/visibility.h" + +struct ArrowSchema; + +namespace paimon { + +/// Interface of a database in a catalog. +class PAIMON_EXPORT Database { + public: + /// ================== Table Metadata ===================== + + /// A name to identify this database. + virtual std::string Name() = 0; + + /// Get the table-level options associated with this schema. + /// @return Options + virtual const std::map& Options() const = 0; + + /// Get an optional comment describing the table. + /// @return The table comment if set, or std::nullopt otherwise. + virtual std::optional Comment() const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/core/io/data_file_meta.h b/include/paimon/core/io/data_file_meta.h similarity index 100% rename from src/paimon/core/io/data_file_meta.h rename to include/paimon/core/io/data_file_meta.h diff --git a/src/paimon/core/manifest/file_entry.h b/include/paimon/core/manifest/file_entry.h similarity index 100% rename from src/paimon/core/manifest/file_entry.h rename to include/paimon/core/manifest/file_entry.h diff --git a/src/paimon/core/manifest/manifest_entry.h b/include/paimon/core/manifest/manifest_entry.h similarity index 100% rename from src/paimon/core/manifest/manifest_entry.h rename to include/paimon/core/manifest/manifest_entry.h diff --git a/src/paimon/core/manifest/manifest_file_meta.h b/include/paimon/core/manifest/manifest_file_meta.h similarity index 100% rename from src/paimon/core/manifest/manifest_file_meta.h rename to include/paimon/core/manifest/manifest_file_meta.h diff --git a/src/paimon/core/partition/partition_info.h b/include/paimon/core/partition/partition_info.h similarity index 100% rename from src/paimon/core/partition/partition_info.h rename to include/paimon/core/partition/partition_info.h diff --git a/src/paimon/core/partition/partition_statistics.h b/include/paimon/core/partition/partition_statistics.h similarity index 100% rename from src/paimon/core/partition/partition_statistics.h rename to include/paimon/core/partition/partition_statistics.h diff --git a/src/paimon/core/schema/table_schema.h b/include/paimon/core/schema/table_schema.h similarity index 100% rename from src/paimon/core/schema/table_schema.h rename to include/paimon/core/schema/table_schema.h diff --git a/src/paimon/core/snapshot.h b/include/paimon/core/snapshot.h similarity index 100% rename from src/paimon/core/snapshot.h rename to include/paimon/core/snapshot.h diff --git a/src/paimon/core/stats/simple_stats.h b/include/paimon/core/stats/simple_stats.h similarity index 100% rename from src/paimon/core/stats/simple_stats.h rename to include/paimon/core/stats/simple_stats.h diff --git a/include/paimon/core/table/table.h b/include/paimon/core/table/table.h new file mode 100644 index 00000000..76ebcda4 --- /dev/null +++ b/include/paimon/core/table/table.h @@ -0,0 +1,55 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/type_fwd.h" +#include "paimon/visibility.h" + +struct ArrowSchema; + +namespace paimon { + +/// A table provides basic abstraction for table type. +class PAIMON_EXPORT Table { + public: + /// A name to identify this table. + virtual std::string Name() = 0; + + /// Full name of the table, default is database.tableName. + virtual std::string FullName() { + return Name(); + } + + /// UUID of the table, metastore can provide the true UUID of this table, default is the full + /// name. + virtual std::string Uuid() { + return FullName(); + } + + /// Loads the latest schema of table. + virtual std::shared_ptr LatestSchema() = 0; +}; + +} // namespace paimon diff --git a/include/paimon/core/view/view.h b/include/paimon/core/view/view.h new file mode 100644 index 00000000..2a2282a1 --- /dev/null +++ b/include/paimon/core/view/view.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/type_fwd.h" +#include "paimon/visibility.h" + +struct ArrowSchema; + +namespace paimon { + +/// Interface for view definition. +class PAIMON_EXPORT View { + public: + /// A name to identify this view. + virtual std::string Name() = 0; + + /// Full name of the view, default is database.tableName. + virtual std::string FullName() { + return Name(); + } + + /// Returns the view representation. + virtual std::string Query() = 0; + + /// Loads the schema of view. + virtual std::shared_ptr GetSchema() = 0; +}; + +} // namespace paimon From b201f4a5b18e44b7778ad92580336c1d1b4cde29 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Mon, 2 Feb 2026 16:33:50 +0800 Subject: [PATCH 2/8] add impl --- include/paimon/core/table/table.h | 23 +++++++++++++++---- .../core/catalog/file_system_catalog.cpp | 16 +++++++++++++ src/paimon/core/catalog/file_system_catalog.h | 1 + .../core/catalog/file_system_catalog_test.cpp | 11 +++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/include/paimon/core/table/table.h b/include/paimon/core/table/table.h index 76ebcda4..1996bdc4 100644 --- a/include/paimon/core/table/table.h +++ b/include/paimon/core/table/table.h @@ -34,22 +34,37 @@ namespace paimon { /// A table provides basic abstraction for table type. class PAIMON_EXPORT Table { public: + Table(const std::shared_ptr& schema, const std::string& database, + const std::string& table_name) + : schema_(schema), database_(database), table_name_(table_name) {} + + virtual ~Table() {} + /// A name to identify this table. - virtual std::string Name() = 0; + virtual std::string Name() const { + return database_ + "." + table_name_; + } /// Full name of the table, default is database.tableName. - virtual std::string FullName() { + virtual std::string FullName() const { return Name(); } /// UUID of the table, metastore can provide the true UUID of this table, default is the full /// name. - virtual std::string Uuid() { + virtual std::string Uuid() const { return FullName(); } /// Loads the latest schema of table. - virtual std::shared_ptr LatestSchema() = 0; + virtual std::shared_ptr LatestSchema() { + return schema_; + } + + private: + std::shared_ptr schema_; + std::string database_; + std::string table_name_; }; } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index bfb4bdb2..aa02474a 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -240,4 +240,20 @@ Result> FileSystemCatalog::LoadTableSchema( return std::static_pointer_cast(*latest_schema); } +Result> FileSystemCatalog::GetTable(const Identifier& identifier) const { + std::string table_path = GetTableLocation(identifier); + PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); + if (!exist) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + TableSchemaExists(identifier)); + if (!latest_schema) { + return Status::NotExist( + fmt::format("load table schema for {} failed", identifier.ToString())); + } + auto schema = std::static_pointer_cast(*latest_schema); + return std::make_shared(schema, identifier.GetDatabaseName(), identifier.GetTableName()); +} + } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 4cece2f5..b2c6551d 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -56,6 +56,7 @@ class FileSystemCatalog : public Catalog { Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; + Result> GetTable(const Identifier& identifier) const override; private: static std::string NewDatabasePath(const std::string& warehouse, const std::string& db_name); diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index ede6918f..fa57dbc4 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -190,6 +190,17 @@ TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { ASSERT_OK_AND_ASSIGN(auto arrow_schema, table_schema->GetArrowSchema()); auto loaded_schema = arrow::ImportSchema(arrow_schema.get()).ValueOrDie(); ASSERT_TRUE(typed_schema.Equals(loaded_schema)); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog.GetTable(Identifier("db1", "tbl1"))); + ASSERT_OK_AND_ASSIGN(auto arrow_schema_from_get_table, table->LatestSchema()->GetArrowSchema()); + auto schema_from_get_table = + arrow::ImportSchema(arrow_schema_from_get_table.get()).ValueOrDie(); + ASSERT_TRUE(typed_schema.Equals(schema_from_get_table)); + ASSERT_EQ(table->FullName(), "db1.tbl1"); + + ASSERT_NOK_WITH_MSG(catalog.GetTable(Identifier("db1", "table_xaxa")), + "Identifier{database='db1', table='table_xaxa'} not exist"); + ArrowSchemaRelease(&schema); } From e41494bd1957de87234ce12eba8b1be564e9be10 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Mon, 2 Feb 2026 18:13:25 +0800 Subject: [PATCH 3/8] address --- include/paimon/core/table/table.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/paimon/core/table/table.h b/include/paimon/core/table/table.h index 1996bdc4..dafcda5e 100644 --- a/include/paimon/core/table/table.h +++ b/include/paimon/core/table/table.h @@ -38,7 +38,7 @@ class PAIMON_EXPORT Table { const std::string& table_name) : schema_(schema), database_(database), table_name_(table_name) {} - virtual ~Table() {} + virtual ~Table() = default; /// A name to identify this table. virtual std::string Name() const { From 65f968c80f55a5f36897194569ebbf972ace216d Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Mon, 2 Feb 2026 19:59:44 +0800 Subject: [PATCH 4/8] snapshot and tag api --- include/paimon/catalog/catalog.h | 168 +++++++++++++++++++++++++++++++ include/paimon/core/tag_info.h | 93 +++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 include/paimon/core/tag_info.h diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 28eef25e..49a207f9 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -25,7 +25,9 @@ #include "paimon/catalog/identifier.h" #include "paimon/core/partition/partition_statistics.h" #include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" #include "paimon/core/table/table.h" +#include "paimon/core/tag_info.h" #include "paimon/core/view/view.h" #include "paimon/result.h" #include "paimon/status.h" @@ -330,6 +332,172 @@ class PAIMON_EXPORT Catalog { return false; } + // ==================== Version management methods ========================== + + /// \brief Whether this catalog supports version management for tables. + /// + /// If not supported, corresponding methods will return NotImplemented status. + /// Affected methods: + /// - CommitSnapshot() + /// - LoadSnapshot() + /// - RollbackTo() + /// - CreateBranch() + /// - DropBranch() + /// - ListBranches() + /// - GetTag() + /// - CreateTag() + /// - DeleteTag() + virtual bool SupportsVersionManagement() const { + return false; + } + + /// \brief Commit the Snapshot for table identified by the given Identifier. + /// + /// \param identifier Path of the table + /// \param table_uuid Uuid of the table to avoid wrong commit + /// \param snapshot Snapshot to be committed + /// \param statistics Statistics information of this change + /// \return Success or not + /// \return TableNotExist error if the target does not exist + /// \return NotImplemented error if the catalog does not support version management + virtual Result CommitSnapshot(const Identifier& identifier, const std::string& table_uuid, + const std::shared_ptr& snapshot, + const std::vector& statistics) { + return Status::NotImplemented("CommitSnapshot not implemented"); + } + + /// \brief Return the snapshot of table for given version. + /// + /// Version parsing order: + /// 1. If it is 'EARLIEST', get the earliest snapshot + /// 2. If it is 'LATEST', get the latest snapshot + /// 3. If it is a number, get snapshot by snapshot id + /// 4. Else try to get snapshot from Tag name + /// + /// \param identifier Path of the table + /// \param version Version to snapshot + /// \return The requested snapshot + /// \return TableNotExist error if the target does not exist + /// \return NotImplemented error if the catalog does not support version management + virtual Result> LoadSnapshot(const Identifier& identifier, + const std::string& version) const { + return Status::NotImplemented("LoadSnapshot not implemented"); + } + + /// \brief Rollback table by the given Identifier and instant. + /// + /// \param identifier Path of the table + /// \param instant Like snapshotId or tagName + /// \return TableNotExist error if the table does not exist + /// \return NotImplemented error if the catalog does not support version management + virtual Status RollbackTo(const Identifier& identifier, + const std::chrono::system_clock::time_point& instant) { + return RollbackTo(identifier, instant, std::nullopt); + } + + /// \brief Rollback table by the given Identifier and instant. + /// + /// \param identifier Path of the table + /// \param instant Like snapshotId or tagName + /// \param from_snapshot Snapshot from, success only occurs when the latest snapshot is this + /// snapshot + /// \return TableNotExist error if the table does not exist + /// \return NotImplemented error if the catalog does not support version management + virtual Status RollbackTo(const Identifier& identifier, + const std::chrono::system_clock::time_point& instant, + const std::optional& from_snapshot) { + return Status::NotImplemented("RollbackTo not implemented"); + } + + /// \brief Create a new branch for this table. + /// + /// By default, an empty branch will be created using the latest schema. + /// If from_tag is provided, a branch will be created from the tag and the + /// data files will be inherited from it. + /// + /// \param identifier Path of the table, cannot be system or branch name + /// \param branch The branch name + /// \param from_tag From the tag + /// \return TableNotExist error if the table in identifier doesn't exist + /// \return BranchAlreadyExist error if the branch already exists + /// \return TagNotExist error if the tag doesn't exist + /// \return NotImplemented error if the catalog does not support version management + virtual Status CreateBranch(const Identifier& identifier, const std::string& branch, + const std::optional& from_tag = std::nullopt) { + return Status::NotImplemented("CreateBranch not implemented"); + } + + /// \brief Drop the branch for this table. + /// + /// \param identifier Path of the table, cannot be system or branch name + /// \param branch The branch name + /// \return BranchNotExist error if the branch doesn't exist + /// \return NotImplemented error if the catalog does not support version management + virtual Status DropBranch(const Identifier& identifier, const std::string& branch) { + return Status::NotImplemented("DropBranch not implemented"); + } + + /// \brief Fast-forward a branch to main branch. + /// + /// \param identifier Path of the table, cannot be system or branch name + /// \param branch The branch name + /// \return BranchNotExist error if the branch doesn't exist + /// \return NotImplemented error if the catalog does not support version management + virtual Status FastForward(const Identifier& identifier, const std::string& branch) { + return Status::NotImplemented("FastForward not implemented"); + } + + /// \brief List all branches of the table. + /// + /// \param identifier Path of the table, cannot be system or branch name + /// \return TableNotExist error if the table in identifier doesn't exist + /// \return NotImplemented error if the catalog does not support version management + virtual Result> ListBranches(const Identifier& identifier) const { + return Status::NotImplemented("ListBranches not implemented"); + } + + /// Get tag for table. + /// + /// @param identifier path of the table, cannot be system name. + /// @param tag_name tag name + /// @return TagInfo containing tag information + /// @return TableNotExist error if the table does not exist + /// @return TagNotExist error if the tag does not exist + /// @return NotImplemented error if the catalog does not support version management + virtual Result GetTag(const Identifier& identifier, + const std::string& tag_name) const { + return Status::NotImplemented("GetTag not implemented"); + } + + /// Create tag for table. + /// + /// @param identifier path of the table, cannot be system name. + /// @param tag_name tag name + /// @param snapshot_id optional snapshot id, if not provided uses latest snapshot + /// @param time_retained optional time retained as string (e.g., "1d", "12h", "30m") + /// @param ignore_if_exists if true, ignore if tag already exists + /// @return TableNotExist error if the table does not exist + /// @return SnapshotNotExist error if the snapshot does not exist + /// @return TagAlreadyExist error if the tag already exists and ignore_if_exists is false + /// @return NotImplemented error if the catalog does not support version management + virtual Status CreateTag(const Identifier& identifier, const std::string& tag_name, + const std::optional& snapshot_id, + const std::optional& time_retained, + bool ignore_if_exists) { + return Status::NotImplemented("CreateTag not implemented"); + } + + /// Delete tag for table. + /// + /// @param identifier path of the table, cannot be system name. + /// @param tag_name tag name + /// @return TableNotExist error if the table does not exist + /// @return TagNotExist error if the tag does not exist + /// @return NotImplemented error if the catalog does not support version management + virtual Status DeleteTag(const Identifier& identifier, const std::string& tag_name) { + return Status::NotImplemented("DeleteTag not implemented"); + } + /// Checks whether a database with the specified name exists in the catalog. /// /// @param db_name The name of the database to check for existence. diff --git a/include/paimon/core/tag_info.h b/include/paimon/core/tag_info.h new file mode 100644 index 00000000..07fc167d --- /dev/null +++ b/include/paimon/core/tag_info.h @@ -0,0 +1,93 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" +#include "paimon/type_fwd.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +class TagInfo : public Jsonizable { + public: + static constexpr char FIELD_TAG_NAME[] = "tagName"; + static constexpr char FIELD_SNAPSHOT[] = "snapshot"; + static constexpr char FIELD_TAG_CREATE_TIME[] = "tagCreateTime"; + static constexpr char FIELD_TAG_TIME_RETAINED[] = "tagTimeRetained"; + + JSONIZABLE_FRIEND_AND_DEFAULT_CTOR(TagInfo); + + TagInfo(const std::string& tag_name, const std::shared_ptr& snapshot, + const std::optional& tag_create_time, + const std::optional& tag_time_retained) + : tag_name_(tag_name), + snapshot_(snapshot), + tag_create_time_(tag_create_time), + tag_time_retained_(tag_time_retained) {} + + const std::string& TagName() const { + return tag_name_; + } + void SetTagName(const std::string& tag_name) { + tag_name_ = tag_name; + } + + const std::shared_ptr& GetSnapshot() const { + return snapshot_; + } + void SetSnapshot(const std::shared_ptr& snapshot) { + snapshot_ = snapshot; + } + + const std::optional& TagCreateTime() const { + return tag_create_time_; + } + void SetTagCreateTime(const std::optional& tag_create_time) { + tag_create_time_ = tag_create_time; + } + + const std::optional& TagTimeRetained() const { + return tag_time_retained_; + } + void SetTagTimeRetained(const std::optional& tag_time_retained) { + tag_time_retained_ = tag_time_retained; + } + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + private: + std::string tag_name_; + std::shared_ptr snapshot_; + std::optional tag_create_time_; + std::optional tag_time_retained_; +}; + +} // namespace paimon From 819a29905f091ae2cc12a9cb8f4a1d7bd8ff12dc Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Thu, 5 Feb 2026 13:44:12 +0800 Subject: [PATCH 5/8] refactor --- include/paimon/catalog/catalog.h | 21 ++-- include/paimon/core/tag_info.h | 93 -------------- src/paimon/CMakeLists.txt | 1 + .../core/catalog/file_system_catalog.cpp | 1 - src/paimon/core/catalog/file_system_catalog.h | 2 + .../paimon/core/io/data_file_meta.h | 0 .../paimon/core/manifest/file_entry.h | 0 .../paimon/core/manifest/manifest_entry.h | 0 .../paimon/core/manifest/manifest_file_meta.h | 0 .../paimon/core/partition/partition_info.h | 0 .../core/partition/partition_statistics.h | 0 .../paimon/core/schema/table_schema.h | 0 {include => src}/paimon/core/snapshot.h | 0 .../paimon/core/stats/simple_stats.h | 0 {include => src}/paimon/core/table/table.h | 0 src/paimon/core/tag/tag.cpp | 115 ++++++++++++++++++ src/paimon/core/tag/tag.h | 86 +++++++++++++ {include => src}/paimon/core/view/view.h | 3 + 18 files changed, 218 insertions(+), 104 deletions(-) delete mode 100644 include/paimon/core/tag_info.h rename {include => src}/paimon/core/io/data_file_meta.h (100%) rename {include => src}/paimon/core/manifest/file_entry.h (100%) rename {include => src}/paimon/core/manifest/manifest_entry.h (100%) rename {include => src}/paimon/core/manifest/manifest_file_meta.h (100%) rename {include => src}/paimon/core/partition/partition_info.h (100%) rename {include => src}/paimon/core/partition/partition_statistics.h (100%) rename {include => src}/paimon/core/schema/table_schema.h (100%) rename {include => src}/paimon/core/snapshot.h (100%) rename {include => src}/paimon/core/stats/simple_stats.h (100%) rename {include => src}/paimon/core/table/table.h (100%) create mode 100644 src/paimon/core/tag/tag.cpp create mode 100644 src/paimon/core/tag/tag.h rename {include => src}/paimon/core/view/view.h (96%) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 49a207f9..0258b865 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -16,19 +16,13 @@ #pragma once +#include #include #include #include #include -#include "paimon/catalog/database.h" #include "paimon/catalog/identifier.h" -#include "paimon/core/partition/partition_statistics.h" -#include "paimon/core/schema/table_schema.h" -#include "paimon/core/snapshot.h" -#include "paimon/core/table/table.h" -#include "paimon/core/tag_info.h" -#include "paimon/core/view/view.h" #include "paimon/result.h" #include "paimon/status.h" #include "paimon/type_fwd.h" @@ -37,6 +31,13 @@ struct ArrowSchema; namespace paimon { +class Database; +class Table; +class View; +class Schema; +class Snapshot; +class PartitionStatistics; +class Tag; class Identifier; /// This interface is responsible for reading and writing metadata such as database/table from a /// paimon catalog. @@ -460,12 +461,12 @@ class PAIMON_EXPORT Catalog { /// /// @param identifier path of the table, cannot be system name. /// @param tag_name tag name - /// @return TagInfo containing tag information + /// @return Tag containing tag information /// @return TableNotExist error if the table does not exist /// @return TagNotExist error if the tag does not exist /// @return NotImplemented error if the catalog does not support version management - virtual Result GetTag(const Identifier& identifier, - const std::string& tag_name) const { + virtual Result> GetTag(const Identifier& identifier, + const std::string& tag_name) const { return Status::NotImplemented("GetTag not implemented"); } diff --git a/include/paimon/core/tag_info.h b/include/paimon/core/tag_info.h deleted file mode 100644 index 07fc167d..00000000 --- a/include/paimon/core/tag_info.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2026-present Alibaba Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "paimon/common/utils/jsonizable.h" -#include "paimon/core/snapshot.h" -#include "paimon/result.h" -#include "paimon/type_fwd.h" -#include "rapidjson/allocators.h" -#include "rapidjson/document.h" -#include "rapidjson/rapidjson.h" - -namespace paimon { - -class TagInfo : public Jsonizable { - public: - static constexpr char FIELD_TAG_NAME[] = "tagName"; - static constexpr char FIELD_SNAPSHOT[] = "snapshot"; - static constexpr char FIELD_TAG_CREATE_TIME[] = "tagCreateTime"; - static constexpr char FIELD_TAG_TIME_RETAINED[] = "tagTimeRetained"; - - JSONIZABLE_FRIEND_AND_DEFAULT_CTOR(TagInfo); - - TagInfo(const std::string& tag_name, const std::shared_ptr& snapshot, - const std::optional& tag_create_time, - const std::optional& tag_time_retained) - : tag_name_(tag_name), - snapshot_(snapshot), - tag_create_time_(tag_create_time), - tag_time_retained_(tag_time_retained) {} - - const std::string& TagName() const { - return tag_name_; - } - void SetTagName(const std::string& tag_name) { - tag_name_ = tag_name; - } - - const std::shared_ptr& GetSnapshot() const { - return snapshot_; - } - void SetSnapshot(const std::shared_ptr& snapshot) { - snapshot_ = snapshot; - } - - const std::optional& TagCreateTime() const { - return tag_create_time_; - } - void SetTagCreateTime(const std::optional& tag_create_time) { - tag_create_time_ = tag_create_time; - } - - const std::optional& TagTimeRetained() const { - return tag_time_retained_; - } - void SetTagTimeRetained(const std::optional& tag_time_retained) { - tag_time_retained_ = tag_time_retained; - } - - rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const - noexcept(false) override; - - void FromJson(const rapidjson::Value& obj) noexcept(false) override; - - private: - std::string tag_name_; - std::shared_ptr snapshot_; - std::optional tag_create_time_; - std::optional tag_time_retained_; -}; - -} // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 33c45ba1..24ed89dd 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -265,6 +265,7 @@ set(PAIMON_CORE_SRCS core/table/source/table_read.cpp core/table/source/table_scan.cpp core/table/source/data_evolution_batch_scan.cpp + core/tag/tag.cpp core/utils/field_mapping.cpp core/utils/fields_comparator.cpp core/utils/file_store_path_factory.cpp diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index aa02474a..921e37a8 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -27,7 +27,6 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" -#include "paimon/core/schema/schema_manager.h" #include "paimon/fs/file_system.h" #include "paimon/logging.h" #include "paimon/result.h" diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index b2c6551d..408f6769 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -22,6 +22,8 @@ #include #include "paimon/catalog/catalog.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/table/table.h" #include "paimon/logging.h" #include "paimon/result.h" #include "paimon/status.h" diff --git a/include/paimon/core/io/data_file_meta.h b/src/paimon/core/io/data_file_meta.h similarity index 100% rename from include/paimon/core/io/data_file_meta.h rename to src/paimon/core/io/data_file_meta.h diff --git a/include/paimon/core/manifest/file_entry.h b/src/paimon/core/manifest/file_entry.h similarity index 100% rename from include/paimon/core/manifest/file_entry.h rename to src/paimon/core/manifest/file_entry.h diff --git a/include/paimon/core/manifest/manifest_entry.h b/src/paimon/core/manifest/manifest_entry.h similarity index 100% rename from include/paimon/core/manifest/manifest_entry.h rename to src/paimon/core/manifest/manifest_entry.h diff --git a/include/paimon/core/manifest/manifest_file_meta.h b/src/paimon/core/manifest/manifest_file_meta.h similarity index 100% rename from include/paimon/core/manifest/manifest_file_meta.h rename to src/paimon/core/manifest/manifest_file_meta.h diff --git a/include/paimon/core/partition/partition_info.h b/src/paimon/core/partition/partition_info.h similarity index 100% rename from include/paimon/core/partition/partition_info.h rename to src/paimon/core/partition/partition_info.h diff --git a/include/paimon/core/partition/partition_statistics.h b/src/paimon/core/partition/partition_statistics.h similarity index 100% rename from include/paimon/core/partition/partition_statistics.h rename to src/paimon/core/partition/partition_statistics.h diff --git a/include/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h similarity index 100% rename from include/paimon/core/schema/table_schema.h rename to src/paimon/core/schema/table_schema.h diff --git a/include/paimon/core/snapshot.h b/src/paimon/core/snapshot.h similarity index 100% rename from include/paimon/core/snapshot.h rename to src/paimon/core/snapshot.h diff --git a/include/paimon/core/stats/simple_stats.h b/src/paimon/core/stats/simple_stats.h similarity index 100% rename from include/paimon/core/stats/simple_stats.h rename to src/paimon/core/stats/simple_stats.h diff --git a/include/paimon/core/table/table.h b/src/paimon/core/table/table.h similarity index 100% rename from include/paimon/core/table/table.h rename to src/paimon/core/table/table.h diff --git a/src/paimon/core/tag/tag.cpp b/src/paimon/core/tag/tag.cpp new file mode 100644 index 00000000..e98c8634 --- /dev/null +++ b/src/paimon/core/tag/tag.cpp @@ -0,0 +1,115 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/tag/tag.h" + +#include +#include +#include + +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/fs/file_system.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +Tag::Tag(const std::optional& version, int64_t id, int64_t schema_id, + const std::string& base_manifest_list, + const std::optional& base_manifest_list_size, + const std::string& delta_manifest_list, + const std::optional& delta_manifest_list_size, + const std::optional& changelog_manifest_list, + const std::optional& changelog_manifest_list_size, + const std::optional& index_manifest, const std::string& commit_user, + int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, + const std::optional>& log_offsets, + const std::optional& total_record_count, + const std::optional& delta_record_count, + const std::optional& changelog_record_count, + const std::optional& watermark, const std::optional& statistics, + const std::optional>& properties, + const std::optional& next_row_id, const std::optional& tag_create_time, + const std::optional& tag_time_retained) + : Snapshot(version, id, schema_id, base_manifest_list, base_manifest_list_size, + delta_manifest_list, delta_manifest_list_size, changelog_manifest_list, + changelog_manifest_list_size, index_manifest, commit_user, commit_identifier, + commit_kind, time_millis, log_offsets, total_record_count, delta_record_count, + changelog_record_count, watermark, statistics, properties, next_row_id), + tag_create_time_(tag_create_time), + tag_time_retained_(tag_time_retained) {} + +bool Tag::operator==(const Tag& other) const { + if (this == &other) { + return true; + } + return Snapshot::operator==(other) && tag_create_time_ == other.tag_create_time_ && + tag_time_retained_ == other.tag_time_retained_; +} + +bool Tag::TEST_Equal(const Tag& other) const { + if (this == &other) { + return true; + } + + return Snapshot::TEST_Equal(other) && tag_create_time_ == other.tag_create_time_ && + tag_time_retained_ == other.tag_time_retained_; +} + +Result Tag::TrimToSnapshot() const { + return Snapshot(Version(), Id(), SchemaId(), BaseManifestList(), BaseManifestListSize(), + DeltaManifestList(), DeltaManifestListSize(), ChangelogManifestList(), + ChangelogManifestListSize(), IndexManifest(), CommitUser(), CommitIdentifier(), + GetCommitKind(), TimeMillis(), LogOffsets(), TotalRecordCount(), + DeltaRecordCount(), ChangelogRecordCount(), Watermark(), Statistics(), + Properties(), NextRowId()); +} + +rapidjson::Value Tag::ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj = Snapshot::ToJson(allocator); + if (tag_create_time_ != std::nullopt) { + obj.AddMember(rapidjson::StringRef(FIELD_TAG_CREATE_TIME), + RapidJsonUtil::SerializeValue(tag_create_time_.value(), allocator).Move(), + *allocator); + } + if (tag_time_retained_ != std::nullopt) { + obj.AddMember(rapidjson::StringRef(FIELD_TAG_TIME_RETAINED), + RapidJsonUtil::SerializeValue(tag_time_retained_.value(), allocator).Move(), + *allocator); + } + return obj; +} + +void Tag::FromJson(const rapidjson::Value& obj) noexcept(false) { + Snapshot::FromJson(obj); + tag_create_time_ = + RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_TAG_CREATE_TIME); + tag_time_retained_ = + RapidJsonUtil::DeserializeKeyValue>(obj, FIELD_TAG_TIME_RETAINED); +} + +Result Tag::FromPath(const std::shared_ptr& fs, const std::string& path) { + std::string json_str; + PAIMON_RETURN_NOT_OK(fs->ReadFile(path, &json_str)); + Tag tag; + PAIMON_RETURN_NOT_OK(RapidJsonUtil::FromJsonString(json_str, &tag)); + return tag; +} +} // namespace paimon \ No newline at end of file diff --git a/src/paimon/core/tag/tag.h b/src/paimon/core/tag/tag.h new file mode 100644 index 00000000..88fcfa75 --- /dev/null +++ b/src/paimon/core/tag/tag.h @@ -0,0 +1,86 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License");Expand commentComment on lines R1 to R4Resolved + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/core/snapshot.h" +#include "paimon/result.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { +class FileSystem; + +/// Snapshot with tagCreateTime and tagTimeRetained. +class Tag : public Snapshot { + public: + static constexpr char FIELD_TAG_CREATE_TIME[] = "tagCreateTime"; + static constexpr char FIELD_TAG_TIME_RETAINED[] = "tagTimeRetained"; + + JSONIZABLE_FRIEND_AND_DEFAULT_CTOR(Tag); + + Tag(const std::optional& version, int64_t id, int64_t schema_id, + const std::string& base_manifest_list, + const std::optional& base_manifest_list_size, + const std::string& delta_manifest_list, + const std::optional& delta_manifest_list_size, + const std::optional& changelog_manifest_list, + const std::optional& changelog_manifest_list_size, + const std::optional& index_manifest, const std::string& commit_user, + int64_t commit_identifier, CommitKind commit_kind, int64_t time_millis, + const std::optional>& log_offsets, + const std::optional& total_record_count, + const std::optional& delta_record_count, + const std::optional& changelog_record_count, + const std::optional& watermark, const std::optional& statistics, + const std::optional>& properties, + const std::optional& next_row_id, const std::optional& tag_create_time, + const std::optional& tag_time_retained); + + bool operator==(const Tag& other) const; + bool TEST_Equal(const Tag& other) const; + + std::optional TagCreateTime() const { + return tag_create_time_; + } + + std::optional TagTimeRetained() const { + return tag_time_retained_; + } + + Result TrimToSnapshot() const; + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + static Result FromPath(const std::shared_ptr& fs, const std::string& path); + + private: + std::optional tag_create_time_; + std::optional tag_time_retained_; +}; +} // namespace paimon \ No newline at end of file diff --git a/include/paimon/core/view/view.h b/src/paimon/core/view/view.h similarity index 96% rename from include/paimon/core/view/view.h rename to src/paimon/core/view/view.h index 2a2282a1..b1d70cc3 100644 --- a/include/paimon/core/view/view.h +++ b/src/paimon/core/view/view.h @@ -34,6 +34,9 @@ namespace paimon { /// Interface for view definition. class PAIMON_EXPORT View { public: + View() = default; + virtual ~View() = default; + /// A name to identify this view. virtual std::string Name() = 0; From 9f9ea7ba09ca1229ddfe17023502eaea9247e0ef Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Thu, 5 Feb 2026 15:35:35 +0800 Subject: [PATCH 6/8] address --- include/paimon/catalog/catalog.h | 117 +++++++++++++------------------ src/paimon/core/tag/tag.h | 4 +- 2 files changed, 51 insertions(+), 70 deletions(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 0258b865..beb8773b 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -208,6 +208,7 @@ class PAIMON_EXPORT Catalog { const Identifier& identifier, const std::vector>& partitions) { return Status::NotImplemented("CreatePartitions not implemented"); + ` } /// Drops partitions. @@ -335,7 +336,7 @@ class PAIMON_EXPORT Catalog { // ==================== Version management methods ========================== - /// \brief Whether this catalog supports version management for tables. + /// Whether this catalog supports version management for tables. /// /// If not supported, corresponding methods will return NotImplemented status. /// Affected methods: @@ -352,22 +353,20 @@ class PAIMON_EXPORT Catalog { return false; } - /// \brief Commit the Snapshot for table identified by the given Identifier. + /// Commit the Snapshot for table identified by the given Identifier. /// - /// \param identifier Path of the table - /// \param table_uuid Uuid of the table to avoid wrong commit - /// \param snapshot Snapshot to be committed - /// \param statistics Statistics information of this change - /// \return Success or not - /// \return TableNotExist error if the target does not exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table + /// @param table_uuid Uuid of the table to avoid wrong commit + /// @param snapshot Snapshot to be committed + /// @param statistics Statistics information of this change + /// @return A result containing true if commit succeeded, or an error status. virtual Result CommitSnapshot(const Identifier& identifier, const std::string& table_uuid, const std::shared_ptr& snapshot, const std::vector& statistics) { return Status::NotImplemented("CommitSnapshot not implemented"); } - /// \brief Return the snapshot of table for given version. + /// Return the snapshot of table for given version. /// /// Version parsing order: /// 1. If it is 'EARLIEST', get the earliest snapshot @@ -375,96 +374,83 @@ class PAIMON_EXPORT Catalog { /// 3. If it is a number, get snapshot by snapshot id /// 4. Else try to get snapshot from Tag name /// - /// \param identifier Path of the table - /// \param version Version to snapshot - /// \return The requested snapshot - /// \return TableNotExist error if the target does not exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table + /// @param version Version to snapshot + /// @return A result containing the requested snapshot, or an error status. virtual Result> LoadSnapshot(const Identifier& identifier, const std::string& version) const { return Status::NotImplemented("LoadSnapshot not implemented"); } - /// \brief Rollback table by the given Identifier and instant. + /// Rollback table by the given Identifier and instant. /// - /// \param identifier Path of the table - /// \param instant Like snapshotId or tagName - /// \return TableNotExist error if the table does not exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table + /// @param instant Like snapshotId or tagName + /// @return A status indicating success or failure. virtual Status RollbackTo(const Identifier& identifier, const std::chrono::system_clock::time_point& instant) { return RollbackTo(identifier, instant, std::nullopt); } - /// \brief Rollback table by the given Identifier and instant. + /// Rollback table by the given Identifier and instant. /// - /// \param identifier Path of the table - /// \param instant Like snapshotId or tagName - /// \param from_snapshot Snapshot from, success only occurs when the latest snapshot is this + /// @param identifier Path of the table + /// @param instant Like snapshotId or tagName + /// @param from_snapshot Snapshot from, success only occurs when the latest snapshot is this /// snapshot - /// \return TableNotExist error if the table does not exist - /// \return NotImplemented error if the catalog does not support version management + /// @return A status indicating success or failure. virtual Status RollbackTo(const Identifier& identifier, const std::chrono::system_clock::time_point& instant, const std::optional& from_snapshot) { return Status::NotImplemented("RollbackTo not implemented"); } - /// \brief Create a new branch for this table. + /// Create a new branch for this table. /// /// By default, an empty branch will be created using the latest schema. /// If from_tag is provided, a branch will be created from the tag and the /// data files will be inherited from it. /// - /// \param identifier Path of the table, cannot be system or branch name - /// \param branch The branch name - /// \param from_tag From the tag - /// \return TableNotExist error if the table in identifier doesn't exist - /// \return BranchAlreadyExist error if the branch already exists - /// \return TagNotExist error if the tag doesn't exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system or branch name + /// @param branch The branch name + /// @param from_tag From the tag + /// @return A status indicating success or failure. virtual Status CreateBranch(const Identifier& identifier, const std::string& branch, const std::optional& from_tag = std::nullopt) { return Status::NotImplemented("CreateBranch not implemented"); } - /// \brief Drop the branch for this table. + /// Drop the branch for this table. /// - /// \param identifier Path of the table, cannot be system or branch name - /// \param branch The branch name - /// \return BranchNotExist error if the branch doesn't exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system or branch name + /// @param branch The branch name + /// @return A status indicating success or failure. virtual Status DropBranch(const Identifier& identifier, const std::string& branch) { return Status::NotImplemented("DropBranch not implemented"); } - /// \brief Fast-forward a branch to main branch. + /// Fast-forward a branch to main branch. /// - /// \param identifier Path of the table, cannot be system or branch name - /// \param branch The branch name - /// \return BranchNotExist error if the branch doesn't exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system or branch name + /// @param branch The branch name + /// @return A status indicating success or failure. virtual Status FastForward(const Identifier& identifier, const std::string& branch) { return Status::NotImplemented("FastForward not implemented"); } - /// \brief List all branches of the table. + /// List all branches of the table. /// - /// \param identifier Path of the table, cannot be system or branch name - /// \return TableNotExist error if the table in identifier doesn't exist - /// \return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system or branch name + /// @return A result containing a list of branch names, or an error status. virtual Result> ListBranches(const Identifier& identifier) const { return Status::NotImplemented("ListBranches not implemented"); } /// Get tag for table. /// - /// @param identifier path of the table, cannot be system name. - /// @param tag_name tag name - /// @return Tag containing tag information - /// @return TableNotExist error if the table does not exist - /// @return TagNotExist error if the tag does not exist - /// @return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system name. + /// @param tag_name Tag name + /// @return A result containing the tag information, or an error status. virtual Result> GetTag(const Identifier& identifier, const std::string& tag_name) const { return Status::NotImplemented("GetTag not implemented"); @@ -472,15 +458,12 @@ class PAIMON_EXPORT Catalog { /// Create tag for table. /// - /// @param identifier path of the table, cannot be system name. - /// @param tag_name tag name - /// @param snapshot_id optional snapshot id, if not provided uses latest snapshot - /// @param time_retained optional time retained as string (e.g., "1d", "12h", "30m") - /// @param ignore_if_exists if true, ignore if tag already exists - /// @return TableNotExist error if the table does not exist - /// @return SnapshotNotExist error if the snapshot does not exist - /// @return TagAlreadyExist error if the tag already exists and ignore_if_exists is false - /// @return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system name. + /// @param tag_name Tag name + /// @param snapshot_id Optional snapshot id, if not provided uses latest snapshot + /// @param time_retained Optional time retained as string (e.g., "1d", "12h", "30m") + /// @param ignore_if_exists If true, ignore if tag already exists + /// @return A status indicating success or failure. virtual Status CreateTag(const Identifier& identifier, const std::string& tag_name, const std::optional& snapshot_id, const std::optional& time_retained, @@ -490,11 +473,9 @@ class PAIMON_EXPORT Catalog { /// Delete tag for table. /// - /// @param identifier path of the table, cannot be system name. - /// @param tag_name tag name - /// @return TableNotExist error if the table does not exist - /// @return TagNotExist error if the tag does not exist - /// @return NotImplemented error if the catalog does not support version management + /// @param identifier Path of the table, cannot be system name. + /// @param tag_name Tag name + /// @return A status indicating success or failure. virtual Status DeleteTag(const Identifier& identifier, const std::string& tag_name) { return Status::NotImplemented("DeleteTag not implemented"); } diff --git a/src/paimon/core/tag/tag.h b/src/paimon/core/tag/tag.h index 88fcfa75..e509678b 100644 --- a/src/paimon/core/tag/tag.h +++ b/src/paimon/core/tag/tag.h @@ -1,7 +1,7 @@ /* * Copyright 2026-present Alibaba Inc. * - * Licensed under the Apache License, Version 2.0 (the "License");Expand commentComment on lines R1 to R4Resolved + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * @@ -83,4 +83,4 @@ class Tag : public Snapshot { std::optional tag_create_time_; std::optional tag_time_retained_; }; -} // namespace paimon \ No newline at end of file +} // namespace paimon From a2a22e44f98ffe3154f3ac81c98c74925b6c7fe3 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Thu, 5 Feb 2026 15:37:14 +0800 Subject: [PATCH 7/8] address --- src/paimon/core/table/table.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/paimon/core/table/table.h b/src/paimon/core/table/table.h index dafcda5e..46df411c 100644 --- a/src/paimon/core/table/table.h +++ b/src/paimon/core/table/table.h @@ -27,8 +27,6 @@ #include "paimon/type_fwd.h" #include "paimon/visibility.h" -struct ArrowSchema; - namespace paimon { /// A table provides basic abstraction for table type. From 2c0762df85291542e26edfd02041c74388ca3f52 Mon Sep 17 00:00:00 2001 From: "zhangchaoming.zcm" Date: Thu, 5 Feb 2026 16:23:56 +0800 Subject: [PATCH 8/8] fix --- include/paimon/catalog/catalog.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index beb8773b..47bd66ce 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -208,7 +208,6 @@ class PAIMON_EXPORT Catalog { const Identifier& identifier, const std::vector>& partitions) { return Status::NotImplemented("CreatePartitions not implemented"); - ` } /// Drops partitions.