diff --git a/data/api/pom.xml b/data/api/pom.xml new file mode 100644 index 0000000..b6ed4de --- /dev/null +++ b/data/api/pom.xml @@ -0,0 +1,27 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm.data + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.data.api + Daanse CWM — Data API + + jar + diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/FieldMapping.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/FieldMapping.java new file mode 100644 index 0000000..4dfc483 --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/FieldMapping.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.api; + +import java.util.Optional; +import java.util.function.Function; + +/** + * A single field-level mapping from a source field name to a target feature + * name, with an optional value converter. + */ +public interface FieldMapping { + + /** + * @return the source field name + */ + String sourceFieldName(); + + /** + * @return the target column or feature name + */ + String targetFeatureName(); + + /** + * @return an optional converter from the raw string to the target value + */ + Optional> converter(); +} diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RawRecord.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RawRecord.java new file mode 100644 index 0000000..05d5107 --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RawRecord.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.api; + +import java.util.Map; + +/** + * Represents a single parsed row from a data source as field name to raw string + * value pairs. This is the common data type flowing through CSV pipelines. + */ +public interface RawRecord { + + /** + * @return the field values keyed by field name + */ + Map fields(); + + /** + * @return the line number in the source (1-based) + */ + long lineNumber(); +} diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSink.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSink.java new file mode 100644 index 0000000..06372a0 --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSink.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.api; + +import java.util.concurrent.Flow; + +/** + * A sink that consumes records. Extends {@link Flow.Subscriber} to support + * backpressure signaling. + * + * @param the record type consumed + */ +public interface RecordSink extends Flow.Subscriber { +} diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSource.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSource.java new file mode 100644 index 0000000..09d998d --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/RecordSource.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.api; + +import java.util.concurrent.Flow; + +/** + * A source of records. Extends {@link Flow.Publisher} to support + * backpressure-aware data delivery. + * + * @param the record type published + */ +public interface RecordSource extends Flow.Publisher { +} diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/ValidationResult.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/ValidationResult.java new file mode 100644 index 0000000..1d12421 --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/ValidationResult.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.api; + +import java.util.List; + +/** + * Result of validating CSV headers against a CWM RecordDef's field definitions. + */ +public interface ValidationResult { + + /** + * @return true if all RecordDef fields are present in the CSV header + */ + boolean isValid(); + + /** + * @return field names defined in RecordDef but missing from CSV header + */ + List missingFields(); + + /** + * @return field names present in CSV header but not defined in RecordDef + */ + List extraFields(); + + /** + * @return field names that match between CSV header and RecordDef + */ + List matchedFields(); +} diff --git a/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/package-info.java b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/package-info.java new file mode 100644 index 0000000..e994a5f --- /dev/null +++ b/data/api/src/main/java/org/eclipse/daanse/cwm/data/api/package-info.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("1.0") +package org.eclipse.daanse.cwm.data.api; diff --git a/data/pom.xml b/data/pom.xml new file mode 100644 index 0000000..783430f --- /dev/null +++ b/data/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.data + pom + Daanse CWM Data (aggregator) + + + api + source.csv + sink.jdbc + + diff --git a/data/sink.jdbc/pom.xml b/data/sink.jdbc/pom.xml new file mode 100644 index 0000000..81b2976 --- /dev/null +++ b/data/sink.jdbc/pom.xml @@ -0,0 +1,61 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm.data + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.data.sink.jdbc + Daanse CWM — JDBC Sink + JDBC table sink: writes RawRecord batches into a database table via + PreparedStatement with batch support. Dialect-aware quoting via the jdbc.db + Dialect/DdlGenerator framework. Symmetric counterpart to the CSV source. + + jar + + + + org.eclipse.daanse + org.eclipse.daanse.cwm.data.api + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.api + ${jdbc-db.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.api + ${jdbc-db.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.record + ${jdbc-db.version} + compile + + + org.slf4j + slf4j-api + + + diff --git a/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/DatabaseRecordSink.java b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/DatabaseRecordSink.java new file mode 100644 index 0000000..b912919 --- /dev/null +++ b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/DatabaseRecordSink.java @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.sink.jdbc; + +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.concurrent.Flow; + +import javax.sql.DataSource; + +import org.eclipse.daanse.cwm.data.api.FieldMapping; +import org.eclipse.daanse.cwm.data.api.RawRecord; +import org.eclipse.daanse.cwm.data.api.RecordSink; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnMetaData; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnDefinitionRecord; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnMetaDataRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes {@link RawRecord} items to a database table via PreparedStatement with + * batch support. Maps fields to columns using {@link FieldMapping} definitions. + */ +public class DatabaseRecordSink implements RecordSink { + + private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseRecordSink.class); + + private final DataSource dataSource; + private final Dialect dialect; + private final TableReference targetTable; + private final List fieldMappings; + private final List jdbcTypes; + private final int batchSize; + + private Flow.Subscription subscription; + private Connection connection; + private PreparedStatement preparedStatement; + private int batchCount; + + public DatabaseRecordSink(DataSource dataSource, Dialect dialect, TableReference targetTable, + List fieldMappings, List jdbcTypes, int batchSize) { + this.dataSource = dataSource; + this.dialect = dialect; + this.targetTable = targetTable; + this.fieldMappings = fieldMappings; + this.jdbcTypes = jdbcTypes; + this.batchSize = batchSize; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + try { + connection = dataSource.getConnection(); + connection.setAutoCommit(false); + + List columns = fieldMappings.stream().map(m -> { + ColumnReference ref = new ColumnReference(java.util.Optional.empty(), m.targetFeatureName()); + ColumnMetaData meta = new ColumnMetaDataRecord(java.sql.JDBCType.OTHER, "OTHER", + java.util.OptionalInt.empty(), java.util.OptionalInt.empty(), java.util.OptionalInt.empty(), + ColumnMetaData.Nullability.UNKNOWN, java.util.OptionalInt.empty(), java.util.Optional.empty(), + java.util.Optional.empty(), ColumnMetaData.AutoIncrement.UNKNOWN, + ColumnMetaData.GeneratedColumn.UNKNOWN); + return (ColumnDefinition) new ColumnDefinitionRecord(ref, meta); + }).toList(); + String sql = dialect.ddlGenerator().insertInto(targetTable, columns); + + preparedStatement = connection.prepareStatement(sql); + batchCount = 0; + + subscription.request(Long.MAX_VALUE); + } catch (SQLException e) { + throw new JdbcSinkException("Failed to initialize database sink", e); + } + } + + @Override + public void onNext(RawRecord item) { + try { + int colIndex = 1; + for (int i = 0; i < fieldMappings.size(); i++) { + FieldMapping mapping = fieldMappings.get(i); + String rawValue = item.fields().get(mapping.sourceFieldName()); + JDBCType jdbcType = i < jdbcTypes.size() ? jdbcTypes.get(i) : JDBCType.VARCHAR; + + if (mapping.converter().isPresent()) { + Object converted = rawValue == null ? null : mapping.converter().get().apply(rawValue); + if (converted == null) { + preparedStatement.setNull(colIndex++, jdbcType.getVendorTypeNumber()); + } else { + preparedStatement.setObject(colIndex++, converted, jdbcType.getVendorTypeNumber()); + } + } else { + TypeConverter.setTypedValue(preparedStatement, colIndex++, jdbcType, rawValue); + } + } + + preparedStatement.addBatch(); + preparedStatement.clearParameters(); + batchCount++; + + if (batchCount % batchSize == 0) { + executeBatch(); + } + } catch (SQLException e) { + subscription.cancel(); + closeResources(); + throw new JdbcSinkException("Error writing record at line " + item.lineNumber(), e); + } + } + + @Override + public void onError(Throwable throwable) { + LOGGER.error("Error in database ETL pipeline", throwable); + try { + if (connection != null) { + connection.rollback(); + } + } catch (SQLException e) { + LOGGER.warn("Error rolling back transaction", e); + } + closeResources(); + } + + @Override + public void onComplete() { + try { + if (batchCount % batchSize != 0) { + executeBatch(); + } + connection.commit(); + connection.setAutoCommit(true); + LOGGER.debug("Database import completed for table {}", targetTable.name()); + } catch (SQLException e) { + throw new JdbcSinkException("Error completing database import", e); + } finally { + closeResources(); + } + } + + private void executeBatch() throws SQLException { + long start = System.currentTimeMillis(); + preparedStatement.executeBatch(); + connection.commit(); + LOGGER.debug("Batch executed in {}ms", System.currentTimeMillis() - start); + } + + private void closeResources() { + try { + if (preparedStatement != null) { + preparedStatement.close(); + } + } catch (SQLException e) { + LOGGER.warn("Error closing PreparedStatement", e); + } + try { + if (connection != null) { + connection.close(); + } + } catch (SQLException e) { + LOGGER.warn("Error closing Connection", e); + } + } +} diff --git a/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/JdbcSinkException.java b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/JdbcSinkException.java new file mode 100644 index 0000000..d8b5d5f --- /dev/null +++ b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/JdbcSinkException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.sink.jdbc; + +/** + * Thrown when writing records to the database fails. + */ +public class JdbcSinkException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public JdbcSinkException(String message, Throwable cause) { + super(message, cause); + } + + public JdbcSinkException(String message) { + super(message); + } +} diff --git a/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/TypeConverter.java b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/TypeConverter.java new file mode 100644 index 0000000..1f5904c --- /dev/null +++ b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/TypeConverter.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.sink.jdbc; + +import java.sql.Date; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; + +/** + * Utility for setting typed values on a PreparedStatement from raw String + * input. + */ +public class TypeConverter { + + private TypeConverter() { + } + + /** + * Sets a typed value on a PreparedStatement at the given index. + * + * @param ps the prepared statement + * @param index the parameter index (1-based) + * @param jdbcType the target JDBC type + * @param value the raw string value (may be null) + * @throws SQLException if setting the value fails + */ + public static void setTypedValue(PreparedStatement ps, int index, JDBCType jdbcType, String value) + throws SQLException { + if (value == null) { + ps.setObject(index, null); + return; + } + + switch (jdbcType) { + case BOOLEAN: + ps.setBoolean(index, value.isEmpty() ? Boolean.FALSE : Boolean.valueOf(value)); + break; + case BIGINT: + ps.setLong(index, value.isEmpty() ? 0L : Long.valueOf(value)); + break; + case DATE: + ps.setDate(index, Date.valueOf(value)); + break; + case INTEGER: + ps.setInt(index, value.isEmpty() ? 0 : Integer.valueOf(value)); + break; + case DECIMAL: + case NUMERIC: + case REAL: + case DOUBLE: + case FLOAT: + ps.setDouble(index, value.isEmpty() ? 0.0 : Double.valueOf(value)); + break; + case SMALLINT: + ps.setShort(index, value.isEmpty() ? (short) 0 : Short.valueOf(value)); + break; + case TIMESTAMP: + ps.setTimestamp(index, Timestamp.valueOf(value)); + break; + case TIME: + ps.setTime(index, Time.valueOf(value)); + break; + case VARCHAR: + case CHAR: + case LONGVARCHAR: + case NVARCHAR: + case NCHAR: + ps.setString(index, value); + break; + default: + ps.setString(index, value); + break; + } + } +} diff --git a/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/package-info.java b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/package-info.java new file mode 100644 index 0000000..3519d5c --- /dev/null +++ b/data/sink.jdbc/src/main/java/org/eclipse/daanse/cwm/data/sink/jdbc/package-info.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("1.0") +package org.eclipse.daanse.cwm.data.sink.jdbc; diff --git a/data/source.csv/pom.xml b/data/source.csv/pom.xml new file mode 100644 index 0000000..1de4652 --- /dev/null +++ b/data/source.csv/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm.data + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.data.source.csv + Daanse CWM — CSV Source + Schema-driven CSV source. Reads CSV files described by a CWM + RecordDef (header-only, no SQL-type row) and publishes rows as RawRecord + items via java.util.concurrent.Flow with backpressure. + + jar + + + + org.eclipse.daanse + org.eclipse.daanse.cwm.data.api + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.model.cwm + ${revision} + compile + + + de.siegmar + fastcsv + 3.1.0 + compile + + + org.slf4j + slf4j-api + + + diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvEtlException.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvEtlException.java new file mode 100644 index 0000000..e0a9e17 --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvEtlException.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv; + +/** + * Thrown when reading or parsing a CSV source fails. + */ +public class CsvEtlException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public CsvEtlException(String message, Throwable cause) { + super(message, cause); + } + + public CsvEtlException(String message) { + super(message); + } +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisher.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisher.java new file mode 100644 index 0000000..8c6742e --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisher.java @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.Feature; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordDef; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordFile; +import org.eclipse.daanse.cwm.data.api.RawRecord; +import org.eclipse.daanse.cwm.data.api.RecordSource; +import org.eclipse.daanse.cwm.data.api.ValidationResult; +import org.eclipse.daanse.cwm.data.source.csv.record.RawRecordR; +import org.eclipse.emf.common.util.EList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.siegmar.fastcsv.reader.CloseableIterator; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.CsvRecord; +import de.siegmar.fastcsv.reader.NamedCsvRecord; + +/** + * Reads CSV files according to CWM RecordDef/RecordFile definitions and + * publishes rows as {@link RawRecord} items with backpressure support. + */ +public class CsvRecordPublisher implements RecordSource { + + private static final Logger LOGGER = LoggerFactory.getLogger(CsvRecordPublisher.class); + + private final Path csvFilePath; + private final RecordFile recordFile; + private final RecordDef recordDef; + + public CsvRecordPublisher(Path csvFilePath, RecordFile recordFile, RecordDef recordDef) { + this.csvFilePath = csvFilePath; + this.recordFile = recordFile; + this.recordDef = recordDef; + } + + @Override + public void subscribe(Flow.Subscriber subscriber) { + CsvSubscription subscription = new CsvSubscription(subscriber, csvFilePath, recordFile, recordDef); + subscriber.onSubscribe(subscription); + } + + private static class CsvSubscription implements Flow.Subscription { + + private final Flow.Subscriber subscriber; + private final Path csvFilePath; + private final RecordFile recordFile; + private final RecordDef recordDef; + private final AtomicLong demand = new AtomicLong(0); + private final AtomicBoolean cancelled = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); + + private CloseableIterator iterator; + private List fieldNames; + private long lineCounter; + + CsvSubscription(Flow.Subscriber subscriber, Path csvFilePath, RecordFile recordFile, + RecordDef recordDef) { + this.subscriber = subscriber; + this.csvFilePath = csvFilePath; + this.recordFile = recordFile; + this.recordDef = recordDef; + } + + @Override + public void request(long n) { + if (n <= 0) { + subscriber.onError(new IllegalArgumentException("request count must be positive: " + n)); + return; + } + if (cancelled.get()) { + return; + } + + demand.addAndGet(n); + + if (started.compareAndSet(false, true)) { + try { + initialize(); + } catch (Exception e) { + subscriber.onError(e); + return; + } + } + + drain(); + } + + @Override + public void cancel() { + if (cancelled.compareAndSet(false, true)) { + closeResources(); + } + } + + private void initialize() throws IOException { + char fieldSeparator = ','; + char quoteCharacter = '"'; + + String delimiter = recordDef.getFieldDelimiter(); + if (delimiter != null && !delimiter.isEmpty()) { + fieldSeparator = delimiter.charAt(0); + } + String textDelim = recordDef.getTextDelimiter(); + if (textDelim != null && !textDelim.isEmpty()) { + quoteCharacter = textDelim.charAt(0); + } + + CsvReader.CsvReaderBuilder builder = CsvReader.builder().fieldSeparator(fieldSeparator) + .quoteCharacter(quoteCharacter).skipEmptyLines(true); + + boolean isSelfDescribing = recordFile.isIsSelfDescribing(); + + EList features = recordDef.getFeature(); + List recordDefFieldNames = new ArrayList<>(); + for (Feature feature : features) { + recordDefFieldNames.add(feature.getName()); + } + + if (isSelfDescribing) { + CloseableIterator namedIterator = builder.ofNamedCsvRecord(csvFilePath).iterator(); + if (!namedIterator.hasNext()) { + namedIterator.close(); + throw new IllegalStateException("CSV file is empty, no header found: " + csvFilePath); + } + + NamedCsvRecord firstRecord = namedIterator.next(); + List csvHeaders = firstRecord.getHeader(); + + ValidationResult validation = HeaderValidator.validate(csvHeaders, recordDef); + if (!validation.isValid()) { + namedIterator.close(); + throw new IllegalStateException("CSV header validation failed. Missing fields: " + + validation.missingFields() + ", Extra fields: " + validation.extraFields()); + } + + LOGGER.debug("CSV header validated successfully for {}", csvFilePath); + fieldNames = csvHeaders; + + namedIterator.close(); + + iterator = builder.ofCsvRecord(csvFilePath).iterator(); + lineCounter = 0; + + if (iterator.hasNext()) { + iterator.next(); + lineCounter = 1; + } + + long skipRecordsVal = recordFile.getSkipRecords(); + int skip = (int) skipRecordsVal; + for (int i = 0; i < skip && iterator.hasNext(); i++) { + iterator.next(); + lineCounter++; + } + } else { + fieldNames = recordDefFieldNames; + + iterator = builder.ofCsvRecord(csvFilePath).iterator(); + lineCounter = 0; + + long skipRecordsVal = recordFile.getSkipRecords(); + int skip = (int) skipRecordsVal; + for (int i = 0; i < skip && iterator.hasNext(); i++) { + iterator.next(); + lineCounter++; + } + } + } + + private void drain() { + while (demand.get() > 0 && !cancelled.get()) { + if (iterator == null || !iterator.hasNext()) { + if (!cancelled.getAndSet(true)) { + closeResources(); + subscriber.onComplete(); + } + return; + } + + CsvRecord csvRecord = iterator.next(); + lineCounter++; + demand.decrementAndGet(); + + Map fields = new LinkedHashMap<>(); + for (int i = 0; i < fieldNames.size() && i < csvRecord.getFieldCount(); i++) { + fields.put(fieldNames.get(i), csvRecord.getField(i)); + } + + RawRecord rawRecord = new RawRecordR(Map.copyOf(fields), lineCounter); + + try { + subscriber.onNext(rawRecord); + } catch (Exception e) { + if (!cancelled.getAndSet(true)) { + closeResources(); + subscriber.onError(e); + } + return; + } + } + } + + private void closeResources() { + try { + if (iterator != null) { + iterator.close(); + } + } catch (Exception e) { + LOGGER.warn("Error closing CSV iterator", e); + } + } + } +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisherFactory.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisherFactory.java new file mode 100644 index 0000000..ed395b4 --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/CsvRecordPublisherFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv; + +import java.nio.file.Path; + +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordDef; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordFile; +import org.eclipse.daanse.cwm.data.api.RawRecord; +import org.eclipse.daanse.cwm.data.api.RecordSource; +import org.osgi.service.component.annotations.Component; + +/** + * Factory service for creating {@link CsvRecordPublisher} instances configured + * from CWM RecordFile and RecordDef models. + */ +@Component(service = CsvRecordPublisherFactory.class) +public class CsvRecordPublisherFactory { + + public RecordSource create(Path csvFile, RecordFile recordFile, RecordDef recordDef) { + return new CsvRecordPublisher(csvFile, recordFile, recordDef); + } +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/HeaderValidator.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/HeaderValidator.java new file mode 100644 index 0000000..bc7c70f --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/HeaderValidator.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.Feature; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordDef; +import org.eclipse.daanse.cwm.data.api.ValidationResult; +import org.eclipse.daanse.cwm.data.source.csv.record.ValidationResultR; +import org.eclipse.emf.common.util.EList; + +/** + * Validates CSV header fields against a CWM RecordDef's field definitions. + */ +public class HeaderValidator { + + private HeaderValidator() { + } + + /** + * Checks the CSV header against the fields declared in {@code recordDef}, + * reporting matched, missing and extra field names. + */ + public static ValidationResult validate(List csvHeaders, RecordDef recordDef) { + Set headerSet = new LinkedHashSet<>(csvHeaders); + + EList features = recordDef.getFeature(); + Set expectedFields = new LinkedHashSet<>(); + for (Feature feature : features) { + expectedFields.add(feature.getName()); + } + + List matched = new ArrayList<>(); + List missing = new ArrayList<>(); + List extra = new ArrayList<>(); + + for (String expected : expectedFields) { + if (headerSet.contains(expected)) { + matched.add(expected); + } else { + missing.add(expected); + } + } + + for (String header : csvHeaders) { + if (!expectedFields.contains(header)) { + extra.add(header); + } + } + + boolean isValid = missing.isEmpty(); + return new ValidationResultR(isValid, List.copyOf(missing), List.copyOf(extra), List.copyOf(matched)); + } +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/package-info.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/package-info.java new file mode 100644 index 0000000..cd9d498 --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/package-info.java @@ -0,0 +1,17 @@ +/* +* Copyright (c) 2024 Contributors to the Eclipse Foundation. +* +* This program and the accompanying materials are made +* available under the terms of the Eclipse Public License 2.0 +* which is available at https://www.eclipse.org/legal/epl-2.0/ +* +* SPDX-License-Identifier: EPL-2.0 +* +* Contributors: +* SmartCity Jena - initial +* Stefan Bischof (bipolis.org) - initial +*/ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("0.0.1") + +package org.eclipse.daanse.cwm.data.source.csv; diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/FieldMappingR.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/FieldMappingR.java new file mode 100644 index 0000000..e2a32df --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/FieldMappingR.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv.record; + +import java.util.Optional; +import java.util.function.Function; + +import org.eclipse.daanse.cwm.data.api.FieldMapping; + +/** + * Record implementation of {@link FieldMapping}. + */ +public record FieldMappingR(String sourceFieldName, String targetFeatureName, + Optional> converter) implements FieldMapping { +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/RawRecordR.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/RawRecordR.java new file mode 100644 index 0000000..6e16b75 --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/RawRecordR.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv.record; + +import java.util.Map; + +import org.eclipse.daanse.cwm.data.api.RawRecord; + +/** + * Record implementation of {@link RawRecord}. + */ +public record RawRecordR(Map fields, long lineNumber) implements RawRecord { +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/ValidationResultR.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/ValidationResultR.java new file mode 100644 index 0000000..3fba0ad --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/ValidationResultR.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.data.source.csv.record; + +import java.util.List; + +import org.eclipse.daanse.cwm.data.api.ValidationResult; + +/** + * Record implementation of {@link ValidationResult}. + */ +public record ValidationResultR(boolean isValid, List missingFields, List extraFields, + List matchedFields) implements ValidationResult { +} diff --git a/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/package-info.java b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/package-info.java new file mode 100644 index 0000000..21542c7 --- /dev/null +++ b/data/source.csv/src/main/java/org/eclipse/daanse/cwm/data/source/csv/record/package-info.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("1.0") +package org.eclipse.daanse.cwm.data.source.csv.record; diff --git a/model/cwm/model/cwm.ecore b/model/cwm/model/cwm.ecore index 6d1c795..0838828 100644 --- a/model/cwm/model/cwm.ecore +++ b/model/cwm/model/cwm.ecore @@ -51,6 +51,7 @@
+
@@ -1623,15 +1624,15 @@
-
+
-
+
-
+
@@ -2873,6 +2874,7 @@
+
@@ -3341,6 +3343,23 @@
+ +
+ + +
+
+ + +
+
+
+
+
+
+
+
+ @@ -3600,7 +3619,7 @@
-
+
@@ -3746,15 +3765,15 @@
-
+
-
+
-
+
@@ -3867,7 +3886,7 @@
-
+
@@ -5366,6 +5385,7 @@
+
@@ -5638,16 +5658,21 @@ -
+
+
+
+
+
+
@@ -5673,16 +5698,21 @@ -
+
+
+
+
+
+
diff --git a/pom.xml b/pom.xml index d77c3de..b7df7da 100644 --- a/pom.xml +++ b/pom.xml @@ -50,13 +50,12 @@ model util + data + resource + testkit - diff --git a/resource/pom.xml b/resource/pom.xml new file mode 100644 index 0000000..d3a0491 --- /dev/null +++ b/resource/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.resource + pom + Daanse CWM Resource (aggregator) + Aggregator for tooling that supports the CWM resource layers + (relational, ...). + + + relational + + diff --git a/resource/relational/ddl/pom.xml b/resource/relational/ddl/pom.xml new file mode 100644 index 0000000..c26e519 --- /dev/null +++ b/resource/relational/ddl/pom.xml @@ -0,0 +1,234 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm.resource.relational + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.resource.relational.ddl + Daanse CWM Relational DDL Generation + Generate dialect-specific DDL from a CWM relational Schema: + CwmDdlGenerator emits ordered CREATE/DROP statements via the jdbc.db + dialect, using the SQL-99 type helpers in cwm.util. + + + org.eclipse.daanse + org.eclipse.daanse.cwm.model.cwm + ${project.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.util + ${project.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.api + ${jdbc-db.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.api + ${jdbc-db.version} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.record + ${jdbc-db.version} + compile + + + org.eclipse.emf + org.eclipse.emf.common + compile + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.impl + ${jdbc-db.version} + test + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.db.postgresql + ${jdbc-db.version} + test + + + org.postgresql + postgresql + 42.7.3 + test + + + org.testcontainers + postgresql + 1.19.7 + test + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.datasource.testkit.postgresql + 0.0.1-SNAPSHOT + test + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.db.h2 + ${jdbc-db.version} + test + + + com.h2database + h2 + 2.2.224 + test + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.datasource.testkit.h2 + 0.0.1-SNAPSHOT + test + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.db.mariadb + ${jdbc-db.version} + test + + + org.mariadb.jdbc + mariadb-java-client + 3.3.3 + test + + + org.testcontainers + mariadb + 1.19.7 + test + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.datasource.testkit.mariadb + 0.0.1-SNAPSHOT + test + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.db.mssqlserver + ${jdbc-db.version} + test + + + com.microsoft.sqlserver + mssql-jdbc + 12.6.1.jre11 + test + + + org.testcontainers + mssqlserver + 1.19.7 + test + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.datasource.testkit.mssqlserver + 0.0.1-SNAPSHOT + test + + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.db.oracle + ${jdbc-db.version} + test + + + com.oracle.database.jdbc + ojdbc11 + 23.3.0.23.09 + test + + + org.testcontainers + oracle-free + 1.19.7 + test + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.datasource.testkit.oracle + 0.0.1-SNAPSHOT + test + + + + org.testcontainers + junit-jupiter + 1.19.7 + test + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + + maven-jar-plugin + + + + test-jar + + + + + + maven-surefire-plugin + + + ${env.DOCKER_HOST} + + + + + + diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/CwmSchemaMapper.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/CwmSchemaMapper.java new file mode 100644 index 0000000..de66cc1 --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/CwmSchemaMapper.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.api; + +import java.sql.JDBCType; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import org.eclipse.daanse.cwm.util.resource.relational.SqlSimpleTypes; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.Classifier; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.Feature; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.StructuralFeature; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.NamedColumnSet; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.PrimaryKey; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLDataType; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLSimpleType; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.NullableType; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnMetaData; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnDefinitionRecord; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnMetaDataRecord; +import org.eclipse.daanse.jdbc.db.record.schema.PrimaryKeyRecord; + +/** + * Pure, stateless bridge from individual CWM elements ({@link Column}, + * {@link PrimaryKey}) to the jdbc.db {@code ColumnDefinition} / + * {@code ColumnMetaData} / {@code PrimaryKey} records that + * {@link Dialect#ddlGenerator()} consumes. Every method takes an explicit + * {@link TableReference} — schema/catalog derivation and DDL emission live in + * the {@link DdlGenerator}. SQL type-code mapping is delegated to + * {@link org.eclipse.daanse.cwm.util.resource.relational.SqlSimpleTypes}. + */ +public final class CwmSchemaMapper { + + private CwmSchemaMapper() { + } + + public static List columnDefinitions(TableReference table, NamedColumnSet cwmTable) { + List out = new ArrayList<>(); + for (Feature f : cwmTable.getFeature()) { + if (!(f instanceof Column col)) { + continue; + } + ColumnReference colRef = new ColumnReference(Optional.of(table), col.getName()); + out.add(new ColumnDefinitionRecord(colRef, columnMetaData(col))); + } + return out; + } + + public static org.eclipse.daanse.jdbc.db.api.schema.PrimaryKey primaryKey(TableReference table, PrimaryKey cwmPk) { + List cols = new ArrayList<>(); + for (StructuralFeature sf : cwmPk.getFeature()) { + cols.add(new ColumnReference(Optional.of(table), sf.getName())); + } + return new PrimaryKeyRecord(table, cols, Optional.ofNullable(cwmPk.getName())); + } + + public static ColumnMetaData columnMetaData(Column col) { + Classifier type = col.getType(); + SQLDataType sqlType = type instanceof SQLDataType sdt ? sdt : null; + + JDBCType jdbcType = SqlSimpleTypes.toJdbcType(sqlType); + String typeName; + if (sqlType != null && sqlType.getName() != null && !sqlType.getName().isBlank()) { + typeName = sqlType.getName(); + } else if (jdbcType != null) { + typeName = jdbcType.getName(); + } else { + typeName = ""; + } + + OptionalInt size = OptionalInt.empty(); + OptionalInt scale = OptionalInt.empty(); + if (sqlType instanceof SQLSimpleType simple) { + long maxLen = simple.getCharacterMaximumLength(); + long numPrec = simple.getNumericPrecision(); + long numScale = simple.getNumericScale(); + if (maxLen > 0) { + size = OptionalInt.of((int) maxLen); + } + if (numPrec > 0) { + size = OptionalInt.of((int) numPrec); + } + if (numScale != 0) { + scale = OptionalInt.of((int) numScale); + } + } + long colPrec = col.getPrecision(); + long colScale = col.getScale(); + long colLen = col.getLength(); + if (colLen > 0) { + size = OptionalInt.of((int) colLen); + } + if (colPrec > 0) { + size = OptionalInt.of((int) colPrec); + } + if (colScale != 0) { + scale = OptionalInt.of((int) colScale); + } + + Optional columnDefault = Optional.ofNullable(col.getInitialValue()).map(e -> e.getBody()) + .filter(b -> b != null && !b.isBlank()); + + return new ColumnMetaDataRecord(jdbcType, typeName, size, scale, OptionalInt.empty(), + toJdbcNullability(col.getIsNullable()), OptionalInt.empty(), Optional.empty(), columnDefault, + ColumnMetaData.AutoIncrement.UNKNOWN, ColumnMetaData.GeneratedColumn.UNKNOWN); + } + + /** Map a CWM {@link NullableType} to the jdbc.db {@link ColumnMetaData.Nullability}. */ + private static ColumnMetaData.Nullability toJdbcNullability(NullableType n) { + if (n == null) { + return ColumnMetaData.Nullability.UNKNOWN; + } + return switch (n) { + case COLUMN_NO_NULLS -> ColumnMetaData.Nullability.NO_NULLS; + case COLUMN_NULLABLE -> ColumnMetaData.Nullability.NULLABLE; + case COLUMN_NULLABLE_UNKNOWN -> ColumnMetaData.Nullability.UNKNOWN; + }; + } +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGenerator.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGenerator.java new file mode 100644 index 0000000..c0069d6 --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGenerator.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.api; + +import java.util.List; +import java.util.Set; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.NamedColumnSet; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; + +/** + * Serialises a CWM relational {@link Schema} to an ordered list of dialect- + * specific DDL statements. An instance pairs a dialect with immutable + * {@link DdlSettings}, so it is reusable and thread-safe. Obtain one from a + * {@link DdlGeneratorFactory}. + */ +public interface DdlGenerator { + + /** The settings this generator was created with. */ + DdlSettings settings(); + + /** Serialise all features of {@code schema}. */ + List createSchema(Schema schema); + + /** Serialise {@code schema}, emitting only the entities in {@code features}. */ + List createSchema(Schema schema, Set features); + + /** Reverse of {@link #createSchema(Schema)} — ordered {@code DROP}s. */ + List dropSchema(Schema schema); + + /** Reverse of {@link #createSchema(Schema, Set)} for the given {@code features}. */ + List dropSchema(Schema schema, Set features); + + /** {@code CREATE TABLE} for a single table (unqualified). */ + String createTable(Table table); + + /** {@code CREATE TABLE} for a table qualified by {@code schema}. */ + String createTable(Schema schema, Table table); + + /** The dialect {@link TableReference} for a CWM table/view. */ + TableReference tableReference(NamedColumnSet table); + + /** The dialect column definitions for a CWM table/view. */ + List columnDefinitions(NamedColumnSet table); +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGeneratorFactory.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGeneratorFactory.java new file mode 100644 index 0000000..009dc05 --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlGeneratorFactory.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.api; + +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** Creates {@link DdlGenerator}s for a given dialect. Registered as an OSGi service. */ +public interface DdlGeneratorFactory { + + /** Generator for {@code dialect} with default {@link DdlSettings}. */ + DdlGenerator create(Dialect dialect); + + /** Generator for {@code dialect} with the given {@code settings}. */ + DdlGenerator create(Dialect dialect, DdlSettings settings); +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlSettings.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlSettings.java new file mode 100644 index 0000000..06f2786 --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/DdlSettings.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.api; + +/** + * Instance configuration for a {@link DdlGenerator}. Immutable; use + * {@link #defaults()} plus the {@code with*} methods to derive a variant. + * + *
    + *
  • {@code includeSchema} — qualify table references as {@code schema.table} + * ({@code true}, default) or emit bare {@code table} names ({@code false}, e.g. + * for connection-scoped databases). The catalog is never emitted.
    • + *
  • {@code ifNotExists} — emit {@code CREATE ... IF NOT EXISTS} / + * {@code DROP ... IF EXISTS} where the dialect supports it.
    • + *
  • {@code cascade} — append {@code CASCADE} to table/schema drops (ignored by + * dialects that don't support it).
    • + *
+ */ +public record DdlSettings(boolean includeSchema, boolean ifNotExists, boolean cascade) { + + public static DdlSettings defaults() { + return new DdlSettings(true, true, false); + } + + public DdlSettings withIncludeSchema(boolean value) { + return new DdlSettings(value, ifNotExists, cascade); + } + + public DdlSettings withIfNotExists(boolean value) { + return new DdlSettings(includeSchema, value, cascade); + } + + public DdlSettings withCascade(boolean value) { + return new DdlSettings(includeSchema, ifNotExists, value); + } +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/Feature.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/Feature.java new file mode 100644 index 0000000..7b08f2d --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/Feature.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.api; + +import java.util.EnumSet; +import java.util.Set; + +/** Schema entity kinds a {@link DdlGenerator} can emit or skip. */ +public enum Feature { + SCHEMA, TABLE, PRIMARY_KEY, UNIQUE, CHECK, INDEX, FOREIGN_KEY, VIEW, TRIGGER; + + /** All features. */ + public static final Set ALL = EnumSet.allOf(Feature.class); +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/package-info.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/package-info.java new file mode 100644 index 0000000..7bd350d --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/api/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +/** + * CWM → SQL DDL generation API. A {@link DdlGenerator} (obtained from a + * {@link DdlGeneratorFactory}) emits ordered {@code CREATE}/{@code DROP} + * statements for a CWM relational {@code Schema} via the jdbc.db dialect, + * configured by {@link DdlSettings} and scoped per {@link Feature}. + * {@link CwmSchemaMapper} is the pure element-to-record bridge it builds on. + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("0.0.1") +package org.eclipse.daanse.cwm.resource.relational.ddl.api; diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorFactoryImpl.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorFactoryImpl.java new file mode 100644 index 0000000..585bb8e --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorFactoryImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlGenerator; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlGeneratorFactory; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlSettings; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.osgi.service.component.annotations.Component; + +/** Default {@link DdlGeneratorFactory}, producing {@link DdlGeneratorImpl}s. */ +@Component(service = DdlGeneratorFactory.class) +public class DdlGeneratorFactoryImpl implements DdlGeneratorFactory { + + @Override + public DdlGenerator create(Dialect dialect) { + return new DdlGeneratorImpl(dialect); + } + + @Override + public DdlGenerator create(Dialect dialect, DdlSettings settings) { + return new DdlGeneratorImpl(dialect, settings); + } +} diff --git a/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorImpl.java b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorImpl.java new file mode 100644 index 0000000..05315f8 --- /dev/null +++ b/resource/relational/ddl/src/main/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/DdlGeneratorImpl.java @@ -0,0 +1,561 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena - initial + * Stefan Bischof (bipolis.org) - initial + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.CwmSchemaMapper; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlGenerator; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlSettings; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.Feature; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.ModelElement; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.StructuralFeature; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.CheckConstraint; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.ForeignKey; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.NamedColumnSet; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.PrimaryKey; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLIndex; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Trigger; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.UniqueConstraint; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.View; +import org.eclipse.daanse.cwm.util.objectmodel.core.Namespaces; +import org.eclipse.daanse.cwm.util.resource.relational.ColumnSets; +import org.eclipse.daanse.cwm.util.resource.relational.ForeignKeys; +import org.eclipse.daanse.cwm.util.resource.relational.Indexes; +import org.eclipse.daanse.cwm.util.resource.relational.NamedColumnSets; +import org.eclipse.daanse.cwm.util.resource.relational.PrimaryKeys; +import org.eclipse.daanse.cwm.util.resource.relational.Schemas; +import org.eclipse.daanse.cwm.util.resource.relational.Tables; +import org.eclipse.daanse.cwm.util.resource.relational.UniqueConstraints; +import org.eclipse.daanse.cwm.util.resource.relational.Views; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.SchemaReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.api.schema.Trigger.TriggerEvent; +import org.eclipse.daanse.jdbc.db.api.schema.Trigger.TriggerScope; +import org.eclipse.daanse.jdbc.db.api.schema.Trigger.TriggerTiming; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** + * Serialises a CWM relational {@link Schema} to an ordered list of dialect- + * specific DDL statements via {@link Dialect#ddlGenerator()}. No JDBC. + * + *

+ * An instance pairs a {@link Dialect} with immutable {@link DdlSettings}, so it + * is reusable and thread-safe. Table references are schema-qualified by default + * (toggle via {@link DdlSettings#includeSchema()}); the catalog is never + * emitted. + * + *

+ * Create order: schema, table (+PK), unique, check, index, foreign key, view, + * trigger. {@link #dropSchema} uses the reverse. + */ +public final class DdlGeneratorImpl implements DdlGenerator { + + private final Dialect dialect; + private final DdlSettings settings; + + public DdlGeneratorImpl(Dialect dialect) { + this(dialect, DdlSettings.defaults()); + } + + public DdlGeneratorImpl(Dialect dialect, DdlSettings settings) { + if (dialect == null) { + throw new IllegalArgumentException("dialect must not be null"); + } + this.dialect = dialect; + this.settings = settings == null ? DdlSettings.defaults() : settings; + } + + public DdlSettings settings() { + return settings; + } + + /** Serialise all features of {@code schema}. */ + public List createSchema(Schema schema) { + return createSchema(schema, Feature.ALL); + } + + /** + * Serialise {@code schema} into ordered {@code CREATE} statements, emitting + * only entities whose {@link Feature} is in {@code features}. Returned + * statements have no trailing semicolons. + */ + public List createSchema(Schema schema, Set features) { + if (schema == null) { + throw new IllegalArgumentException("schema must not be null"); + } + if (features == null || features.isEmpty()) { + return List.of(); + } + List out = new ArrayList<>(); + List tables = Schemas.tables(schema); + List views = Schemas.views(schema); + + if (features.contains(Feature.SCHEMA) && schema.getName() != null && !schema.getName().isBlank()) { + out.add(dialect.ddlGenerator().createSchema(schema.getName(), true)); + } + + if (features.contains(Feature.TABLE)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + List cols = CwmSchemaMapper.columnDefinitions(tref, table); + org.eclipse.daanse.jdbc.db.api.schema.PrimaryKey pkRef = null; + if (features.contains(Feature.PRIMARY_KEY)) { + pkRef = primaryKeyRef(tref, table); + } + out.add(dialect.ddlGenerator().createTable(tref, cols, pkRef, settings.ifNotExists())); + } + } + + if (features.contains(Feature.UNIQUE)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + for (UniqueConstraint uc : Tables.findUniqueConstraints(table)) { + if (uc instanceof PrimaryKey) { + continue; + } + List colNames = UniqueConstraints.columns(uc).stream().map(Column::getName).toList(); + if (colNames.isEmpty()) { + continue; + } + String name = nameOrDefault(uc, "uc_" + table.getName()); + out.add(dialect.ddlGenerator().addUniqueConstraint(tref, name, colNames)); + } + } + } + + if (features.contains(Feature.CHECK)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (CheckConstraint cc : checkConstraintsOf(table)) { + String body = cc.getBody() == null ? null : cc.getBody().getBody(); + if (body == null || body.isBlank()) { + continue; + } + String name = nameOrDefault(cc, "ck_" + table.getName() + "_" + (++idx)); + out.add(dialect.ddlGenerator().addCheckConstraint(tref, name, body)); + } + } + } + + if (features.contains(Feature.INDEX)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (SQLIndex i : Indexes.spanning(table)) { + List colNames = indexColumnNames(i); + if (colNames.isEmpty()) { + continue; + } + String name = nameOrDefault(i, "idx_" + table.getName() + "_" + (++idx)); + out.add(dialect.ddlGenerator().createIndex(name, tref, colNames, i.isIsUnique(), true)); + } + } + } + + if (features.contains(Feature.FOREIGN_KEY)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (ForeignKey fk : Tables.findForeignKeys(table)) { + Optional
refTable = ForeignKeys.targetTable(fk); + if (refTable.isEmpty()) { + continue; + } + List fkCols = ForeignKeys.columns(fk).stream().map(Column::getName).toList(); + if (fkCols.isEmpty()) { + continue; + } + List refCols = refColumnNames(fk); + if (refCols.isEmpty()) { + continue; + } + Schema refSchema = NamedColumnSets.findSchema(refTable.get()).orElse(null); + TableReference refTref = tableRef(refSchema, refTable.get(), TableReference.TYPE_TABLE); + String name = nameOrDefault(fk, "fk_" + table.getName() + "_" + (++idx)); + String onDelete = fk.getDeleteRule() == null ? null + : referentialAction(fk.getDeleteRule().getName()); + String onUpdate = fk.getUpdateRule() == null ? null + : referentialAction(fk.getUpdateRule().getName()); + out.add(dialect.ddlGenerator().addForeignKeyConstraint(tref, name, fkCols, refTref, refCols, + onDelete, onUpdate)); + } + } + } + + if (features.contains(Feature.VIEW)) { + for (View view : views) { + Optional body = Views.queryBody(view); + if (body.isEmpty() || body.get().isBlank()) { + continue; + } + TableReference vref = tableRef(schema, view, TableReference.TYPE_VIEW); + out.add(dialect.ddlGenerator().createView(vref, body.get(), false)); + } + } + + if (features.contains(Feature.TRIGGER)) { + record TT(Table table, Trigger trigger, String name, String body, TriggerTiming timing, TriggerEvent event, + TriggerScope scope, String when) { + } + List all = new ArrayList<>(); + for (Table table : tables) { + for (Trigger t : table.getTrigger()) { + String body = triggerBody(t); + if (body == null) { + continue; + } + TriggerTiming timing = triggerTiming(t); + TriggerEvent event = triggerEvent(t); + if (timing == null || event == null) { + continue; + } + TriggerScope scope = triggerScope(t); + String name = nameOrDefault(t, "trg_" + table.getName()); + String when = triggerWhen(t); + all.add(new TT(table, t, name, body, timing, event, scope, when)); + } + } + + // Group identical bodies so they share one CREATE PROCEDURE/FUNCTION. + Map procNameByBody = new LinkedHashMap<>(); + for (TT tt : all) { + procNameByBody.computeIfAbsent(tt.body(), b -> tt.name() + "_fn"); + } + Map emittedProcs = new LinkedHashMap<>(); + for (Map.Entry e : procNameByBody.entrySet()) { + Optional procSql = dialect.ddlGenerator().createTriggerProcedure(e.getValue(), schema.getName(), + e.getKey()); + if (procSql.isPresent()) { + out.add(procSql.get()); + emittedProcs.put(e.getKey(), e.getValue()); + } + } + for (TT tt : all) { + TableReference tref = tableRef(schema, tt.table(), TableReference.TYPE_TABLE); + String procName = emittedProcs.get(tt.body()); + if (procName != null) { + out.add(dialect.ddlGenerator().createTriggerUsingProcedure(tt.name(), schema.getName(), tt.timing(), + tt.event(), tref, tt.scope(), tt.when(), procName)); + } else { + out.add(dialect.ddlGenerator().createTrigger(tt.name(), tt.timing(), tt.event(), tref, tt.scope(), + tt.when(), tt.body())); + } + } + } + return out; + } + + /** Drop all features of {@code schema}. */ + public List dropSchema(Schema schema) { + return dropSchema(schema, Feature.ALL); + } + + /** + * Drop in reverse of create order. With {@link DdlSettings#cascade()}, + * table and schema drops append SQL-99 {@code CASCADE}; MySQL/MariaDB and + * SQL Server ignore it. + */ + public List dropSchema(Schema schema, Set features) { + if (schema == null) { + throw new IllegalArgumentException("schema must not be null"); + } + if (features == null || features.isEmpty()) { + return List.of(); + } + boolean cascade = settings.cascade(); + List out = new ArrayList<>(); + List
tables = Schemas.tables(schema); + List views = Schemas.views(schema); + + if (features.contains(Feature.TRIGGER)) { + Map bodyToProc = new LinkedHashMap<>(); + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + for (Trigger t : table.getTrigger()) { + String body = triggerBody(t); + if (body == null) { + continue; + } + String name = nameOrDefault(t, "trg_" + table.getName()); + bodyToProc.computeIfAbsent(body, b -> name + "_fn"); + out.addAll(dialect.ddlGenerator().dropTriggerOnTable(name, tref, true)); + } + } + for (String procName : bodyToProc.values()) { + Optional drop = dialect.ddlGenerator().dropProcedure(procName, schema.getName(), true); + if (drop.isPresent() && !out.contains(drop.get())) { + out.add(drop.get()); + } + } + } + + if (features.contains(Feature.VIEW)) { + for (View view : views) { + TableReference vref = tableRef(schema, view, TableReference.TYPE_VIEW); + out.add(dialect.ddlGenerator().dropView(vref, true)); + } + } + + if (features.contains(Feature.FOREIGN_KEY)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (ForeignKey fk : Tables.findForeignKeys(table)) { + String name = nameOrDefault(fk, "fk_" + table.getName() + "_" + (++idx)); + out.add(dialect.ddlGenerator().dropConstraint(tref, name, true)); + } + } + } + + if (features.contains(Feature.INDEX)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (SQLIndex i : Indexes.spanning(table)) { + String name = nameOrDefault(i, "idx_" + table.getName() + "_" + (++idx)); + out.add(dialect.ddlGenerator().dropIndex(name, tref, true)); + } + } + } + + if (features.contains(Feature.CHECK)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + int idx = 0; + for (CheckConstraint cc : checkConstraintsOf(table)) { + String name = nameOrDefault(cc, "ck_" + table.getName() + "_" + (++idx)); + out.add(dialect.ddlGenerator().dropConstraint(tref, name, true)); + } + } + } + + if (features.contains(Feature.UNIQUE)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + for (UniqueConstraint uc : Tables.findUniqueConstraints(table)) { + if (uc instanceof PrimaryKey) { + continue; + } + String name = nameOrDefault(uc, "uc_" + table.getName()); + out.add(dialect.ddlGenerator().dropConstraint(tref, name, true)); + } + } + } + + if (features.contains(Feature.TABLE)) { + for (Table table : tables) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + out.add(dialect.ddlGenerator().dropTable(tref, true, cascade)); + } + } + + if (features.contains(Feature.SCHEMA) && schema.getName() != null && !schema.getName().isBlank()) { + out.add(dialect.ddlGenerator().dropSchema(schema.getName(), true, cascade)); + } + return out; + } + + // -------------------- element-level (container-derived) -------------------- + + /** + * Table reference for {@code table}, deriving its {@link Schema} from the CWM + * namespace chain and honoring {@link DdlSettings#includeSchema()}. + */ + public TableReference tableReference(NamedColumnSet table) { + Schema schema = NamedColumnSets.findSchema(table).orElse(null); + String type = table instanceof View ? TableReference.TYPE_VIEW : TableReference.TYPE_TABLE; + return tableRef(schema, table, type); + } + + public List columnDefinitions(NamedColumnSet table) { + return CwmSchemaMapper.columnDefinitions(tableReference(table), table); + } + + /** {@code CREATE TABLE} for {@code table}, deriving its schema and PK. */ + public String createTable(Table table) { + return createTable(NamedColumnSets.findSchema(table).orElse(null), table); + } + + /** + * {@code CREATE TABLE} for {@code table} qualified by an explicit + * {@code schema}. Use this when the table is not (yet) contained under the + * schema in the namespace chain — e.g. a freshly diffed table. + */ + public String createTable(Schema schema, Table table) { + TableReference tref = tableRef(schema, table, TableReference.TYPE_TABLE); + List cols = CwmSchemaMapper.columnDefinitions(tref, table); + org.eclipse.daanse.jdbc.db.api.schema.PrimaryKey pk = primaryKeyRef(tref, table); + return dialect.ddlGenerator().createTable(tref, cols, pk, settings.ifNotExists()); + } + + // -------------------- internal -------------------- + + private TableReference tableRef(Schema schema, NamedColumnSet ncs, String type) { + Optional sref = (settings.includeSchema() && schema != null && schema.getName() != null + && !schema.getName().isBlank()) ? Optional.of(new SchemaReference(Optional.empty(), schema.getName())) + : Optional.empty(); + return new TableReference(sref, ncs.getName(), type); + } + + private org.eclipse.daanse.jdbc.db.api.schema.PrimaryKey primaryKeyRef(TableReference tref, Table table) { + Optional pkOpt = Tables.findPrimaryKey(table); + if (pkOpt.isEmpty() || PrimaryKeys.columns(pkOpt.get()).isEmpty()) { + return null; + } + return CwmSchemaMapper.primaryKey(tref, pkOpt.get()); + } + + private static List checkConstraintsOf(Table table) { + // CWM stores CheckConstraints either on the table (ownedElement) or on + // the column (Constraint.constrainedElement) — collect from both. + List out = new ArrayList<>(); + Namespaces.ownedElementStream(table, CheckConstraint.class).forEach(out::add); + for (Column col : ColumnSets.columns(table)) { + col.getConstraint().stream().filter(CheckConstraint.class::isInstance).map(CheckConstraint.class::cast) + .filter(c -> !out.contains(c)).forEach(out::add); + } + return out; + } + + private static List indexColumnNames(SQLIndex i) { + List out = new ArrayList<>(); + for (Column c : Indexes.columns(i)) { + if (c.getName() != null) { + out.add(c.getName()); + } + } + return out; + } + + private static List refColumnNames(ForeignKey fk) { + List out = new ArrayList<>(); + if (fk.getUniqueKey() == null) { + return out; + } + for (StructuralFeature f : fk.getUniqueKey().getFeature()) { + if (f instanceof Column c && c.getName() != null) { + out.add(c.getName()); + } + } + return out; + } + + private static String nameOrDefault(ModelElement e, String fallback) { + String n = e == null ? null : e.getName(); + return (n == null || n.isBlank()) ? fallback : n; + } + + private static String referentialAction(String literal) { + // CWM names like IMPORTED_KEY_CASCADE / imported_key_no_action — match + // on upper-cased contains so both underscore and camel forms work. + if (literal == null) { + return null; + } + String upper = literal.toUpperCase(); + if (upper.contains("CASCADE")) { + return "CASCADE"; + } + if (upper.contains("SET_NULL") || upper.contains("SETNULL")) { + return "SET NULL"; + } + if (upper.contains("SET_DEFAULT") || upper.contains("SETDEFAULT")) { + return "SET DEFAULT"; + } + if (upper.contains("RESTRICT")) { + return "RESTRICT"; + } + if (upper.contains("NO_ACTION") || upper.contains("NOACTION")) { + return "NO ACTION"; + } + return null; + } + + private static String triggerBody(Trigger t) { + if (t.getActionStatement() == null) { + return null; + } + String body = t.getActionStatement().getBody(); + return (body == null || body.isBlank()) ? null : body; + } + + private static String triggerWhen(Trigger t) { + if (t.getActionCondition() == null) { + return null; + } + String cond = t.getActionCondition().getBody(); + return (cond == null || cond.isBlank()) ? null : cond; + } + + private static TriggerTiming triggerTiming(Trigger t) { + if (t.getConditionTiming() == null) { + return null; + } + String name = t.getConditionTiming().getName(); + if (name == null) { + return null; + } + return switch (stripEnumPrefix(name).toUpperCase()) { + case "BEFORE" -> TriggerTiming.BEFORE; + case "AFTER" -> TriggerTiming.AFTER; + case "INSTEAD", "INSTEADOF" -> TriggerTiming.INSTEAD_OF; + default -> null; + }; + } + + private static TriggerEvent triggerEvent(Trigger t) { + if (t.getEventManipulation() == null) { + return null; + } + String name = t.getEventManipulation().getName(); + if (name == null) { + return null; + } + return switch (stripEnumPrefix(name).toUpperCase()) { + case "INSERT" -> TriggerEvent.INSERT; + case "UPDATE" -> TriggerEvent.UPDATE; + case "DELETE" -> TriggerEvent.DELETE; + default -> null; + }; + } + + private static TriggerScope triggerScope(Trigger t) { + if (t.getActionOrientation() == null) { + return TriggerScope.STATEMENT; + } + String name = t.getActionOrientation().getName(); + return switch (stripEnumPrefix(name == null ? "" : name).toUpperCase()) { + case "ROW" -> TriggerScope.ROW; + default -> TriggerScope.STATEMENT; + }; + } + + private static String stripEnumPrefix(String literal) { + if (literal == null) { + return ""; + } + int us = literal.lastIndexOf('_'); + return us >= 0 ? literal.substring(us + 1) : literal; + } +} diff --git a/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/RoundTripParameterizedTest.java b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/RoundTripParameterizedTest.java new file mode 100644 index 0000000..f6758f7 --- /dev/null +++ b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/RoundTripParameterizedTest.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.daanse.cwm.resource.relational.ddl.internal.support.SqlGenAssertions.executeAll; + +import java.sql.Connection; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.Feature; +import org.eclipse.daanse.cwm.resource.relational.ddl.internal.support.SqlGenAssertions; +import org.eclipse.daanse.cwm.resource.relational.ddl.internal.support.SqlGenFixture; +import org.eclipse.daanse.cwm.resource.relational.ddl.internal.support.DialectProfile; +import org.eclipse.daanse.jdbc.datasource.testkit.api.ActiveDatabase; +import org.eclipse.daanse.jdbc.db.api.DatabaseService; +import org.eclipse.daanse.jdbc.db.api.meta.MetaInfo; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.eclipse.daanse.jdbc.db.impl.DatabaseServiceImpl; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * One round-trip "source" (build the CUSTOMERS/ORDERS/view fixture, emit DDL, + * execute, snapshot, assert) run across every {@link DialectProfile}. Replaces + * the five per-dialect {@code RoundTripTest} classes; each dialect's quirks + * (schema scoping, feature subset, trigger emit form) live in the profile. + */ +class RoundTripParameterizedTest { + + private static final DatabaseService DB_SERVICE = new DatabaseServiceImpl(); + + static Stream dialects() { + return Stream.of(DialectProfile.values()); + } + + static Stream triggerDialects() { + return Stream.of(DialectProfile.values()).filter(DialectProfile::supportsTriggers); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("dialects") + void create_snapshot_drop(DialectProfile profile) throws Exception { + ActiveDatabase db = activateOrSkip(profile); + try (Connection c = db.dataSource().getConnection()) { + Dialect dialect = db.dialect(); + SqlGenFixture f = SqlGenFixture.build(profile.schemaName(), dialect); + Set features = profile.nonTriggerFeatures(); + try { + executeAll(c, new DdlGeneratorFactoryImpl().create(dialect).createSchema(f.schema, features)); + + profile.prepareForMetadata(c); + MetaInfo info = profile.snapshot(DB_SERVICE, c, dialect); + SqlGenAssertions.assertTableExists(info.structureInfo(), profile.schemaName(), "CUSTOMERS", + TableReference.TYPE_TABLE); + SqlGenAssertions.assertTableExists(info.structureInfo(), profile.schemaName(), "ORDERS", + TableReference.TYPE_TABLE); + SqlGenAssertions.assertTableExists(info.structureInfo(), profile.schemaName(), "CUSTOMER_ORDERS", + TableReference.TYPE_VIEW); + } finally { + profile.cleanup(c, f.schema, dialect, features); + } + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("triggerDialects") + void shared_trigger_body_emits_one_procedure_for_two_triggers(DialectProfile profile) throws Exception { + ActiveDatabase db = activateOrSkip(profile); + try (Connection c = db.dataSource().getConnection()) { + Dialect dialect = db.dialect(); + SqlGenFixture f = SqlGenFixture.build(profile.schemaName(), dialect); + f.attachTrigger(f.customers, "trg_audit_cust", profile.triggerBody(), profile.triggerTiming(), + profile.triggerEvent()); + f.attachTrigger(f.orders, "trg_audit_ord", profile.triggerBody(), profile.triggerTiming(), + profile.triggerEvent()); + Set features = profile.allFeatures(); + try { + List ddl = new DdlGeneratorFactoryImpl().create(dialect).createSchema(f.schema, features); + assertThat(ddl.stream().filter(s -> s.startsWith(profile.triggerProcPrefix())).count()) + .as("one shared procedure/function for two triggers with the same body").isEqualTo(1); + assertThat(ddl.stream().filter(s -> s.startsWith("CREATE TRIGGER")).count()) + .as("two CREATE TRIGGER statements").isEqualTo(2); + if (profile.triggerCallSnippet() != null) { + assertThat(ddl).anyMatch( + s -> s.startsWith("CREATE TRIGGER") && s.contains(profile.triggerCallSnippet())); + } + // Lands on a real engine. + executeAll(c, ddl); + } finally { + profile.cleanup(c, f.schema, dialect, features); + } + } + } + + private static ActiveDatabase activateOrSkip(DialectProfile profile) { + try { + return profile.activate(); + } catch (RuntimeException e) { + Assumptions.assumeTrue(false, "Database '" + profile + "' unavailable (no Docker?): " + e.getMessage()); + throw new AssertionError("unreachable"); + } + } +} diff --git a/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/settings/DdlSettingsTest.java b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/settings/DdlSettingsTest.java new file mode 100644 index 0000000..4fa9435 --- /dev/null +++ b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/settings/DdlSettingsTest.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal.settings; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlSettings; +import org.junit.jupiter.api.Test; + +/** Unit cover for the immutable {@link DdlSettings} record and its withers. */ +class DdlSettingsTest { + + @Test + void defaults_qualify_schema_use_if_not_exists_and_no_cascade() { + DdlSettings d = DdlSettings.defaults(); + assertThat(d.includeSchema()).isTrue(); + assertThat(d.ifNotExists()).isTrue(); + assertThat(d.cascade()).isFalse(); + } + + @Test + void withers_change_one_field_and_leave_the_rest() { + DdlSettings d = DdlSettings.defaults(); + assertThat(d.withIncludeSchema(false)).isEqualTo(new DdlSettings(false, true, false)); + assertThat(d.withIfNotExists(false)).isEqualTo(new DdlSettings(true, false, false)); + assertThat(d.withCascade(true)).isEqualTo(new DdlSettings(true, true, true)); + // original is untouched (immutable) + assertThat(d).isEqualTo(DdlSettings.defaults()); + } + + @Test + void withers_chain() { + DdlSettings d = DdlSettings.defaults().withIncludeSchema(false).withCascade(true); + assertThat(d).isEqualTo(new DdlSettings(false, true, true)); + } +} diff --git a/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/DialectProfile.java b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/DialectProfile.java new file mode 100644 index 0000000..1c55271 --- /dev/null +++ b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/DialectProfile.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal.support; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.Feature; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlSettings; +import org.eclipse.daanse.cwm.resource.relational.ddl.internal.DdlGeneratorFactoryImpl; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.ConditionTimingType; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.EventManipulationType; +import org.eclipse.daanse.jdbc.datasource.testkit.api.ActiveDatabase; +import org.eclipse.daanse.jdbc.datasource.testkit.api.DatabaseProvider; +import org.eclipse.daanse.jdbc.db.api.DatabaseService; +import org.eclipse.daanse.jdbc.db.api.meta.MetaInfo; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** + * One row of cross-dialect test parameterization: the {@code DatabaseProvider} + * id plus the per-dialect knobs that the otherwise-identical test bodies need + * (schema name + scoping, the feature subset that lands cleanly, and the + * trigger profile). Activating yields a live {@link ActiveDatabase} from the + * {@code jdbc.datasource.testkit} ServiceLoader; H2 needs no Docker, the + * container ones throw on {@link #activate()} when Docker is unavailable so the + * caller can skip. + */ +public enum DialectProfile { + + /** In-process; no triggers (H2 wants Java class bodies). */ + H2("h2", "RT_H2", true, true, false, null, null, null, null, null), + + POSTGRES("postgres", "rt_pg", true, true, true, + "BEGIN RAISE NOTICE 'audit'; RETURN NEW; END;", "CREATE OR REPLACE FUNCTION", "EXECUTE FUNCTION", + ConditionTimingType.BEFORE, EventManipulationType.INSERT), + + /** Schema == database "rt" (pre-created by the provider); never dropped. */ + MARIADB("mariadb", "rt", false, false, true, + "BEGIN SET @audit_count = COALESCE(@audit_count, 0) + 1; END", "CREATE PROCEDURE", "CALL ", + ConditionTimingType.BEFORE, EventManipulationType.INSERT), + + /** MSSQL has no BEFORE triggers — AFTER INSERT. createMetaInfo without dialect. */ + MSSQL("mssql", "rt_mssql", true, false, true, + "BEGIN SET NOCOUNT ON; END", "CREATE OR ALTER PROCEDURE", "AS EXEC ", + ConditionTimingType.AFTER, EventManipulationType.INSERT), + + /** Schema == user "RT"; CREATE SCHEMA is skipped (Oracle ties it to a user). */ + ORACLE("oracle", "RT", false, false, true, + "BEGIN NULL; END;", "CREATE OR REPLACE PROCEDURE", null, + ConditionTimingType.BEFORE, EventManipulationType.INSERT); + + private final String providerId; + private final String schemaName; + private final boolean ownsSchemaNamespace; // can CREATE/DROP the schema as a namespace + private final boolean setSchemaBeforeMeta; + private final boolean supportsTriggers; + private final String triggerBody; + private final String triggerProcPrefix; + private final String triggerCallSnippet; // null = no shared-procedure CALL form (Oracle) + private final ConditionTimingType triggerTiming; + private final EventManipulationType triggerEvent; + + DialectProfile(String providerId, String schemaName, boolean ownsSchemaNamespace, boolean setSchemaBeforeMeta, + boolean supportsTriggers, String triggerBody, String triggerProcPrefix, String triggerCallSnippet, + ConditionTimingType triggerTiming, EventManipulationType triggerEvent) { + this.providerId = providerId; + this.schemaName = schemaName; + this.ownsSchemaNamespace = ownsSchemaNamespace; + this.setSchemaBeforeMeta = setSchemaBeforeMeta; + this.supportsTriggers = supportsTriggers; + this.triggerBody = triggerBody; + this.triggerProcPrefix = triggerProcPrefix; + this.triggerCallSnippet = triggerCallSnippet; + this.triggerTiming = triggerTiming; + this.triggerEvent = triggerEvent; + } + + public String providerId() { + return providerId; + } + + public String schemaName() { + return schemaName; + } + + public boolean supportsTriggers() { + return supportsTriggers; + } + + public String triggerBody() { + return triggerBody; + } + + public String triggerProcPrefix() { + return triggerProcPrefix; + } + + public String triggerCallSnippet() { + return triggerCallSnippet; + } + + public ConditionTimingType triggerTiming() { + return triggerTiming; + } + + public EventManipulationType triggerEvent() { + return triggerEvent; + } + + /** Activate the underlying provider (throws when its Docker engine is absent). */ + public ActiveDatabase activate() { + return DatabaseProvider.byId(providerId).activate(); + } + + /** Features that create cleanly here: everything except triggers (+ SCHEMA where the dialect owns no namespace). */ + public Set nonTriggerFeatures() { + EnumSet s = EnumSet.complementOf(EnumSet.of(Feature.TRIGGER)); + if (!ownsSchemaNamespace) { + s.remove(Feature.SCHEMA); + } + return s; + } + + /** Features for a triggered create: everything (minus SCHEMA where unsupported). */ + public Set allFeatures() { + EnumSet s = EnumSet.allOf(Feature.class); + if (!ownsSchemaNamespace) { + s.remove(Feature.SCHEMA); + } + return s; + } + + /** Point the connection at the test schema before metadata reads, where the dialect needs it. */ + public void prepareForMetadata(Connection c) throws SQLException { + if (setSchemaBeforeMeta) { + c.setSchema(schemaName); + } + if (this == MARIADB) { + // MariaDB puts the database in the catalog slot; without scoping, the + // snapshot also returns performance_schema/information_schema tables. + c.setCatalog(schemaName); + } + } + + /** Snapshot the live structure; SQL Server's MetadataProvider is selected without an explicit dialect. */ + public MetaInfo snapshot(DatabaseService service, Connection c, Dialect dialect) throws SQLException { + return this == MSSQL ? service.createMetaInfo(c) : service.createMetaInfo(c, dialect); + } + + /** + * Best-effort teardown: reverse the model via the generator (dialect- and + * model-correct), then drop the schema namespace where the dialect owns one. + * All failures are swallowed — each dialect runs against its own database so + * leftovers never cross-contaminate, and fixtures emit {@code IF NOT EXISTS}. + */ + public void cleanup(Connection c, Schema model, Dialect dialect, Set features) { + try { + List drop = new DdlGeneratorFactoryImpl().create(dialect, DdlSettings.defaults().withCascade(true)) + .dropSchema(model, features); + try (Statement s = c.createStatement()) { + for (String stmt : drop) { + try { + s.execute(stmt); + } catch (SQLException ignored) { + // best effort + } + } + } + } catch (SQLException ignored) { + // best effort + } + if (ownsSchemaNamespace) { + try (Statement s = c.createStatement()) { + s.execute("DROP SCHEMA IF EXISTS \"" + schemaName + "\" CASCADE"); + } catch (SQLException ignored) { + // H2/PG honour this; SQL Server rejects CASCADE — the model-drop above already cleared it. + } + } + } + + @Override + public String toString() { + return providerId; + } +} diff --git a/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenAssertions.java b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenAssertions.java new file mode 100644 index 0000000..47f5ee1 --- /dev/null +++ b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenAssertions.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal.support; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.daanse.jdbc.db.api.meta.StructureInfo; +import org.eclipse.daanse.jdbc.db.api.schema.SchemaReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; + +/** Shared assertions for per-dialect round-trip tests. */ +public final class SqlGenAssertions { + + private SqlGenAssertions() { + } + + public static List tableNamesIn(StructureInfo si, String schema, String type) { + List out = new ArrayList<>(); + for (TableDefinition td : si.tables()) { + TableReference tr = td.table(); + if (!matchesType(type, tr.type())) + continue; + if (!matchesSchema(schema, tr)) + continue; + out.add(tr.name()); + } + return out; + } + + public static void executeAll(Connection connection, List sql) throws SQLException { + try (Statement s = connection.createStatement()) { + for (String stmt : sql) { + if (stmt == null || stmt.startsWith("--")) + continue; + try { + s.execute(stmt); + } catch (SQLException e) { + throw new SQLException("failed: " + stmt + " — " + e.getMessage(), e); + } + } + } + } + + public static void assertTableExists(StructureInfo si, String schema, String tableName, String type) { + assertThat(tableNamesIn(si, schema, type)).as("tables in schema=%s of type=%s", schema, type) + .contains(tableName); + } + + private static boolean matchesType(String expected, String actual) { + if (expected.equals(actual)) + return true; + // H2 reports "BASE TABLE" instead of "TABLE". + return TableReference.TYPE_TABLE.equals(expected) && "BASE TABLE".equals(actual); + } + + private static boolean matchesSchema(String expected, TableReference tr) { + // MariaDB/MySQL drivers don't always populate the schema field; accept + // empty schema as a match when the caller scopes at the connection level. + return tr.schema().map(SchemaReference::name).map(s -> s.equalsIgnoreCase(expected)).orElse(true); + } +} diff --git a/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenFixture.java b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenFixture.java new file mode 100644 index 0000000..6691e5b --- /dev/null +++ b/resource/relational/ddl/src/test/java/org/eclipse/daanse/cwm/resource/relational/ddl/internal/support/SqlGenFixture.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.daanse.cwm.resource.relational.ddl.internal.support; + +import java.sql.Types; + +import org.eclipse.daanse.cwm.model.cwm.foundation.datatypes.DatatypesFactory; +import org.eclipse.daanse.cwm.model.cwm.foundation.datatypes.QueryExpression; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.BooleanExpression; +import org.eclipse.daanse.cwm.model.cwm.objectmodel.core.CoreFactory; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.CheckConstraint; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.ForeignKey; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.PrimaryKey; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.RelationalFactory; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLIndex; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLIndexColumn; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLSimpleType; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.UniqueConstraint; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.View; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.NullableType; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.ReferentialRuleType; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** + * Shared CWM fixture: CUSTOMERS + ORDERS with PK/UC/CHECK/INDEX/FK and a + * CUSTOMER_ORDERS view. Triggers are attached separately via + * {@link #attachTrigger} since their bodies are dialect-specific. + */ +public final class SqlGenFixture { + + public static final RelationalFactory RF = RelationalFactory.eINSTANCE; + public static final CoreFactory CF = CoreFactory.eINSTANCE; + public static final DatatypesFactory DF = DatatypesFactory.eINSTANCE; + + public final Schema schema; + public final Table customers; + public final Table orders; + public final View view; + public final UniqueConstraint uniqueEmail; + public final ForeignKey ordersFk; + public final CheckConstraint emailCheck; + public final SQLIndex nameIndex; + + private SqlGenFixture(Schema schema, Table customers, Table orders, View view, UniqueConstraint uniqueEmail, + ForeignKey ordersFk, CheckConstraint emailCheck, SQLIndex nameIndex) { + this.schema = schema; + this.customers = customers; + this.orders = orders; + this.view = view; + this.uniqueEmail = uniqueEmail; + this.ordersFk = ordersFk; + this.emailCheck = emailCheck; + this.nameIndex = nameIndex; + } + + public static SqlGenFixture build(String schemaName, Dialect dialect) { + Schema schema = RF.createSchema(); + schema.setName(schemaName); + + Table customers = RF.createTable(); + customers.setName("CUSTOMERS"); + Column cId = col("ID", type("INTEGER", Types.INTEGER, 0, 0, 0), true); + Column cEmail = col("EMAIL", type("CHARACTER VARYING", Types.VARCHAR, 100, 0, 0), true); + Column cName = col("NAME", type("CHARACTER VARYING", Types.VARCHAR, 50, 0, 0), false); + customers.getFeature().add(cId); + customers.getFeature().add(cEmail); + customers.getFeature().add(cName); + schema.getOwnedElement().add(customers); + + PrimaryKey customersPk = RF.createPrimaryKey(); + customersPk.setName("PK_CUSTOMERS"); + customersPk.getFeature().add(cId); + cId.getUniqueKey().add(customersPk); + + UniqueConstraint uc = RF.createUniqueConstraint(); + uc.setName("UC_CUSTOMERS_EMAIL"); + uc.getFeature().add(cEmail); + cEmail.getUniqueKey().add(uc); + + CheckConstraint cc = RF.createCheckConstraint(); + cc.setName("CK_CUSTOMERS_ID_POS"); + BooleanExpression be = CF.createBooleanExpression(); + // A numeric comparison works on every major dialect; LENGTH/LEN/CHAR_LENGTH + // diverge between PG/MariaDB and MSSQL. + be.setBody(dialect.quoteIdentifier("ID").toString() + " > 0"); + be.setLanguage("SQL"); + cc.setBody(be); + customers.getOwnedElement().add(cc); + + SQLIndex idx = RF.createSQLIndex(); + idx.setName("IDX_CUSTOMERS_NAME"); + idx.setSpannedClass(customers); + SQLIndexColumn ifc = RF.createSQLIndexColumn(); + ifc.setFeature(cName); + idx.getIndexedFeature().add(ifc); + schema.getOwnedElement().add(idx); + + Table orders = RF.createTable(); + orders.setName("ORDERS"); + Column oId = col("ID", type("INTEGER", Types.INTEGER, 0, 0, 0), true); + Column oCustomerId = col("CUSTOMER_ID", type("INTEGER", Types.INTEGER, 0, 0, 0), true); + Column oTotal = col("TOTAL", type("DECIMAL", Types.DECIMAL, 0, 10, 2), false); + orders.getFeature().add(oId); + orders.getFeature().add(oCustomerId); + orders.getFeature().add(oTotal); + schema.getOwnedElement().add(orders); + + PrimaryKey ordersPk = RF.createPrimaryKey(); + ordersPk.setName("PK_ORDERS"); + ordersPk.getFeature().add(oId); + oId.getUniqueKey().add(ordersPk); + + ForeignKey fk = RF.createForeignKey(); + fk.setName("FK_ORDERS_CUSTOMERS"); + fk.getFeature().add(oCustomerId); + fk.setUniqueKey(customersPk); + oCustomerId.getKeyRelationship().add(fk); + fk.setDeleteRule(ReferentialRuleType.IMPORTED_KEY_CASCADE); + // Oracle FKs don't accept ON UPDATE — emitter respects this on Oracle. + fk.setUpdateRule(ReferentialRuleType.IMPORTED_KEY_NO_ACTION); + + View view = RF.createView(); + view.setName("CUSTOMER_ORDERS"); + QueryExpression qe = DF.createQueryExpression(); + qe.setLanguage("SQL"); + String custQ = dialect.quoteIdentifier(schemaName, "CUSTOMERS").toString(); + String ordQ = dialect.quoteIdentifier(schemaName, "ORDERS").toString(); + String nameQ = dialect.quoteIdentifier("NAME").toString(); + String totalQ = dialect.quoteIdentifier("TOTAL").toString(); + String custIdQ = dialect.quoteIdentifier("CUSTOMER_ID").toString(); + String idQ = dialect.quoteIdentifier("ID").toString(); + qe.setBody("SELECT C." + nameQ + ", O." + totalQ + " FROM " + custQ + " C JOIN " + ordQ + " O " + "ON O." + + custIdQ + " = C." + idQ); + view.setQueryExpression(qe); + schema.getOwnedElement().add(view); + + return new SqlGenFixture(schema, customers, orders, view, uc, fk, cc, idx); + } + + /** Attach a {@code BEFORE INSERT FOR EACH ROW} trigger. */ + public org.eclipse.daanse.cwm.model.cwm.resource.relational.Trigger attachTrigger(Table table, String triggerName, + String procedureBody) { + return attachTrigger(table, triggerName, procedureBody, + org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.ConditionTimingType.BEFORE, + org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.EventManipulationType.INSERT); + } + + public org.eclipse.daanse.cwm.model.cwm.resource.relational.Trigger attachTrigger(Table table, String triggerName, + String procedureBody, + org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.ConditionTimingType timing, + org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.EventManipulationType event) { + org.eclipse.daanse.cwm.model.cwm.resource.relational.Trigger trg = RF.createTrigger(); + trg.setName(triggerName); + trg.setTable(table); + trg.setConditionTiming(timing); + trg.setEventManipulation(event); + trg.setActionOrientation( + org.eclipse.daanse.cwm.model.cwm.resource.relational.enumerations.ActionOrientationType.ROW); + org.eclipse.daanse.cwm.model.cwm.objectmodel.core.ProcedureExpression body = CF.createProcedureExpression(); + body.setLanguage("SQL"); + body.setBody(procedureBody); + trg.setActionStatement(body); + table.getTrigger().add(trg); + return trg; + } + + private static SQLSimpleType type(String name, int jdbc, long max, long prec, long scale) { + SQLSimpleType t = RF.createSQLSimpleType(); + t.setName(name); + t.setTypeNumber(jdbc); + if (max > 0) + t.setCharacterMaximumLength(max); + if (prec > 0) + t.setNumericPrecision(prec); + if (scale > 0) + t.setNumericScale(scale); + return t; + } + + private static Column col(String name, SQLSimpleType type, boolean notNull) { + Column c = RF.createColumn(); + c.setName(name); + c.setType(type); + c.setIsNullable(notNull ? NullableType.COLUMN_NO_NULLS : NullableType.COLUMN_NULLABLE); + return c; + } +} diff --git a/resource/relational/pom.xml b/resource/relational/pom.xml new file mode 100644 index 0000000..950b9b5 --- /dev/null +++ b/resource/relational/pom.xml @@ -0,0 +1,40 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm.resource + ${revision} + ../pom.xml + + org.eclipse.daanse.cwm.resource.relational + pom + Daanse CWM Resource — Relational (aggregator) + Aggregator for the CWM relational resource tooling, split into + focused modules: DDL generation + (ddl), JDBC load (load.jdbc), schema diff/migration (diff), SQL/view + resolution (sql.resolve) and index advice (index.advise). + + + ddl + + + diff --git a/testkit/README.md b/testkit/README.md new file mode 100644 index 0000000..8e87e54 --- /dev/null +++ b/testkit/README.md @@ -0,0 +1,71 @@ +# Daanse CWM — Test Kit (aggregator) + +CWM-side test-kit: SPIs for describing the desired database state via +CWM, plus executors that materialise that state into a live JDBC +database and assert it back. + +## Why it exists + +Daanse tests have two concerns that overlap but are not the same: + +1. **What database structure should exist** — tables, columns, types, + indexes, foreign keys, views, triggers. This is naturally + expressed as a CWM `Schema` (the EMF model already in + `org.eclipse.daanse.cwm.model.cwm`). +2. **How to drive a real JDBC database** to that state and to load + data into it. This was previously hand-rolled in every test + (CREATE TABLE strings + raw INSERTs + JDBC metadata walks). + +This module group bridges the two: a test declares the desired state +once as a CWM `Schema` plus optional `DataSupplier`/`DatabaseCheckSuiteSupplier`, +and the executors here do the rest. All dialect-specific DDL/SQL +emission goes through the existing `cwm.sql.gen` + `jdbc.db.dialect.*` +chain, so the same CWM declaration works across H2 / PG / MySQL / +MSSQL / MariaDB / Oracle without changes. + +## What ships here + +| Module | Artifact | Purpose | +|--------|----------|---------| +| `api` | `org.eclipse.daanse.cwm.testkit.api` | SPIs: `DatabaseSupplier`, `DataSupplier`, `DatabaseCheckSuiteSupplier`, `CsvAutoDetect`, and the `dbcheck.*` record family | +| `database` | `org.eclipse.daanse.cwm.testkit.database` | `DatabaseLayer.apply(...)` — CWM Schema → DDL → JDBC. `DatabaseCheckExecutor` — diff live DB metadata against expectations | +| `data` | `org.eclipse.daanse.cwm.testkit.data` | `DataLayer.apply(...)` — load CSV resources into the tables created by `DatabaseLayer`, typed by the CWM Schema's columns | + +## Composition + +The three modules layer cleanly: + +``` +DatabaseSupplier → DatabaseLayer.apply(ds, dialect, schema) +DataSupplier → DataLayer.apply(ds, dialect, schema, supplier) +DatabaseCheckSuite → DatabaseCheckExecutor.execute(breadcrumb, ds, suite) +``` + +Each layer is optional and independently usable. A test that just +wants tables-but-no-data uses Layer 1 only; one that wants tables + +CSV-loaded data uses Layers 1+2; one that wants tables + data + a +post-load assertion uses all three. + +## Position in the daanse stack + +``` + ┌─────────────────────────────────────┐ + │ cwm.testkit.api (SPIs + records) │ + └────┬───────────┬─────────────┬──────┘ + │ │ │ + ┌────────▼──┐ ┌─────▼──┐ ┌───────▼──────┐ + │ database │ │ data │ │ (consumer: │ + │ Layer + │ │ Layer │ │ rolap │ + │ Check Exec│ │ │ │ testkit) │ + └───────────┘ └────────┘ └──────────────┘ + │ │ + ▼ ▼ + cwm.sql.gen cwm.csv.read + cwm.sink.jdbc + (DDL emission) (CSV → JDBC pipeline) +``` + +## Related modules + +- `jdbc.datasource.testkit` — provides the `(DataSource, Dialect)` pair the executors consume. +- `rolap.testkit.core.CatalogTestHarness` — top-level composer that wires `DatabaseLayer + DataLayer + DatabaseCheckExecutor + olap.testkit.OlapCheckSuiteRunner + TestContext`. +- `cwm.csv.read` + `cwm.sink.jdbc` — the underlying CSV→DB pipeline `DataLayer` drives. diff --git a/testkit/pom.xml b/testkit/pom.xml new file mode 100644 index 0000000..d5289d8 --- /dev/null +++ b/testkit/pom.xml @@ -0,0 +1,92 @@ + + + + 4.0.0 + + org.eclipse.daanse + org.eclipse.daanse.cwm + ${revision} + + org.eclipse.daanse.cwm.testkit + jar + Daanse CWM Test Kit + CWM-side testkit in a single jar. The api package holds the + SPI types: DatabaseSupplier (returns CWM Schema), DataSupplier (CSV + resources + programmatic load), DatabaseCheckSuiteSupplier (DB + structural assertions), CsvAutoDetect, and the DatabaseCheckSuite + record family. The database package materializes a CWM Schema into a + JDBC DataSource via the dialect's DDL generator and asserts live + structure (DatabaseLayer, DatabaseCheckExecutor). The data package + loads CSV resources into those tables via the cwm.data pipeline + (DataLayer). + + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.dialect.api + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.model.cwm + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.util + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.resource.relational.ddl + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.impl + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.data.source.csv + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.cwm.data.sink.jdbc + ${revision} + compile + + + org.eclipse.daanse + org.eclipse.daanse.jdbc.db.record + ${revision} + compile + + + org.junit.jupiter + junit-jupiter-api + 5.12.2 + compile + + + + diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/CsvAutoDetect.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/CsvAutoDetect.java new file mode 100644 index 0000000..db6cc68 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/CsvAutoDetect.java @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api; + +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Stream; + +/** + * Finds CSV files packaged as classpath resources beside an anchor class. + */ +public final class CsvAutoDetect { + + private CsvAutoDetect() { + } + + /** + * Returns the {@code .csv} resources under + * {@code //}, keyed by file name without extension. + * The map keeps alphabetical order, so name files to load parents before + * children when foreign keys require it. + */ + public static Map detect(Class anchor, String subFolder) { + String pkgPath = anchor.getPackageName().replace('.', '/'); + String prefix = pkgPath + "/" + subFolder + "/"; + + TreeSet sortedNames = new TreeSet<>(); + Map byName = new LinkedHashMap<>(); + + ClassLoader cl = anchor.getClassLoader(); + if (cl == null) { + cl = ClassLoader.getSystemClassLoader(); + } + + try { + Enumeration roots = cl.getResources(prefix); + for (URL root : Collections.list(roots)) { + collectFrom(root, prefix, sortedNames, byName); + } + } catch (IOException e) { + throw new IllegalStateException( + "Failed to scan classpath for " + prefix + " (anchor=" + anchor.getName() + ")", e); + } + + LinkedHashMap ordered = new LinkedHashMap<>(); + for (String name : sortedNames) { + ordered.put(name, byName.get(name)); + } + return ordered; + } + + private static void collectFrom(URL root, String prefix, TreeSet sortedNames, Map byName) + throws IOException { + switch (root.getProtocol()) { + case "file" -> collectFromFile(root, sortedNames, byName); + case "jar" -> collectFromJar(root, prefix, sortedNames, byName); + default -> collectFromGenericFs(root, prefix, sortedNames, byName); + } + } + + private static void collectFromFile(URL root, TreeSet sortedNames, Map byName) + throws IOException { + Path dir = Paths.get(URI.create(root.toString())); + try (Stream walk = Files.list(dir)) { + walk.filter(p -> p.toString().endsWith(".csv")) + .forEach(p -> add(p.getFileName().toString(), pathToUrl(p), sortedNames, byName)); + } + } + + private static void collectFromJar(URL root, String prefix, TreeSet sortedNames, Map byName) + throws IOException { + String spec = root.getFile(); + int bang = spec.indexOf("!/"); + String jarPath = spec.substring("file:".length(), bang); + try (JarFile jar = new JarFile(jarPath)) { + Enumeration entries = jar.entries(); + while (entries.hasMoreElements()) { + JarEntry e = entries.nextElement(); + if (e.isDirectory()) { + continue; + } + String name = e.getName(); + if (!name.startsWith(prefix) || !name.endsWith(".csv")) { + continue; + } + String fileName = name.substring(prefix.length()); + if (fileName.contains("/")) { + continue; + } + URL entryUrl = new URL("jar:file:" + jarPath + "!/" + name); + add(fileName, entryUrl, sortedNames, byName); + } + } + } + + private static void collectFromGenericFs(URL root, String prefix, TreeSet sortedNames, + Map byName) throws IOException { + URI uri; + try { + uri = root.toURI(); + } catch (Exception e) { + throw new IOException("Bad URI: " + root, e); + } + try (FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + Path dir = fs.getPath("/" + prefix); + try (Stream walk = Files.list(dir)) { + walk.filter(p -> p.toString().endsWith(".csv")) + .forEach(p -> add(p.getFileName().toString(), pathToUrl(p), sortedNames, byName)); + } + } + } + + private static void add(String fileName, URL url, TreeSet sortedNames, Map byName) { + String key = fileName.substring(0, fileName.length() - ".csv".length()); + sortedNames.add(key); + byName.put(key, url); + } + + private static URL pathToUrl(Path p) { + try { + return p.toUri().toURL(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DataSupplier.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DataSupplier.java new file mode 100644 index 0000000..dcc8a6b --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DataSupplier.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api; + +import java.net.URL; +import java.sql.Connection; +import java.util.Map; + +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** + * Supplies the rows to load into the tables created from the + * {@link DatabaseSupplier} schema. Both entry points are optional; CSVs load + * first, then {@link #load(Connection, Dialect)}. + */ +public interface DataSupplier { + + /** + * CSV resources keyed by target table name (optionally {@code "schema.table"}). + * Iteration order is the load order — use a {@link java.util.LinkedHashMap} to + * load parent rows before child rows. CSVs are header-only; column types come + * from the CWM table. + */ + default Map csvResources() { + return Map.of(); + } + + /** + * Loads rows programmatically, after the CSVs. Use the given connection and + * dialect to issue your own INSERTs. + */ + default void load(Connection connection, Dialect dialect) throws Exception { + // default: nothing + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseCheckSuiteSupplier.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseCheckSuiteSupplier.java new file mode 100644 index 0000000..d6b287f --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseCheckSuiteSupplier.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api; + +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseCheckSuite; + +/** + * Supplies the database-shape checks (schemas, tables, columns, queries) to run + * against the loaded {@code DataSource}. + */ +public interface DatabaseCheckSuiteSupplier { + + DatabaseCheckSuite get(); +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseSupplier.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseSupplier.java new file mode 100644 index 0000000..7493429 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/DatabaseSupplier.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; + +/** + * Supplies the CWM {@code Schema} to create before any data is loaded. Its + * tables, columns, keys, indexes and views are materialized via the dialect's + * DDL generator. + */ +public interface DatabaseSupplier { + + /** + * The CWM Schema to create. A blank name or {@code "default"} creates the + * tables in the default schema, unqualified. + */ + Schema schema(); +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCellCheck.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCellCheck.java new file mode 100644 index 0000000..3ae369d --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCellCheck.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +/** + * Expected value at one cell of a {@link DatabaseQueryCheck} result. Compared + * on the string form, case-sensitive; {@code null} {@link #expectedValue} + * asserts SQL NULL. + * + * @param name display name for reports + * @param rowIndex 0-based row index + * @param columnIndex 0-based column index + * @param expectedValue expected value as a string, or {@code null} for SQL NULL + */ +public record DatabaseCellCheck(String name, int rowIndex, int columnIndex, String expectedValue) { +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCheckSuite.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCheckSuite.java new file mode 100644 index 0000000..4e94636 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseCheckSuite.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +import java.util.List; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; + +/** + * A set of database-shape assertions to run against the loaded + * {@code DataSource}: schema/table/column existence plus SQL queries. + * + *

+ * Build the records by hand, or call {@link #fromCwm(String, Schema)} to derive + * existence checks for every table and column in a CWM {@link Schema}. + * + * @param name display name for reports + * @param schemaChecks schemas to assert present (or absent) + * @param queryChecks SQL queries to run against the DataSource + */ +public record DatabaseCheckSuite(String name, List schemaChecks, + List queryChecks) { + + public DatabaseCheckSuite { + schemaChecks = List.copyOf(schemaChecks); + queryChecks = List.copyOf(queryChecks); + } + + public DatabaseCheckSuite(String name, List schemaChecks) { + this(name, schemaChecks, List.of()); + } + + /** + * Builds a suite asserting that every table and column in the CWM + * {@code Schema} exists at its declared type. + */ + public static DatabaseCheckSuite fromCwm(String name, Schema cwmSchema) { + return new DatabaseCheckSuite(name, List.of(DatabaseSchemaCheck.fromCwm(cwmSchema)), List.of()); + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseColumnCheck.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseColumnCheck.java new file mode 100644 index 0000000..8c3d1a7 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseColumnCheck.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +import java.sql.JDBCType; +import java.util.Optional; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLDataType; + +/** + * Asserts a column exists (or is absent, per {@link #expectAbsent()}) with the + * expected SQL type and nullability. + * + * @param columnName column name in the parent table + * @param expectAbsent {@code true} → passes when the column is missing + * @param expectedType JDBC type name (e.g. {@code "INTEGER"}); empty = + * don't check + * @param expectedNullable nullable flag; empty = don't check + */ +public record DatabaseColumnCheck(String columnName, boolean expectAbsent, Optional expectedType, + Optional expectedNullable) { + + public DatabaseColumnCheck(String columnName, String expectedType) { + this(columnName, false, Optional.ofNullable(expectedType), Optional.empty()); + } + + /** + * Derives a presence check from a CWM {@link Column}, asserting its declared + * SQL type where it maps to a {@link JDBCType}. + */ + public static DatabaseColumnCheck fromCwm(Column cwmColumn) { + Optional typeName = Optional.empty(); + if (cwmColumn.getType() instanceof SQLDataType sdt) { + long n = sdt.getTypeNumber(); + if (n != 0) { + try { + typeName = Optional.of(JDBCType.valueOf((int) n).getName()); + } catch (IllegalArgumentException ignore) { + // unknown JDBC type code — skip type assertion + } + } + if (typeName.isEmpty() && sdt.getName() != null && !sdt.getName().isBlank()) { + typeName = Optional.of(sdt.getName()); + } + } + return new DatabaseColumnCheck(cwmColumn.getName(), false, typeName, Optional.empty()); + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseQueryCheck.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseQueryCheck.java new file mode 100644 index 0000000..1d7093f --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseQueryCheck.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +import java.util.List; + +/** + * A SQL query run directly against the {@code DataSource} via plain JDBC, with + * optional expectations on row count, column count and cell values. + * + * @param name display name for reports + * @param query SQL to execute + * @param expectedRowCount expected rows; {@code -1} = don't check + * @param expectedColumnCount expected columns; {@code -1} = don't check + * @param cellChecks per-cell value expectations + */ +public record DatabaseQueryCheck(String name, String query, int expectedRowCount, int expectedColumnCount, + List cellChecks) { + + public DatabaseQueryCheck { + cellChecks = List.copyOf(cellChecks); + } + + public DatabaseQueryCheck(String name, String query, List cellChecks) { + this(name, query, -1, -1, cellChecks); + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseSchemaCheck.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseSchemaCheck.java new file mode 100644 index 0000000..1994060 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseSchemaCheck.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.cwm.util.resource.relational.Schemas; + +/** + * Asserts a schema exists (or is absent, per {@link #expectAbsent()}) and + * contains the listed tables. A blank {@code schemaName} targets the default, + * unqualified schema. + */ +public record DatabaseSchemaCheck(String schemaName, boolean expectAbsent, List tableChecks) { + + public DatabaseSchemaCheck { + tableChecks = List.copyOf(tableChecks); + } + + public DatabaseSchemaCheck(String schemaName, List tableChecks) { + this(schemaName, false, tableChecks); + } + + /** + * Derives a presence check for every table in the CWM {@link Schema}. + */ + public static DatabaseSchemaCheck fromCwm(Schema cwmSchema) { + List tables = new ArrayList<>(); + for (Table t : Schemas.tables(cwmSchema)) { + tables.add(DatabaseTableCheck.fromCwm(t)); + } + return new DatabaseSchemaCheck(cwmSchema.getName() == null ? "" : cwmSchema.getName(), false, tables); + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseTableCheck.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseTableCheck.java new file mode 100644 index 0000000..fd27348 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/DatabaseTableCheck.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.api.dbcheck; + +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.cwm.util.resource.relational.ColumnSets; + +/** + * Asserts a table exists (or is absent, per {@link #expectAbsent()}) with the + * listed columns. + */ +public record DatabaseTableCheck(String tableName, boolean expectAbsent, List columnChecks) { + + public DatabaseTableCheck { + columnChecks = List.copyOf(columnChecks); + } + + public DatabaseTableCheck(String tableName, List columnChecks) { + this(tableName, false, columnChecks); + } + + /** + * Derives a presence check for every column in the CWM {@link Table}. + */ + public static DatabaseTableCheck fromCwm(Table cwmTable) { + List cols = new ArrayList<>(); + for (Column c : ColumnSets.columns(cwmTable)) { + cols.add(DatabaseColumnCheck.fromCwm(c)); + } + return new DatabaseTableCheck(cwmTable.getName(), false, cols); + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/package-info.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/package-info.java new file mode 100644 index 0000000..ed96669 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/dbcheck/package-info.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("1.0") +package org.eclipse.daanse.cwm.testkit.api.dbcheck; diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/package-info.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/package-info.java new file mode 100644 index 0000000..28ce5ef --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/api/package-info.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + */ +@org.osgi.annotation.bundle.Export +@org.osgi.annotation.versioning.Version("1.0") +package org.eclipse.daanse.cwm.testkit.api; diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/data/DataLayer.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/data/DataLayer.java new file mode 100644 index 0000000..1715eab --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/data/DataLayer.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.data; + +import java.io.InputStream; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.sql.Connection; +import java.sql.JDBCType; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.sql.DataSource; + +import org.eclipse.daanse.cwm.data.source.csv.CsvRecordPublisher; +import org.eclipse.daanse.cwm.model.cwm.resource.record.Field; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordDef; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordFactory; +import org.eclipse.daanse.cwm.model.cwm.resource.record.RecordFile; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Column; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Table; +import org.eclipse.daanse.cwm.testkit.api.DataSupplier; +import org.eclipse.daanse.cwm.util.resource.relational.ColumnSets; +import org.eclipse.daanse.cwm.util.resource.relational.Schemas; +import org.eclipse.daanse.jdbc.db.api.schema.SchemaReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.eclipse.daanse.cwm.data.api.RawRecord; +import org.eclipse.daanse.cwm.data.api.FieldMapping; +import org.eclipse.daanse.cwm.data.source.csv.record.FieldMappingR; +import org.eclipse.daanse.cwm.data.sink.jdbc.DatabaseRecordSink; + +/** + * Loads a {@link DataSupplier}'s CSV resources, then its programmatic rows, + * into tables already created from the CWM {@link Schema}. + * + *

+ * CSVs are header-only; column types come from the CWM table. CSV headers must + * match the table column names (case-insensitive). + */ +public final class DataLayer { + + private static final RecordFactory REC = RecordFactory.eINSTANCE; + private static final int DEFAULT_BATCH = 100; + private static final int CSV_LOAD_TIMEOUT_SECONDS = 60; + + private DataLayer() { + } + + /** + * Loads {@code data} into the tables of {@code cwmSchema}: CSVs first, then + * {@link DataSupplier#load}. No-op when {@code data} is {@code null}. + */ + public static void apply(DataSource dataSource, Dialect dialect, Schema cwmSchema, DataSupplier data) + throws Exception { + if (data == null) { + return; + } + Map csv = data.csvResources(); + if (csv != null && !csv.isEmpty()) { + Path tempDir = Files.createTempDirectory("daanse-testkit-data-"); + try { + for (Map.Entry e : csv.entrySet()) { + Path target = tempDir.resolve(e.getKey() + ".csv"); + try (InputStream in = e.getValue().openStream()) { + Files.copy(in, target, StandardCopyOption.REPLACE_EXISTING); + } + Table table = lookupTable(cwmSchema, e.getKey()); + if (table == null) { + throw new IllegalStateException("No CWM Table named '" + e.getKey() + "' in schema '" + + (cwmSchema.getName() == null ? "" : cwmSchema.getName()) + "' for CSV " + + e.getValue()); + } + loadCsv(dataSource, dialect, cwmSchema, table, target); + } + } finally { + deleteTree(tempDir); + } + } + try (Connection conn = dataSource.getConnection()) { + data.load(conn, dialect); + } + } + + private static Table lookupTable(Schema schema, String name) { + for (Table t : Schemas.tables(schema)) { + if (name.equalsIgnoreCase(t.getName())) { + return t; + } + } + return null; + } + + private static void loadCsv(DataSource ds, Dialect dialect, Schema schema, Table table, Path csvPath) + throws Exception { + RecordFile recordFile = REC.createRecordFile(); + recordFile.setIsSelfDescribing(true); + recordFile.setSkipRecords(0); + + RecordDef recordDef = REC.createRecordDef(); + recordDef.setFieldDelimiter(","); + recordDef.setTextDelimiter("\""); + recordDef.setIsFixedWidth(false); + for (Column c : ColumnSets.columns(table)) { + Field f = REC.createField(); + f.setName(c.getName().toLowerCase()); + recordDef.getFeature().add(f); + } + + String schemaName = schema.getName(); + TableReference tableRef = new TableReference( + (schemaName == null || schemaName.isBlank()) ? Optional.empty() + : Optional.of(new SchemaReference(Optional.empty(), schemaName)), + table.getName(), TableReference.TYPE_TABLE); + + List mappings = new ArrayList<>(); + List types = new ArrayList<>(); + for (Column c : ColumnSets.columns(table)) { + mappings.add(new FieldMappingR(c.getName().toLowerCase(), c.getName(), Optional.empty())); + types.add(jdbcTypeOf(c)); + } + + DatabaseRecordSink sink = new DatabaseRecordSink(ds, dialect, tableRef, mappings, types, DEFAULT_BATCH); + CsvRecordPublisher publisher = new CsvRecordPublisher(csvPath, recordFile, recordDef); + + CountDownLatch latch = new CountDownLatch(1); + Throwable[] error = { null }; + publisher.subscribe(new Flow.Subscriber() { + @Override + public void onSubscribe(Flow.Subscription s) { + sink.onSubscribe(s); + } + + @Override + public void onNext(RawRecord r) { + sink.onNext(r); + } + + @Override + public void onError(Throwable t) { + error[0] = t; + sink.onError(t); + latch.countDown(); + } + + @Override + public void onComplete() { + sink.onComplete(); + latch.countDown(); + } + }); + if (!latch.await(CSV_LOAD_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new IllegalStateException("CSV load for " + table.getName() + " timed out"); + } + if (error[0] != null) { + throw new RuntimeException("CSV load failed for " + table.getName(), error[0]); + } + } + + private static JDBCType jdbcTypeOf(Column c) { + if (c.getType() instanceof org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLDataType sdt) { + long n = sdt.getTypeNumber(); + if (n != 0) { + try { + return JDBCType.valueOf((int) n); + } catch (IllegalArgumentException ignore) { + // fall through + } + } + } + return JDBCType.VARCHAR; + } + + private static void deleteTree(Path root) { + try (Stream walk = Files.walk(root)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (Exception ignored) { + } + }); + } catch (Exception ignored) { + } + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseCheckExecutor.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseCheckExecutor.java new file mode 100644 index 0000000..1c73464 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseCheckExecutor.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.database; + +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import javax.sql.DataSource; + +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseCellCheck; +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseCheckSuite; +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseColumnCheck; +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseQueryCheck; +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseSchemaCheck; +import org.eclipse.daanse.cwm.testkit.api.dbcheck.DatabaseTableCheck; +import org.eclipse.daanse.jdbc.db.api.DatabaseService; +import org.eclipse.daanse.jdbc.db.api.meta.MetaInfo; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.TableDefinition; +import org.eclipse.daanse.jdbc.db.impl.DatabaseServiceImpl; +import org.junit.jupiter.api.DynamicTest; + +/** + * Runs a {@link DatabaseCheckSuite} against a live {@code DataSource} and + * returns one JUnit {@link DynamicTest} per assertion. + */ +public final class DatabaseCheckExecutor { + + private DatabaseCheckExecutor() { + } + + /** + * Builds the dynamic tests for {@code suite} against {@code dataSource}; + * {@code breadcrumb} prefixes each test name. Returns an empty list when + * {@code suite} is {@code null}. + */ + public static List execute(String breadcrumb, DataSource dataSource, DatabaseCheckSuite suite) { + if (suite == null) { + return List.of(); + } + DatabaseService databaseService = new DatabaseServiceImpl(); + MetaInfo meta; + try { + meta = databaseService.createMetaInfo(dataSource); + } catch (Exception e) { + return List.of(DynamicTest.dynamicTest(breadcrumb + " » db-check setup", () -> { + throw new AssertionError("Failed to introspect DataSource", e); + })); + } + List tests = new ArrayList<>(); + String head = breadcrumb + " » " + safe(suite.name()); + for (DatabaseSchemaCheck schemaCheck : suite.schemaChecks()) { + String schemaHead = head + " » schema=" + safe(schemaCheck.schemaName()); + checkSchema(schemaHead, meta, schemaCheck, tests); + } + for (DatabaseQueryCheck queryCheck : suite.queryChecks()) { + checkQuery(head + " » query=" + safe(queryCheck.name()), dataSource, queryCheck, tests); + } + return tests; + } + + private static void checkQuery(String head, DataSource dataSource, DatabaseQueryCheck c, List tests) { + List> rows; + int colCount; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(c.query())) { + colCount = rs.getMetaData().getColumnCount(); + rows = new ArrayList<>(); + while (rs.next()) { + List row = new ArrayList<>(colCount); + for (int i = 1; i <= colCount; i++) { + row.add(rs.getObject(i)); + } + rows.add(row); + } + } catch (Exception e) { + tests.add(DynamicTest.dynamicTest(head + " execute", () -> { + throw new AssertionError("query execution failed: " + e.getMessage(), e); + })); + return; + } + final int rowCount = rows.size(); + final int finalColCount = colCount; + if (c.expectedRowCount() >= 0) { + int expected = c.expectedRowCount(); + tests.add(DynamicTest.dynamicTest(head + " rowCount=" + expected, () -> { + if (rowCount != expected) { + throw new AssertionError("expected " + expected + " rows but got " + rowCount); + } + })); + } + if (c.expectedColumnCount() >= 0) { + int expected = c.expectedColumnCount(); + tests.add(DynamicTest.dynamicTest(head + " columnCount=" + expected, () -> { + if (finalColCount != expected) { + throw new AssertionError("expected " + expected + " columns but got " + finalColCount); + } + })); + } + for (DatabaseCellCheck cell : c.cellChecks()) { + tests.add(DynamicTest.dynamicTest( + head + " cell[" + cell.rowIndex() + "," + cell.columnIndex() + "]=" + cell.name(), () -> { + if (cell.rowIndex() < 0 || cell.rowIndex() >= rows.size()) { + throw new AssertionError( + "row " + cell.rowIndex() + " out of bounds (have " + rows.size() + " rows)"); + } + List row = rows.get(cell.rowIndex()); + if (cell.columnIndex() < 0 || cell.columnIndex() >= row.size()) { + throw new AssertionError("column " + cell.columnIndex() + " out of bounds (have " + + row.size() + " columns)"); + } + Object actual = row.get(cell.columnIndex()); + String actualStr = actual == null ? null : actual.toString(); + if (!Objects.equals(cell.expectedValue(), actualStr)) { + throw new AssertionError("expected " + cell.expectedValue() + " but got " + actualStr); + } + })); + } + } + + private static void checkSchema(String head, MetaInfo meta, DatabaseSchemaCheck c, List tests) { + boolean isDefault = c.schemaName() == null || c.schemaName().isBlank(); + if (!isDefault) { + boolean present = meta.structureInfo().schemas().stream() + .anyMatch(s -> nameEquals(s.name(), c.schemaName())); + if (c.expectAbsent()) { + tests.add(DynamicTest.dynamicTest(head + " expectAbsent", () -> { + if (present) { + throw new AssertionError("schema " + c.schemaName() + " should be absent but is present"); + } + })); + return; + } + tests.add(DynamicTest.dynamicTest(head + " present", () -> { + if (!present) { + throw new AssertionError("schema " + c.schemaName() + " is absent"); + } + })); + } + for (DatabaseTableCheck tableCheck : c.tableChecks()) { + String tableHead = head + " » table=" + safe(tableCheck.tableName()); + checkTable(tableHead, meta, c.schemaName(), tableCheck, tests); + } + } + + private static void checkTable(String head, MetaInfo meta, String schemaName, DatabaseTableCheck c, + List tests) { + Optional found = findTable(meta, schemaName, c.tableName()); + if (c.expectAbsent()) { + tests.add(DynamicTest.dynamicTest(head + " expectAbsent", () -> { + if (found.isPresent()) { + throw new AssertionError("table " + c.tableName() + " should be absent but is present"); + } + })); + return; + } + tests.add(DynamicTest.dynamicTest(head + " present", () -> { + if (found.isEmpty()) { + throw new AssertionError("table " + c.tableName() + " is absent"); + } + })); + if (found.isEmpty()) { + return; + } + for (DatabaseColumnCheck colCheck : c.columnChecks()) { + String colHead = head + " » column=" + safe(colCheck.columnName()); + checkColumn(colHead, meta, found.get(), colCheck, tests); + } + } + + private static void checkColumn(String head, MetaInfo meta, TableDefinition table, DatabaseColumnCheck c, + List tests) { + String targetTableName = table.table().name(); + Optional col = meta.structureInfo().columns().stream() + .filter(cd -> cd.column().table().map(t -> nameEquals(t.name(), targetTableName)).orElse(false)) + .filter(cd -> nameEquals(cd.column().name(), c.columnName())).findFirst(); + if (c.expectAbsent()) { + tests.add(DynamicTest.dynamicTest(head + " expectAbsent", () -> { + if (col.isPresent()) { + throw new AssertionError("column " + c.columnName() + " should be absent but is present"); + } + })); + return; + } + tests.add(DynamicTest.dynamicTest(head + " present", () -> { + if (col.isEmpty()) { + throw new AssertionError("column " + c.columnName() + " is absent"); + } + })); + if (col.isEmpty()) { + return; + } + c.expectedType() + .ifPresent(expectedType -> tests.add(DynamicTest.dynamicTest(head + " type=" + expectedType, () -> { + JDBCType actual = col.get().columnMetaData().dataType(); + if (!actual.getName().equalsIgnoreCase(expectedType)) { + throw new AssertionError("column " + c.columnName() + " expected type " + expectedType + + " but found " + actual.getName()); + } + }))); + c.expectedNullable().ifPresent( + expectedNullable -> tests.add(DynamicTest.dynamicTest(head + " nullable=" + expectedNullable, () -> { + boolean nullable = col.get().columnMetaData() + .nullability() == org.eclipse.daanse.jdbc.db.api.schema.ColumnMetaData.Nullability.NULLABLE; + if (nullable != expectedNullable) { + throw new AssertionError("column " + c.columnName() + " expected nullable=" + expectedNullable + + " but found nullable=" + nullable); + } + }))); + } + + private static Optional findTable(MetaInfo meta, String schemaName, String tableName) { + return meta.structureInfo().tables().stream().filter(td -> nameEquals(td.table().name(), tableName)) + .filter(td -> { + if (schemaName == null || schemaName.isBlank()) { + return true; + } + return td.table().schema().filter(s -> nameEquals(s.name(), schemaName)).isPresent(); + }).findFirst(); + } + + private static boolean nameEquals(String a, String b) { + if (a == null) { + return b == null || b.isBlank(); + } + return a.equalsIgnoreCase(b); + } + + private static String safe(String s) { + return (s == null || s.isBlank()) ? "" : s; + } +} diff --git a/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseLayer.java b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseLayer.java new file mode 100644 index 0000000..36789a4 --- /dev/null +++ b/testkit/src/main/java/org/eclipse/daanse/cwm/testkit/database/DatabaseLayer.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * SmartCity Jena, Stefan Bischof - initial + */ +package org.eclipse.daanse.cwm.testkit.database; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Set; + +import javax.sql.DataSource; + +import org.eclipse.daanse.cwm.resource.relational.ddl.api.DdlSettings; +import org.eclipse.daanse.cwm.resource.relational.ddl.api.Feature; +import org.eclipse.daanse.cwm.resource.relational.ddl.internal.DdlGeneratorFactoryImpl; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.Schema; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; + +/** + * Creates the tables (and keys, indexes, views) of a CWM {@link Schema} in a + * JDBC {@link DataSource} using the dialect's DDL generator. + */ +public final class DatabaseLayer { + + private DatabaseLayer() { + } + + /** + * Creates the full schema (all features). + */ + public static void apply(DataSource dataSource, Dialect dialect, Schema schema) throws SQLException { + apply(dataSource, dialect, schema, Feature.ALL); + } + + /** + * Creates only the listed features, e.g. {@code SCHEMA, TABLE, + * PRIMARY_KEY} to skip indexes or foreign keys. + */ + public static void apply(DataSource dataSource, Dialect dialect, Schema schema, Set features) + throws SQLException { + apply(dataSource, dialect, schema, features, DdlSettings.defaults()); + } + + /** + * Creates the listed features with explicit {@link DdlSettings}, e.g. to drop + * schema qualification for connection-scoped databases. + */ + public static void apply(DataSource dataSource, Dialect dialect, Schema schema, Set features, + DdlSettings settings) throws SQLException { + List ddl = new DdlGeneratorFactoryImpl().create(dialect, settings).createSchema(schema, features); + if (ddl.isEmpty()) { + return; + } + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + for (String sql : ddl) { + stmt.execute(sql); + } + } + } +} diff --git a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/Indexes.java b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/Indexes.java index 92ef9d8..9750466 100644 --- a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/Indexes.java +++ b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/Indexes.java @@ -56,6 +56,9 @@ public static List columns(SQLIndex idx) { } public static Stream columnStream(SQLIndex idx) { + if (idx == null) { + return Stream.empty(); + } return idx.getIndexedFeature().stream() .map(IndexedFeature::getFeature) .filter(Column.class::isInstance) diff --git a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/NamedColumnSets.java b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/NamedColumnSets.java index ed090ef..9519513 100644 --- a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/NamedColumnSets.java +++ b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/NamedColumnSets.java @@ -53,12 +53,9 @@ public static String qualifiedName(NamedColumnSet columnSet) { if (columnSet == null) { return null; } - Optional schema = findSchema(columnSet); - Optional catalog = schema.flatMap(Schemas::findCatalog); - StringBuilder sb = new StringBuilder(); - catalog.ifPresent(c -> sb.append(c.getName()).append('.')); - schema.ifPresent(s -> sb.append(s.getName()).append('.')); - sb.append(columnSet.getName()); - return sb.toString(); + return findSchema(columnSet) + .map(Schemas::qualifiedName) + .map(prefix -> prefix + "." + columnSet.getName()) + .orElseGet(columnSet::getName); } } diff --git a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/SqlSimpleTypes.java b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/SqlSimpleTypes.java index a5efdca..6696116 100644 --- a/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/SqlSimpleTypes.java +++ b/util/src/main/java/org/eclipse/daanse/cwm/util/resource/relational/SqlSimpleTypes.java @@ -13,13 +13,16 @@ */ package org.eclipse.daanse.cwm.util.resource.relational; +import java.sql.JDBCType; import java.sql.Types; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.function.Supplier; import org.eclipse.daanse.cwm.model.cwm.resource.relational.RelationalFactory; +import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLDataType; import org.eclipse.daanse.cwm.model.cwm.resource.relational.SQLSimpleType; import org.eclipse.emf.ecore.util.EcoreUtil; @@ -150,21 +153,107 @@ public static int jdbcType(SQLSimpleType type) { return type == null ? Types.OTHER : (int) type.getTypeNumber(); } - public static boolean isNumeric(SQLSimpleType type) { - switch (jdbcType(type)) { - case Types.TINYINT: - case Types.SMALLINT: - case Types.INTEGER: - case Types.BIGINT: - case Types.REAL: - case Types.FLOAT: - case Types.DOUBLE: - case Types.DECIMAL: - case Types.NUMERIC: - return true; - default: - return false; + /** + * Best-effort {@link JDBCType} for a CWM {@link SQLDataType}: prefers the + * explicit type-number, then falls back to a name match. This is the pure + * CWM → JDBC type-code direction (no JDBC runtime types involved). + */ + public static JDBCType toJdbcType(SQLDataType sqlType) { + if (sqlType == null) { + return JDBCType.OTHER; + } + long typeNumber = sqlType.getTypeNumber(); + if (typeNumber != 0) { + try { + return JDBCType.valueOf((int) typeNumber); + } catch (IllegalArgumentException ignore) { + // fall through to name-based match + } + } + String name = sqlType.getName(); + if (name == null) { + return JDBCType.OTHER; + } + return switch (name.trim().toUpperCase()) { + case "BOOLEAN", "BOOL", "BIT" -> JDBCType.BOOLEAN; + case "TINYINT" -> JDBCType.TINYINT; + case "SMALLINT" -> JDBCType.SMALLINT; + case "INT", "INTEGER" -> JDBCType.INTEGER; + case "BIGINT" -> JDBCType.BIGINT; + case "REAL", "FLOAT" -> JDBCType.REAL; + case "DOUBLE", "DOUBLE PRECISION" -> JDBCType.DOUBLE; + case "DECIMAL", "NUMERIC" -> JDBCType.DECIMAL; + case "DATE" -> JDBCType.DATE; + case "TIME" -> JDBCType.TIME; + case "TIMESTAMP", "DATETIME" -> JDBCType.TIMESTAMP; + case "CHAR", "CHARACTER" -> JDBCType.CHAR; + case "VARCHAR", "CHARACTER VARYING" -> JDBCType.VARCHAR; + case "CLOB", "TEXT" -> JDBCType.CLOB; + case "BLOB" -> JDBCType.BLOB; + case "BINARY" -> JDBCType.BINARY; + case "VARBINARY" -> JDBCType.VARBINARY; + default -> JDBCType.OTHER; + }; + } + + /** + * Build a CWM {@link SQLSimpleType} from the parts of a JDBC column-metadata + * row, reusing the SQL-99 registry where {@code typeName} is known and + * overriding the type-number with the actual JDBC {@code dataType} code. + * Takes plain values (no JDBC metadata wrapper) so this stays the pure + * JDBC → CWM type direction without a jdbc.db dependency. + * + * @param typeName DBMS type name (e.g. {@code "varchar"}), may be null + * @param dataType JDBC type code, may be null + * @param columnSize column size / precision, if reported + * @param decimalDigits scale, if reported + */ + public static SQLSimpleType toCwmType(String typeName, JDBCType dataType, + OptionalInt columnSize, OptionalInt decimalDigits) { + SQLSimpleType t = byName(typeName).orElse(null); + if (t == null) { + t = RelationalFactory.eINSTANCE.createSQLSimpleType(); + t.setName(typeName == null ? jdbcName(dataType) : typeName); + t.setTypeNumber(dataType == null ? Types.OTHER : dataType.getVendorTypeNumber()); + } else if (dataType != null) { + // Override the type-number with the actual JDBC code so it survives + // dialects that report a different keyword (e.g. PG returns 'int4'). + t.setTypeNumber(dataType.getVendorTypeNumber()); + } + if (columnSize != null && columnSize.isPresent()) { + int s = columnSize.getAsInt(); + if (isNumericJdbc(dataType)) { + t.setNumericPrecision(s); + } else { + t.setCharacterMaximumLength(s); + } } + if (decimalDigits != null && decimalDigits.isPresent()) { + t.setNumericScale(decimalDigits.getAsInt()); + } + return t; + } + + private static boolean isNumericJdbc(JDBCType t) { + return t != null && isNumericCode(t.getVendorTypeNumber()); + } + + private static String jdbcName(JDBCType t) { + return t == null ? "OTHER" : t.getName(); + } + + public static boolean isNumeric(SQLSimpleType type) { + return isNumericCode(jdbcType(type)); + } + + /** Whether {@code jdbcCode} (a {@link Types} constant) is a numeric type. */ + private static boolean isNumericCode(int jdbcCode) { + return switch (jdbcCode) { + case Types.TINYINT, Types.SMALLINT, Types.INTEGER, Types.BIGINT, + Types.REAL, Types.FLOAT, Types.DOUBLE, Types.DECIMAL, Types.NUMERIC -> + true; + default -> false; + }; } public static boolean isText(SQLSimpleType type) {