diff --git a/mondrian/pom.xml b/mondrian/pom.xml index 054fcc6ade..143880a3db 100644 --- a/mondrian/pom.xml +++ b/mondrian/pom.xml @@ -302,6 +302,14 @@ test + + org.eclipse.daanse + + org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal + 0.0.1-SNAPSHOT + test + + org.eclipse.daanse @@ -392,6 +400,12 @@ 0.0.1-SNAPSHOT compile + + org.eclipse.daanse + org.eclipse.daanse.olap.check.runtime + 0.0.1-SNAPSHOT + test + org.eclipse.daanse org.eclipse.daanse.olap.spi diff --git a/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExecution.java b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExecution.java new file mode 100644 index 0000000000..70730307f2 --- /dev/null +++ b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExecution.java @@ -0,0 +1,38 @@ +package org.eclipse.daanse.olap.check; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.eclipse.daanse.olap.api.Context; +import org.eclipse.daanse.olap.check.model.check.CheckExecutionResult; +import org.eclipse.daanse.olap.check.model.check.CheckResult; +import org.eclipse.daanse.olap.check.runtime.api.CheckExecutor; +import org.eclipse.daanse.olap.check.runtime.impl.CheckExecutorImpl; +import org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal.CheckSuiteSupplier; +import org.eclipse.daanse.rolap.mapping.model.provider.CatalogMappingSupplier; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class CheckExecution { + private CheckExecutor executor = new CheckExecutorImpl(); + public final List RESULTS = Collections.synchronizedList(new ArrayList<>()); + + public void printResult(ExtensionContext context) { + System.out.println("Test result for " + context.getDisplayName()); + System.out.println("_____________________________________________"); + for (CheckExecutionResult result : RESULTS) { + System.out.println(result.getName() + " SuccessCount " + result.getSuccessCount() + " FailureCount " + + result.getFailureCount()); + for (CheckResult chr : result.getCheckResults()) { + System.out.println(" " + chr.getCheckName() + " Status " + chr.getStatus().getName() + " ExecutionTimeMs " + chr.getExecutionTimeMs()); + } + } + System.out.println("_____________________________________________"); + } + + public void execute(Context context, + CheckSuiteSupplier checkSuiteSupplier, + CatalogMappingSupplier catalogSupplier) { + List result = executor.execute(checkSuiteSupplier.get(), context); + RESULTS.addAll(result); + } +} diff --git a/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExtension.java b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExtension.java new file mode 100644 index 0000000000..54567e76ac --- /dev/null +++ b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckExtension.java @@ -0,0 +1,190 @@ +package org.eclipse.daanse.olap.check; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Enumeration; +import java.util.Map.Entry; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +import javax.sql.DataSource; + +import org.eclipse.daanse.jdbc.db.api.SqlStatementGenerator; +import org.eclipse.daanse.jdbc.db.api.meta.MetaInfo; +import org.eclipse.daanse.jdbc.db.core.DatabaseServiceImpl; +import org.eclipse.daanse.jdbc.db.core.SqlStatementGeneratorImpl; +import org.eclipse.daanse.jdbc.db.dialect.api.Dialect; +import org.eclipse.daanse.olap.api.Context; +import org.eclipse.daanse.olap.check.runtime.api.OlapCheckSuiteSupplier; +import org.eclipse.daanse.rolap.common.aggregator.AggregationFactoryImpl; +import org.eclipse.daanse.rolap.mapping.model.provider.CatalogMappingSupplier; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.api.extension.TestWatcher; +import org.opencube.junit5.context.TestContextImpl; +import org.opencube.junit5.dbprovider.H2DatabaseProvider; + +public class CheckExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, + TestWatcher, ParameterResolver { + private TestContextImpl context; + private static final Namespace NS = Namespace.create(CheckExecution.class); + + /* ===== SUITE / CLASS ===== */ + + @Override + public void beforeAll(ExtensionContext context) { + CheckSuite suite = CheckSuite.start(context.getRequiredTestClass().getName()); + context.getStore(NS).put("suite", suite); + } + + @Override + public void afterAll(ExtensionContext context) { + CheckSuite suite = context.getStore(NS).remove("suite", CheckSuite.class); + if (suite != null) + suite.printResult(); + } + + /* ===== TEST ===== */ + + @Override + public void beforeEach(ExtensionContext context) { + CheckSuite suite = context.getStore(NS).get("suite", CheckSuite.class); + + CheckExecution exec = suite.startExecution(context.getUniqueId(), context.getDisplayName()); + + context.getStore(NS).put(execKey(context), exec); + } + + @Override + public void afterEach(ExtensionContext context) { + CheckExecution exec = context.getStore(NS).remove(execKey(context), CheckExecution.class); + if (exec != null) { + exec.printResult(context); + } + } + + /* ===== RESULT ===== */ + + @Override + public void testFailed(ExtensionContext context, Throwable cause) { + CheckExecution exec = context.getStore(NS).get(execKey(context), CheckExecution.class); + if (exec != null) { + // exec.markFailed(cause); + } + } + + /* ===== CALLBACK ===== */ + + @Override + public boolean supportsParameter(ParameterContext pc, ExtensionContext ec) { + Class[] classes = pc.getParameter().getType().getInterfaces(); + return pc.getParameter().getType() == CheckExecution.class + || checkClasses(classes, OlapCheckSuiteSupplier.class) || pc.getParameter().getType() == Context.class + || checkClasses(classes, CatalogMappingSupplier.class); + } + + private boolean checkClasses(Class[] classes, Class cl) { + if (classes != null) { + for (Class c : classes) { + if (c == cl) { + return true; + } + } + } + return false; + } + + @Override + public Object resolveParameter(ParameterContext pc, ExtensionContext ec) { + + if (pc.getParameter().getType() == CheckExecution.class) { + CheckExecution exec = ec.getStore(NS).get(execKey(ec), CheckExecution.class); + return exec; + } + if (pc.getParameter().getType() == Context.class && pc.getIndex() == 1) { + H2DatabaseProvider dp = new H2DatabaseProvider(); // TODO use other database type too + Entry dataBaseInfo = dp.activate(); + context = new TestContextImpl(); + context.setDataSource(dataBaseInfo.getKey()); + context.setDialect(dataBaseInfo.getValue()); + context.setAggragationFactory( + new AggregationFactoryImpl(dataBaseInfo.getValue(), context.getCustomAggregators())); + context.setName("TestContext"); + return context; + } + if (checkClasses(pc.getParameter().getType().getInterfaces(), OlapCheckSuiteSupplier.class) && pc.getIndex() == 2) { + try { + return pc.getParameter().getType().getConstructor().newInstance(); + } catch (Exception e) { + new ParameterResolutionException("Unsupported parameter"); + } + } + if (checkClasses(pc.getParameter().getType().getInterfaces(), CatalogMappingSupplier.class) && pc.getIndex() == 3) { + try { + CatalogMappingSupplier catalogMappingSupplier = (CatalogMappingSupplier) pc.getParameter().getType() + .getConstructor().newInstance(); + context.setCatalogMappingSupplier(catalogMappingSupplier); + + // load data + loadDatafromPackage(pc.getParameter().getType(), "data"); + + return catalogMappingSupplier; + } catch (Exception e) { + new ParameterResolutionException("Unsupported parameter"); + } + } + throw new ParameterResolutionException("Unsupported parameter"); + } + + public void loadDatafromPackage(Class c, String path) throws Exception { + java.net.URL resource = c.getResource(""); + + if (resource == null) { + return; // resource not find + } + + // important: JAR-файлы use protocol "jar:" + if ("jar".equals(resource.getProtocol())) { + // get path to JAR from URL + String jarPath = resource.getPath().substring(0, resource.getPath().indexOf("!")); + java.net.URL jarUrl = new java.net.URL(jarPath); + try (java.util.jar.JarFile jar = new JarFile(jarUrl.getFile())) { + Enumeration entries = jar.entries(); + + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + // filter by path + if (entryName.startsWith(path) && !entryName.equals(path + "/")) { + // remove '/' + String resourceName = entryName.substring(path.length() + 1); + if (!resourceName.isEmpty()) { + InputStream is = jar.getInputStream(entry); + try (Connection connection = context.getDataSource().getConnection()) { + DatabaseServiceImpl databaseService = new DatabaseServiceImpl(); + MetaInfo metaInfo = databaseService.createMetaInfo(connection); + SqlStatementGenerator sqlStatementGenerator = new SqlStatementGeneratorImpl(metaInfo); + DataLoadUtil.loadTable(connection, sqlStatementGenerator, is, resourceName); + } catch (SQLException e) { + throw new RuntimeException("Database connection error", e); + } + } + } + } + } + } + + } + + private String execKey(ExtensionContext ctx) { + return "exec:" + ctx.getUniqueId(); + } +} diff --git a/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckSuite.java b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckSuite.java new file mode 100644 index 0000000000..8c5acb5538 --- /dev/null +++ b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckSuite.java @@ -0,0 +1,16 @@ +package org.eclipse.daanse.olap.check; + +public class CheckSuite { + + static CheckSuite start(String name) { return new CheckSuite(); } + + public CheckExecution startExecution(String uniqueId, String displayName) { + + return new CheckExecution(); + } + + public void printResult() { + + + } +} diff --git a/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckTests.java b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckTests.java new file mode 100644 index 0000000000..58eb371410 --- /dev/null +++ b/mondrian/src/test/java/org/eclipse/daanse/olap/check/CheckTests.java @@ -0,0 +1,20 @@ +package org.eclipse.daanse.olap.check; + +import org.eclipse.daanse.olap.api.Context; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(CheckExtension.class) +public class CheckTests { + + @Test + void test1(CheckExecution checkExecution, Context context, org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal.CheckSuiteSupplier checkSuiteSupplier, org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal.CatalogSupplier catalogSupplier) { + checkExecution.execute(context, checkSuiteSupplier, catalogSupplier); + } + + @Test + void test2(CheckExecution checkExecution, Context context, org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal.CheckSuiteSupplier checkSuiteSupplier, org.eclipse.daanse.rolap.mapping.instance.emf.tutorial.cube.minimal.CatalogSupplier catalogSupplier) { + checkExecution.execute(context, checkSuiteSupplier, catalogSupplier); + } + +} diff --git a/mondrian/src/test/java/org/eclipse/daanse/olap/check/DataLoadUtil.java b/mondrian/src/test/java/org/eclipse/daanse/olap/check/DataLoadUtil.java new file mode 100644 index 0000000000..bb032c66fc --- /dev/null +++ b/mondrian/src/test/java/org/eclipse/daanse/olap/check/DataLoadUtil.java @@ -0,0 +1,314 @@ +package org.eclipse.daanse.olap.check; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.sql.Connection; +import java.sql.Date; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; + +import org.eclipse.daanse.jdbc.db.api.SqlStatementGenerator; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnMetaData; +import org.eclipse.daanse.jdbc.db.api.schema.ColumnReference; +import org.eclipse.daanse.jdbc.db.api.schema.SchemaReference; +import org.eclipse.daanse.jdbc.db.api.schema.TableDefinition; +import org.eclipse.daanse.jdbc.db.api.schema.TableReference; +import org.eclipse.daanse.jdbc.db.api.sql.InsertSqlStatement; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnDefinitionR; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnMetaDataR; +import org.eclipse.daanse.jdbc.db.record.schema.ColumnReferenceR; +import org.eclipse.daanse.jdbc.db.record.schema.TableDefinitionR; +import org.eclipse.daanse.jdbc.db.record.schema.TableReferenceR; +import org.eclipse.daanse.jdbc.db.record.sql.CreateContainerSqlStatementR; +import org.eclipse.daanse.jdbc.db.record.sql.CreateSchemaSqlStatementR; +import org.eclipse.daanse.jdbc.db.record.sql.DropContainerSqlStatementR; +import org.eclipse.daanse.jdbc.db.record.sql.InsertSqlStatementR; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.siegmar.fastcsv.reader.CloseableIterator; +import de.siegmar.fastcsv.reader.CsvReader; +import de.siegmar.fastcsv.reader.NamedCsvRecord; + +public class DataLoadUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(DataLoadUtil.class); + private static final String EXCEPTION_WHILE_WRITING_DATA = "Exception while writing Data"; + + private static final String EXCEPTION_WHILE_CREATING_SCHEMA = "Exception while creating schema"; + + private static final String EXCEPTION_WHILE_SETTING_VALUE_TO_PREPARED_STATEMENT = "Exception while setting value to PreparedStatement"; + + + public static void loadTable(Connection connection, SqlStatementGenerator sqlStatementGenerator, InputStream is, String path) throws SQLException { + String fileName = getFileNameWithoutExtension(path.toString()); + LOGGER.debug("Load table {}", fileName); + Optional schema = getSchemaFromPath(path); + + schema.ifPresent(s -> { + + if (s.name().isBlank()) { + return; + } + + String statementCreateSchema = sqlStatementGenerator + .getSqlOfStatement(new CreateSchemaSqlStatementR(s, true)); + + try { + connection.createStatement().execute(statementCreateSchema); + } catch (SQLException e) { + // https://github.com/h2database/h2database/issues/4188 + // throw new CsvDataImporterException(EXCEPTION_WHILE_CREATING_SCHEMA, e); + LOGGER.error(EXCEPTION_WHILE_CREATING_SCHEMA, e); + } + }); + + TableReference tableRef = new TableReferenceR(schema, fileName, "TABLE"); + TableDefinition tableDefinition=new TableDefinitionR(tableRef); + dropTable(connection, sqlStatementGenerator, tableRef); + + CsvReader.CsvReaderBuilder builder = CsvReader.builder().fieldSeparator(',') + .quoteCharacter('"').skipEmptyLines(true) + .commentCharacter('#') + .ignoreDifferentFieldCount(true); + + try (CloseableIterator it = builder.ofNamedCsvRecord(new InputStreamReader(is)).iterator()) { + if (!it.hasNext()) { + throw new IllegalStateException("No header found"); + } + NamedCsvRecord types = it.next(); + List headersTypeList = getHeadersTypeList(types); + if (it.hasNext()) { + createTable(connection, sqlStatementGenerator, headersTypeList, tableDefinition); + insertTable(connection, sqlStatementGenerator, it, headersTypeList, tableRef); + } + + } catch (IOException e) { + throw new RuntimeException("Exception while Loading csv", e); + } + } + + private static String getFileNameWithoutExtension(String fileName) { + if (fileName.contains(".")) { + return fileName.substring(0, fileName.lastIndexOf(".")); + } else { + return fileName; + } + } + + private static Optional getSchemaFromPath(String path) { + return Optional.empty(); + } + + private static void dropTable(Connection connection, SqlStatementGenerator sqlStatementGenerator, TableReference table) throws SQLException { + try { + + String sqlDropTable = sqlStatementGenerator.getSqlOfStatement(new DropContainerSqlStatementR(table, true)); + try (Statement stmnt = connection.createStatement()) { + stmnt.execute(sqlDropTable); + } + + } catch (SQLException e) { + throw new RuntimeException("Exception while drop Table", e); + } + } + + private static List getHeadersTypeList(NamedCsvRecord types) { + List result = new ArrayList<>(); + if (types != null) { + for (String header : types.getHeader()) { + ColumnMetaDataR sqlType = parseColumnDataType(types.getField(header)); + ColumnDefinition dbc = new ColumnDefinitionR(new ColumnReferenceR(header), sqlType); + result.add(dbc); + } + } + return result; + } + + private static void insertTable(Connection connection, SqlStatementGenerator sqlStatementGenerator, CloseableIterator it, + List headersTypeList, TableReference table) throws SQLException { + + List columns = headersTypeList.stream().map(ColumnDefinition::column).toList(); + List values = headersTypeList.stream().map(c -> "?").toList(); + InsertSqlStatement insertSqlStatement = new InsertSqlStatementR(table, columns, values); + + String sql = sqlStatementGenerator.getSqlOfStatement(insertSqlStatement); + + try (PreparedStatement ps = connection.prepareStatement(sql)) { + batchExecute(connection, ps, it, headersTypeList); + } catch (SQLException e) { + throw new RuntimeException(EXCEPTION_WHILE_WRITING_DATA, e); + } + } + + private static void batchExecute(Connection connection, PreparedStatement ps, CloseableIterator it, + List columns) throws SQLException { + + connection.setAutoCommit(false); + long start = System.currentTimeMillis(); + int count = 0; + while (it.hasNext()) { + NamedCsvRecord r = it.next(); + + int colIndex = 1; + for (ColumnDefinition columnDefinition : columns) { + processingTypeValues(ps, columnDefinition, colIndex++, r); + } + ps.addBatch(); + ps.clearParameters(); + if (count % 1000 == 0) { + ps.executeBatch(); + LOGGER.debug("execute batch time {}", (System.currentTimeMillis() - start)); + ps.getConnection().commit(); + LOGGER.debug("execute commit time {}", (System.currentTimeMillis() - start)); + start = System.currentTimeMillis(); + } + count++; + } + + ps.executeBatch(); + LOGGER.debug("execute batch time {}", (System.currentTimeMillis() - start)); + + connection.commit(); + LOGGER.debug("execute commit time {}", (System.currentTimeMillis() - start)); + connection.setAutoCommit(true); + } + + private static void processingTypeValues(PreparedStatement ps, ColumnDefinition columnDefinition, int index, + NamedCsvRecord r) throws SQLException { + + ColumnReference column = columnDefinition.column(); + String field = r.getField(column.name()); + + try { + setPrepareStatement(ps, index, columnDefinition, field); + } catch (SQLException e) { + throw new RuntimeException(EXCEPTION_WHILE_SETTING_VALUE_TO_PREPARED_STATEMENT, e); + } + } + + private static void setPrepareStatement(PreparedStatement ps, int index, ColumnDefinition columnDefinition, String field) + throws SQLException { + + ColumnMetaData type = columnDefinition.columnMetaData(); + + if (field == null || field.equals("NULL")) { + ps.setObject(index, null); + return; + } + switch (type.dataType()) { + case BOOLEAN: { + ps.setBoolean(index, field.equals("") ? Boolean.FALSE : Boolean.valueOf(field)); + return; + } + case BIGINT: { + ps.setLong(index, field.equals("") ? 0l : Long.valueOf(field)); + return; + } + case DATE: { + ps.setDate(index, Date.valueOf(field)); + return; + } + case INTEGER: { + ps.setInt(index, field.equals("") ? 0 : Integer.valueOf(field)); + return; + } + case DECIMAL: { + ps.setDouble(index, field.equals("") ? 0.0 : Double.valueOf(field)); + return; + } + case NUMERIC: { + ps.setDouble(index, field.equals("") ? 0.0 : Double.valueOf(field)); + return; + } + case REAL: { + ps.setDouble(index, field.equals("") ? 0.0 : Double.valueOf(field)); + return; + } + case SMALLINT: { + ps.setShort(index, field.equals("") ? 0 : Short.valueOf(field)); + return; + } + case TIMESTAMP: { + ps.setTimestamp(index, Timestamp.valueOf(field)); + return; + } + case TIME: { + ps.setTime(index, Time.valueOf(field)); + return; + } + case VARCHAR: { + ps.setString(index, field); + return; + } + + default: + ps.setString(index, field); + } + } + + private static ColumnMetaDataR parseColumnDataType(String stringType) { + int indexStart = stringType.indexOf("("); + int indexEnd = stringType.indexOf(")"); + + String sType = null; + + String detail = null; + if (indexStart > 0) { + sType = stringType.substring(0, indexStart); + detail = stringType.substring(indexStart + 1, indexEnd); + } else { + sType = stringType; + } + + String[] det = detail == null ? new String[] {} : detail.split("\\."); + + JDBCType jdbcType = JDBCType.valueOf(sType); + + if (jdbcType == null) { + jdbcType = JDBCType.VARCHAR; + } + + OptionalInt columnSize = OptionalInt.empty(); + OptionalInt decimalDigits = OptionalInt.empty(); + + if (det.length > 0) { + columnSize = OptionalInt.of(Integer.parseInt(det[0])); + if (det.length > 1) { + decimalDigits = OptionalInt.of(Integer.parseInt(det[1])); + } + } + + return new ColumnMetaDataR(jdbcType, jdbcType.getName(), columnSize, decimalDigits, OptionalInt.empty(), + OptionalInt.empty(), ColumnMetaData.Nullability.NULLABLE, OptionalInt.empty(), + java.util.Optional.empty(), java.util.Optional.empty(), + ColumnMetaData.AutoIncrement.NO, ColumnMetaData.GeneratedColumn.NO); + } + + public static void createTable(Connection connection, SqlStatementGenerator sqlStatementGenerator, List headersTypeList, TableDefinition table) + throws SQLException { + try (Statement stmt = connection.createStatement();) { + + CreateContainerSqlStatementR statement = new CreateContainerSqlStatementR(table, headersTypeList, true); + + LOGGER.debug("Created table in given database. {}", statement); + + String sql = sqlStatementGenerator.getSqlOfStatement(statement); + stmt.execute(sql); + connection.commit(); + } catch (SQLException e) { + throw new RuntimeException("Exception wile create table", e); + } + + } + +}