diff --git a/.gitignore b/.gitignore index 47eb39b3..465a733e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ *.*# core !core/ +.idea/ tags .DS_Store *.debug diff --git a/AWS_build_script.txt b/AWS_build_script.txt index 7c66802d..98df76c0 100644 --- a/AWS_build_script.txt +++ b/AWS_build_script.txt @@ -1,20 +1,20 @@ -#ODK Tools - Installation script tested in Ubuntu server 20.04 +#ODK Tools - Installation script tested in Ubuntu server 22.04 sudo add-apt-repository universe sudo add-apt-repository multiverse -sudo apt-get update -sudo apt-get install build-essential qt5-default qtbase5-private-dev qtdeclarative5-dev libqt5sql5-mysql libqt5sql5-sqlite cmake mongodb jq libboost-all-dev unzip zlib1g-dev automake npm sqlite3 libqt5sql5-sqlite golang-go -sudo wget https://dev.mysql.com/get/mysql-apt-config_0.8.17-1_all.deb -sudo dpkg -i ./mysql-apt-config_0.8.17-1_all.deb +wget https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb +sudo dpkg -i ./mysql-apt-config_0.8.32-1_all.deb + sudo apt-get update -sudo apt-get install mysql-shell + +sudo apt-get install -y build-essential qtbase5-dev qtbase5-private-dev qtdeclarative5-dev libqt5sql5-mysql cmake jq libboost-all-dev unzip zlib1g-dev automake libmysqlclient-dev mysql-client sqlite3 libqt5sql5-sqlite golang-go mysql-shell openjdk-17-jre-headless cd /opt sudo wget https://github.com/BurntSushi/xsv/releases/download/0.13.0/xsv-0.13.0-x86_64-unknown-linux-musl.tar.gz -cd /bin -sudo tar xvfz /opt/xsv-0.13.0-x86_64-unknown-linux-musl.tar.gz +sudo tar xvfz xsv-0.13.0-x86_64-unknown-linux-musl.tar.gz +sudo cp xsv /bin cd /opt @@ -26,46 +26,30 @@ sudo cp csv2xlsx /bin sudo git clone https://github.com/qlands/odktools.git sudo mkdir odktools-deps cd odktools-deps -sudo wget https://github.com/mongodb/mongo-c-driver/releases/download/1.21.1/mongo-c-driver-1.21.1.tar.gz -sudo wget https://github.com/mongodb/mongo-cxx-driver/releases/download/r3.6.7/mongo-cxx-driver-r3.6.7.tar.gz -sudo wget https://github.com/jmcnamara/libxlsxwriter/archive/refs/tags/RELEASE_1.1.4.tar.gz -sudo wget https://github.com/stachenov/quazip/archive/refs/tags/v1.3.tar.gz + +sudo wget https://github.com/jmcnamara/libxlsxwriter/archive/refs/tags/v1.1.8.tar.gz +sudo wget https://github.com/stachenov/quazip/archive/refs/tags/v1.4.tar.gz sudo git clone https://github.com/rgamble/libcsv.git -sudo tar xvfz mongo-c-driver-1.21.1.tar.gz -cd /opt/odktools-deps/mongo-c-driver-1.21.1 -sudo mkdir build_here -cd /opt/odktools-deps/mongo-c-driver-1.21.1/build_here -sudo cmake .. -sudo make -sudo make install -cd /opt/odktools-deps -sudo tar xvfz mongo-cxx-driver-r3.6.7.tar.gz -cd /opt/odktools-deps/mongo-cxx-driver-r3.6.7 -sudo mkdir build_here -cd /opt/odktools-deps/mongo-cxx-driver-r3.6.7/build_here -sudo cmake -DCMAKE_C_FLAGS:STRING="-O2 -fPIC" -DCMAKE_CXX_FLAGS:STRING="-O2 -fPIC" -DBSONCXX_POLY_USE_BOOST=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local .. -sudo make -sudo make install cd /opt/odktools-deps -sudo tar xvfz v1.3.tar.gz -cd /opt/odktools-deps/quazip-1.3 +sudo tar xvfz v1.4.tar.gz +cd /opt/odktools-deps/quazip-1.4 sudo mkdir build -cd /opt/odktools-deps/quazip-1.3/build +cd /opt/odktools-deps/quazip-1.4/build sudo cmake -DCMAKE_C_FLAGS:STRING="-fPIC" -DCMAKE_CXX_FLAGS:STRING="-fPIC" .. sudo make sudo make install cd /opt/odktools-deps -sudo ln -s /usr/bin/aclocal-1.15 /usr/bin/aclocal-1.14 -sudo ln -s /usr/bin/automake-1.15 /usr/bin/automake-1.14 +sudo ln -s /usr/bin/aclocal-1.16 /usr/bin/aclocal-1.14 +sudo ln -s /usr/bin/automake-1.16 /usr/bin/automake-1.14 -sudo tar xvfz RELEASE_0.7.6.tar.gz -cd libxlsxwriter-RELEASE_0.7.6 +sudo tar xvfz v1.1.8.tar.gz +cd libxlsxwriter-1.1.8 sudo mkdir build cd build sudo cmake .. diff --git a/JSONToMySQL/mainclass.cpp b/JSONToMySQL/mainclass.cpp index 3672c035..a435064f 100644 --- a/JSONToMySQL/mainclass.cpp +++ b/JSONToMySQL/mainclass.cpp @@ -31,8 +31,9 @@ mainClass::mainClass(QObject *parent) : QObject(parent) void mainClass::run() { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) sqlStream.setCodec("UTF-8"); - +#endif recordMap = QDomDocument("ODKRecordMapFile"); recordMapRoot = recordMap.createElement("ODKRecordMapXML"); recordMapRoot.setAttribute("version", "1.0"); @@ -217,7 +218,9 @@ void mainClass::run() if (XMLLogFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream strXMLLog(&XMLLogFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) strXMLLog.setCodec("UTF-8"); +#endif xmlLog.save(strXMLLog,1,QDomNode::EncodingFromTextStream); XMLLogFile.close(); } @@ -246,9 +249,9 @@ void mainClass::run() } UUIDFile.close(); - QFileInfo fi(json); + //QFileInfo fi(json); // Inserting into the submission database - QString sql; + //QString sql; if (outputType == "h") { @@ -276,7 +279,9 @@ void mainClass::run() if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif recordMap.save(out,1,QDomNode::EncodingFromTextStream); file.close(); } @@ -799,7 +804,11 @@ QList mainClass::createSQL(QSqlDatabase db, QVariantMap jsonData, QS mSelectValues.clear(); //Split the values into a stringlist +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) mSelectValues.append(insertObject.itemValue(pos).simplified().split(" ",QString::SkipEmptyParts)); +#else + mSelectValues.append(insertObject.itemValue(pos).simplified().split(" ",Qt::SkipEmptyParts)); +#endif //Process each value for (nvalue = 0; nvalue < mSelectValues.count();nvalue++) { @@ -1082,7 +1091,11 @@ void mainClass::processLoop(QJsonObject jsonData, QString loopTable, QString loo QString fieldValue; QString xmlKey = loopXMLRoot + "/" + loopItems[iItem] + "/" + fields[pos].xmlCode; fieldValue = jsonData.value(xmlKey).toString(""); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) QStringList parts = fieldValue.split(" ",QString::SkipEmptyParts); +#else + QStringList parts = fieldValue.split(" ",Qt::SkipEmptyParts); +#endif for (int ipart = 0; ipart < parts.count(); ipart++) { recordUUID=QUuid::createUuid(); @@ -1169,7 +1182,11 @@ int mainClass::procTable2(QSqlDatabase db,QJsonObject jsonData, QDomNode table, loopXMLRoot = table.toElement().attribute("xmlcode","none"); QString loopTable; loopTable = table.toElement().attribute("mysqlcode","none"); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) QStringList loopItems = table.toElement().attribute("loopitems","").split(QChar(743),QString::SkipEmptyParts); +#else + QStringList loopItems = table.toElement().attribute("loopitems","").split(QChar(743),Qt::SkipEmptyParts); +#endif if (loopItems.count() > 0) { QDomNode fieldNode = table.firstChild(); @@ -1180,7 +1197,7 @@ int mainClass::procTable2(QSqlDatabase db,QJsonObject jsonData, QDomNode table, { TfieldDef aLoopField; aLoopField.xmlCode = fieldNode.toElement().attribute("xmlcode",""); - aLoopField.decSize = fieldNode.toElement().attribute("decsize","0").toInt(); + aLoopField.decSize = QString::number(fieldNode.toElement().attribute("decsize","0").toInt()); aLoopField.key = false; if (fieldNode.toElement().attribute("isMultiSelect","false") == "true") { @@ -1191,7 +1208,7 @@ int mainClass::procTable2(QSqlDatabase db,QJsonObject jsonData, QDomNode table, aLoopField.multiSelect = false; aLoopField.name = fieldNode.toElement().attribute("mysqlcode",""); aLoopField.ODKType = fieldNode.toElement().attribute("odktype",""); - aLoopField.size = fieldNode.toElement().attribute("size",0).toInt(); + aLoopField.size = QString::number(fieldNode.toElement().attribute("size",0).toInt()); aLoopField.type = fieldNode.toElement().attribute("type","varchar"); loopFields.append(aLoopField); } @@ -1216,8 +1233,8 @@ int mainClass::procTable2(QSqlDatabase db,QJsonObject jsonData, QDomNode table, field.xmlCode = child.toElement().attribute("xmlcode"); field.type = child.toElement().attribute("type","varchar"); field.ODKType = child.toElement().attribute("odktype","text"); - field.size = child.toElement().attribute("size","0").toInt(); - field.decSize = child.toElement().attribute("decsize","0").toInt(); + field.size = QString::number(child.toElement().attribute("size","0").toInt()); + field.decSize = QString::number(child.toElement().attribute("decsize","0").toInt()); if (child.toElement().attribute("key").toStdString() == "true") field.key = true; @@ -1245,15 +1262,16 @@ int mainClass::procTable2(QSqlDatabase db,QJsonObject jsonData, QDomNode table, keys.append(createSQL(db,jsonData.toVariantMap(),tableCode,fields,keys,emptyMap,true)); //Change the variant map to an object later on! genSQL = false; } - if ((osm == false) && (loop == false)) + if ((osm == "false") && (loop == "false")) { - QJsonArray children = jsonData.value(child.toElement().attribute("xmlcode")).toArray(); - for (int chld = 0; chld < children.count(); chld++) - procTable2(db,children.at(chld).toObject(),child,keys); + procTable2(db,jsonData,child,keys); + } else { - procTable2(db,jsonData,child,keys); + QJsonArray children = jsonData.value(child.toElement().attribute("xmlcode")).toArray(); + for (int chld = 0; chld < children.count(); chld++) + procTable2(db,children.at(chld).toObject(),child,keys); } } else diff --git a/JXFormToMysql/jxformtomysql.pro b/JXFormToMysql/jxformtomysql.pro index b046a830..522537a9 100644 --- a/JXFormToMysql/jxformtomysql.pro +++ b/JXFormToMysql/jxformtomysql.pro @@ -15,6 +15,11 @@ CONFIG -= app_bundle TEMPLATE = app INCLUDEPATH += ../3rdparty -LIBS += -lquazip1-qt5 -lcsv +greaterThan(QT_MAJOR_VERSION, 5) { + LIBS += -lquazip1-qt6 -lcsv + QT += core5compat +} else { + LIBS += -lquazip1-qt5 -lcsv +} SOURCES += main.cpp diff --git a/JXFormToMysql/main.cpp b/JXFormToMysql/main.cpp index 014cf915..af490bd6 100644 --- a/JXFormToMysql/main.cpp +++ b/JXFormToMysql/main.cpp @@ -25,8 +25,14 @@ License along with JXFormToMySQL. If not, see #include #include -#include -#include +#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) +#include +#include +#include +#else +#include +#include +#endif #include #include #include @@ -40,6 +46,7 @@ License along with JXFormToMySQL. If not, see #include #include +#include //*******************************************Global variables*********************************************** bool debug; @@ -67,6 +74,7 @@ QStringList extraColumnsInSurvey; QStringList extraColumnsInOptions; QStringList ODKLanguages; bool hasSelects; +bool hasOnlyExternalSelects; QStringList extra_survey_columns; QStringList extra_choices_columns; QStringList extra_invalid_columns; @@ -231,6 +239,7 @@ struct tableDef bool isOSM; bool isGroup; bool hasOther = false; + QString lookupCSV; }; typedef tableDef TtableDef; @@ -242,8 +251,15 @@ struct duplicatedSelectValue QString variableName; QString selectValue; }; +struct invalidSelectValue +{ + QString variableName; + QString selectValue; +}; typedef duplicatedSelectValue TduplicatedSelectValue; +typedef invalidSelectValue TinvalidSelectValue; QList duplicatedSelectValues; +QList invalidSelectValues; struct duplicatedField { @@ -1229,12 +1245,12 @@ void report_file_error(QString file_name) log(XMLResult.toString()); } -void cb1(void *s, size_t, void *) -{ - char* charData; - charData = (char*)s; - CSVvalues.append(QString::fromUtf8(charData)); -} +// void cb1(void *s, size_t, void *) +// { +// char* charData; +// charData = (char*)s; +// CSVvalues.append(QString::fromUtf8(charData)); +// } QString fixColumnName(QString column) { @@ -1259,90 +1275,189 @@ bool isColumnValid(QString column) } } -void cb2(int , void *) -{ - QString sql; - if (CSVRowNumber == 1) +// void cb2(int , void *) +// { +// QString sql; +// if (CSVRowNumber == 1) +// { +// sql = "CREATE TABLE data ("; +// numColumns = 0; +// for (int pos = 0; pos <= CSVvalues.count()-1;pos++) +// { +// numColumns++; +// QString columnName; +// columnName = fixColumnName(CSVvalues[pos]); +// if (isColumnValid(columnName) == false) +// CSVColumError = true; +// sql = sql + columnName + " TEXT,"; +// } +// sql = sql.left(sql.length()-1) + ");"; +// CSVSQLs.append(sql); +// } +// else +// { +// sql = "INSERT INTO data VALUES ("; +// numColumnsInData = 0; +// //Using numColumns so avoids more columns than the heading +// for (int pos = 0; pos <= numColumns-1;pos++) +// { +// numColumnsInData++; +// sql = sql + "\"" + CSVvalues[pos].replace("\"","") + "\","; +// } +// //This will fix if a row has less columns than the heading +// for (int pos =1; pos <= numColumns-numColumnsInData;pos++) +// sql = sql + "\"\""; +// sql = sql.left(sql.length()-1) + ");"; +// CSVSQLs.append(sql); +// } +// CSVvalues.clear(); +// CSVRowNumber++; +// } + +int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase database) +{ + QFileInfo fi(fileName); + + QProcess csvcutProcess; + QProcess awkProcess; + QProcess pasteProcess; + + csvcutProcess.setProgram("csvcut"); + csvcutProcess.setArguments({"-n", fileName}); + + awkProcess.setProgram("awk"); + awkProcess.setArguments({"-F: ", "NR >= 1 {print $2}"}); + + pasteProcess.setProgram("paste"); + pasteProcess.setArguments({"-sd", "|"}); + + csvcutProcess.setStandardOutputProcess(&awkProcess); + awkProcess.setStandardOutputProcess(&pasteProcess); + + csvcutProcess.start(); + awkProcess.start(); + pasteProcess.start(); + CSVColumError = false; + pasteProcess.waitForFinished(); + CSVSQLs.clear(); + if (pasteProcess.exitCode() == 0) { - sql = "CREATE TABLE data ("; - numColumns = 0; - for (int pos = 0; pos <= CSVvalues.count()-1;pos++) + QString columns_str = pasteProcess.readAllStandardOutput(); + columns_str = columns_str.replace("\n",""); + + QStringList columns = columns_str.split("|"); + QString sql = "CREATE TABLE data ("; + for (int pos = 0; pos < columns.count();pos++) { - numColumns++; QString columnName; - columnName = fixColumnName(CSVvalues[pos]); + columnName = fixColumnName(columns[pos]); if (isColumnValid(columnName) == false) CSVColumError = true; sql = sql + columnName + " TEXT,"; } sql = sql.left(sql.length()-1) + ");"; CSVSQLs.append(sql); - } - else - { - sql = "INSERT INTO data VALUES ("; - numColumnsInData = 0; - //Using numColumns so avoids more columns than the heading - for (int pos = 0; pos <= numColumns-1;pos++) - { - numColumnsInData++; - sql = sql + "\"" + CSVvalues[pos].replace("\"","") + "\","; - } - //This will fix if a row has less columns than the heading - for (int pos =1; pos <= numColumns-numColumnsInData;pos++) - sql = sql + "\"\""; - sql = sql.left(sql.length()-1) + ");"; - CSVSQLs.append(sql); - } - CSVvalues.clear(); - CSVRowNumber++; -} -int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase database) -{ - FILE *fp; - struct csv_parser p; - char buf[4096]; - size_t bytes_read; - size_t retval; - unsigned char options = 0; + QString jsonFile = fi.baseName(); + jsonFile = tempDirectory.absolutePath() + tempDirectory.separator() + jsonFile + ".json"; - if (csv_init(&p, CSV_STRICT) != 0) - { - log("Failed to initialize csv parser"); - return 1; - } - fp = fopen(fileName.toUtf8().constData(), "rb"); - if (!fp) - { - log("Failed to open CSV file " + fileName); - return 0; - } - options = CSV_APPEND_NULL; - csv_set_opts(&p, options); - CSVColumError = false; - CSVRowNumber = 1; - CSVvalues.clear(); - CSVSQLs.clear(); - while ((bytes_read=fread(buf, 1, 4096, fp)) > 0) - { - if ((retval = csv_parse(&p, buf, bytes_read, cb1, cb2, NULL)) != bytes_read) + QProcess CSVToJSON; + QStringList args; + args.append("--no-inference"); + args.append(fileName); + CSVToJSON.setProgram("csvjson"); + CSVToJSON.setArguments(args); + CSVToJSON.setStandardOutputFile(jsonFile); + CSVToJSON.start(); + CSVToJSON.waitForFinished(); + if (CSVToJSON.exitCode() == 0) { - if (csv_error(&p) == CSV_EPARSE) + QFile JSONFile(jsonFile); + if (!JSONFile.open(QIODevice::ReadOnly)) { - log("Malformed data at byte " + QString::number((unsigned long)retval + 1) + " in file " + fileName); + log("Cannot open" + jsonFile); return 1; } - else + QByteArray JSONData = JSONFile.readAll(); + QJsonDocument JSONDocument; + JSONDocument = QJsonDocument::fromJson(JSONData); + QJsonArray firstObject = JSONDocument.array(); + if (!firstObject.isEmpty()) { - log("Error \"" + QString::fromUtf8(csv_strerror(csv_error(&p))) + "\" in file " + fileName); - return 1; + for(int tmp=0; tmp < firstObject.count(); tmp++) + { + sql = "INSERT INTO data VALUES ("; + QJsonObject obj = firstObject[tmp].toObject(); + for (int pos = 0; pos < columns.count();pos++) + { + QString columnValue; + columnValue = obj.value(columns[pos]).toString(); + sql = sql + "\"" + columnValue.replace("\"","") + "\","; + } + sql = sql.left(sql.length()-1) + ");"; + CSVSQLs.append(sql); + } } } + else + { + QString StandardError(CSVToJSON.readAllStandardError()); + QString StandardOutput(CSVToJSON.readAllStandardOutput()); + log("Failed to convert CSV to JSON for file " + fileName + "-" + StandardError + "-" + StandardOutput + QString::number(CSVToJSON.exitCode())); + return 1; + } + } + else + { + log("Failed to read csv columnns in file " + fileName); + return 1; } - fclose(fp); - csv_fini(&p, cb1, cb2, NULL); - csv_free(&p); + + // Start of replacement + + // FILE *fp; + // struct csv_parser p; + // char buf[4096]; + // size_t bytes_read; + // size_t retval; + // unsigned char options = 0; + + // if (csv_init(&p, CSV_STRICT) != 0) + // { + // log("Failed to initialize csv parser"); + // return 1; + // } + // fp = fopen(fileName.toUtf8().constData(), "rb"); + // if (!fp) + // { + // log("Failed to open CSV file " + fileName); + // return 0; + // } + // options = CSV_APPEND_NULL; + // csv_set_opts(&p, options); + // CSVColumError = false; + // CSVRowNumber = 1; + // CSVvalues.clear(); + // CSVSQLs.clear(); + // while ((bytes_read=fread(buf, 1, 4096, fp)) > 0) + // { + // if ((retval = csv_parse(&p, buf, bytes_read, cb1, cb2, NULL)) != bytes_read) + // { + // if (csv_error(&p) == CSV_EPARSE) + // { + // log("Malformed data at byte " + QString::number((unsigned long)retval + 1) + " in file " + fileName); + // return 1; + // } + // else + // { + // log("Error \"" + QString::fromUtf8(csv_strerror(csv_error(&p))) + "\" in file " + fileName); + // return 1; + // } + // } + // } + // fclose(fp); + // csv_fini(&p, cb1, cb2, NULL); + // csv_free(&p); if (CSVColumError) { @@ -1355,7 +1470,10 @@ int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase databa exit(14); } - QFileInfo fi(fileName); + + // End of replacement + + QString sqlLiteFile; sqlLiteFile = fi.baseName(); sqlLiteFile = tempDirectory.absolutePath() + tempDirectory.separator() + sqlLiteFile + ".sqlite"; @@ -1377,6 +1495,7 @@ int convertCSVToSQLite(QString fileName, QDir tempDirectory, QSqlDatabase databa query.exec("BEGIN TRANSACTION"); for (int pos = 0; pos <= CSVSQLs.count()-1;pos++) { + //qDebug() << CSVSQLs[pos]; if (!query.exec(CSVSQLs[pos])) { log("Cannot insert data for row: " + QString::number(pos+2) + " in file: " + sqlLiteFile + " reason: " + query.lastError().databaseText()); @@ -1552,7 +1671,7 @@ QString get_related_usage(QString table) //This fuction checkd wheter a lookup table is duplicated. //If there is a match then returns such table -TtableDef checkDuplicatedLkpTable(QString table, QList thisValues) +TtableDef checkDuplicatedLkpTable(QString table, QList thisValues, bool select_from_file, QString lookupCSV) { TtableDef empty; empty.name = "EMPTY"; @@ -1565,7 +1684,7 @@ TtableDef checkDuplicatedLkpTable(QString table, QList thisValues) QString thisDesc; QString currenDesc; //Move the new list of values to a new list and sort it by code - qSort(thisValues.begin(),thisValues.end(),lkpComp); + std::sort(thisValues.begin(),thisValues.end(),lkpComp); QString defLangCode; defLangCode = getLanguageCode(getDefLanguage()); @@ -1579,7 +1698,7 @@ TtableDef checkDuplicatedLkpTable(QString table, QList thisValues) //Move the current list of values to a new list and sort it by code currentValues.clear(); currentValues.append(tables[pos].lkpValues); - qSort(currentValues.begin(),currentValues.end(),lkpComp); + std::sort(currentValues.begin(),currentValues.end(),lkpComp); if (currentValues.count() == thisValues.count()) //Same number of values { @@ -1599,6 +1718,9 @@ TtableDef checkDuplicatedLkpTable(QString table, QList thisValues) } if (found) { + if (select_from_file) + if (tables[pos].lookupCSV != lookupCSV) + return empty; int idx; idx = -1; for (int pos2=0; pos2 < duplicated_lookups.count(); pos2++) @@ -1851,28 +1973,33 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (!sqlInsertFile.open(QIODevice::WriteOnly | QIODevice::Text)) return; QTextStream sqlInsertStrm(&sqlInsertFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) sqlInsertStrm.setCodec("UTF-8"); - +#endif QFile iso639File(transFile); if (!iso639File.open(QIODevice::WriteOnly | QIODevice::Text)) return; QTextStream iso639Strm(&iso639File); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) iso639Strm.setCodec("UTF-8"); - +#endif QFile sqlUpdateFile(metaFile); if (!sqlUpdateFile.open(QIODevice::WriteOnly | QIODevice::Text)) return; QTextStream sqlUpdateStrm(&sqlUpdateFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) sqlUpdateStrm.setCodec("UTF-8"); - +#endif QFile sqlDropFile(dropSQL); if (!sqlDropFile.open(QIODevice::WriteOnly | QIODevice::Text)) return; QTextStream sqlDropStrm(&sqlDropFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) sqlDropStrm.setCodec("UTF-8"); +#endif //Start creating the header or each file. QDateTime date; @@ -1924,7 +2051,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr } iso639Strm << "\n"; - qSort(tables.begin(),tables.end(),tblComp); + std::sort(tables.begin(),tables.end(),tblComp); for (int pos = tables.count()-1; pos >=0;pos--) { @@ -1994,7 +2121,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr for (int nlkp = 0; nlkp < tables[pos].lkpValues.count();nlkp++) { QDomElement aLKPValue = insertValuesXML.createElement("value"); - aLKPValue.setAttribute("code",tables[pos].lkpValues[nlkp].code); + aLKPValue.setAttribute("code",tables[pos].lkpValues[nlkp].code.replace("'","`").simplified()); aLKPValue.setAttribute("description",fixString(getDescForLanguage(tables[pos].lkpValues[nlkp].desc,defLangCode))); // Add other values for (int oth = 0; oth < tables[pos].lkpValues[nlkp].other_values.count(); oth++) @@ -2437,7 +2564,7 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr } insertSQL = insertSQL.left(insertSQL.length()-1) + ") VALUES ('"; - insertSQL = insertSQL + tables[pos].lkpValues[clm].code.replace("'","`") + "',\""; + insertSQL = insertSQL + tables[pos].lkpValues[clm].code.replace("'","`").simplified() + "',\""; insertSQL = insertSQL + fixString(getDescForLanguage(tables[pos].lkpValues[clm].desc,defLangCode)) + "\","; for (int p = 0; p < tables[pos].propertyList.count(); p++) { @@ -2472,7 +2599,9 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif outputdoc.save(out,1,QDomNode::EncodingFromTextStream); file.close(); } @@ -2486,7 +2615,9 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (XMLCreateFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream outXMLCreate(&XMLCreateFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) outXMLCreate.setCodec("UTF-8"); +#endif XMLSchemaStructure.save(outXMLCreate,1,QDomNode::EncodingFromTextStream); XMLCreateFile.close(); } @@ -2500,7 +2631,9 @@ void generateOutputFiles(QString ddlFile,QString insFile, QString metaFile, QStr if (XMLInsertFile.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream outXMLInsert(&XMLInsertFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) outXMLInsert.setCodec("UTF-8"); +#endif insertValuesXML.save(outXMLInsert,1,QDomNode::EncodingFromTextStream); XMLInsertFile.close(); } @@ -3147,8 +3280,21 @@ QList getLabels(QJsonValue labelValue) return labels; } -bool checkSelectValue(QString variableName, QList values, QString value, bool report=true) +bool checkSelectValue(QString variableName, QList values, QString value, bool report=true, bool multiSelect = false) { + if (multiSelect) + { + if (value.indexOf(" ") >= 0) + { + if (report) + { + TinvalidSelectValue invalid; + invalid.variableName = variableName; + invalid.selectValue = value.toLower().trimmed(); + invalidSelectValues.append(invalid); + } + } + } for (int idx = 0; idx < values.count(); idx++) { if (values[idx].code.toLower().trimmed() == value.toLower().trimmed()) @@ -3169,8 +3315,11 @@ bool checkSelectValue(QString variableName, QList values, QString val // This return the values of a select that uses an external xml file. // e.g., "select one from file a_file.xml" and "select multiple from file a_file.xml" -QList getSelectValuesFromGeoJSON(QString variableName, QString fileName, int &result, QDir dir, QString codeColumn, QString descColumn, QStringList &propertyList, QStringList &propertyTypes) +QList getSelectValuesFromGeoJSON(QString variableName, QString variableType, QString fileName, int &result, QDir dir, QString codeColumn, QString descColumn, QStringList &propertyList, QStringList &propertyTypes) { + bool multiselect = false; + if (isSelect(variableType) == 3) + multiselect = true; QList res; result = 0; QString jsonFile; @@ -3275,7 +3424,7 @@ QList getSelectValuesFromGeoJSON(QString variableName, QString fileNa coor_column.column_name = "coordinates"; coor_column.column_value = coordinates_string; value.other_values.append(coor_column); - checkSelectValue(variableName,res,value.code); + checkSelectValue(variableName,res,value.code,true,multiselect); res.append(value); } else @@ -3379,9 +3528,13 @@ QList getSelectValuesFromGeoJSON(QString variableName, QString fileNa // This return the values of a select that uses an external xml file. // e.g., "select one from file a_file.xml" and "select multiple from file a_file.xml" -QList getSelectValuesFromXML(QString variableName, QString fileName, bool hasOrOther, int &result, QDir dir, QString codeColumn="name", QString descColumn="label") +QList getSelectValuesFromXML(QString variableName, QString variableType, QString fileName, bool hasOrOther, int &result, QDir dir, QString codeColumn="name", QString descColumn="label") { - QList res; + bool multiSelect = false; + if (isSelect(variableType) == 3) + multiSelect = true; + + QList res; QStringList descColumns; result = 0; descColumns << descColumn; @@ -3472,7 +3625,7 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, } } } - checkSelectValue(variableName,res,value.code); + checkSelectValue(variableName,res,value.code,true,multiSelect); res.append(value); } else @@ -3488,7 +3641,7 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, } if (hasOrOther) { - bool duplicated = checkSelectValue(variableName,res,"other",false); + bool duplicated = checkSelectValue(variableName,res,"other",false,multiSelect); if (!duplicated) { TlkpValue value; @@ -3533,8 +3686,13 @@ QList getSelectValuesFromXML(QString variableName, QString fileName, // This return the values of a select that uses an external CSV file. // e.g., "select one from file a_file.csv","select multiple from file a_file.csv","select one external" -QList getSelectValuesFromCSV2(QString variableName, QString fileName, bool hasOrOther, int &result, QDir dir, QSqlDatabase database, QString queryValue, QString codeColumn="name", QString descColumn="label") +QList getSelectValuesFromCSV2(QString variableName, QString variableType, QString fileName, bool hasOrOther, int &result, QDir dir, QSqlDatabase database, QString queryValue, QString codeColumn="name", QString descColumn="label") { + bool multiSelect = false; + if (isSelect(variableType) == 3) + { + multiSelect = true; + } QList res; QStringList descColumns; result = 0; @@ -3625,7 +3783,7 @@ QList getSelectValuesFromCSV2(QString variableName, QString fileName, } } } - checkSelectValue(variableName,res,value.code); + checkSelectValue(variableName,res,value.code,true,multiSelect); res.append(value); } else @@ -3641,7 +3799,7 @@ QList getSelectValuesFromCSV2(QString variableName, QString fileName, } if (hasOrOther) { - bool duplicated = checkSelectValue(variableName,res,"other",false); + bool duplicated = checkSelectValue(variableName,res,"other",false,multiSelect); if (!duplicated) { TlkpValue value; @@ -3696,8 +3854,11 @@ QList getSelectValuesFromCSV2(QString variableName, QString fileName, // e.g.: // type: select one canton // appearance: search('cantones', 'matches', 'a_column', ${a_variable}) -QList getSelectValuesFromCSV(QString searchExpresion, QJsonArray choices,QString variableName, bool hasOrOther, int &result, QDir dir, QSqlDatabase database, QString &file, QString &codeColumn, QString &descColumn) +QList getSelectValuesFromCSV(QString searchExpresion, QJsonArray choices,QString variableName, QString variableType, bool hasOrOther, int &result, QDir dir, QSqlDatabase database, QString &file, QString &codeColumn, QString &descColumn) { + bool multiSelect = false; + if (isSelect(variableType) == 3) + multiSelect = true; QList res; codeColumn = ""; result = 0; @@ -3773,12 +3934,12 @@ QList getSelectValuesFromCSV(QString searchExpresion, QJsonArray choi value.desc.append(desc); } } - checkSelectValue(variableName,res,value.code); + checkSelectValue(variableName,res,value.code,true,multiSelect); res.append(value); } if (hasOrOther) { - bool duplicated = checkSelectValue(variableName,res,"other",false); + bool duplicated = checkSelectValue(variableName,res,"other",false,multiSelect); if (!duplicated) { TlkpValue value; @@ -3961,8 +4122,11 @@ bool checkColumnName(QString name) //This return the values of a simple select or select multiple -QList getSelectValues(QString variableName, QJsonArray choices, bool hasOther, QStringList extraColumns) +QList getSelectValues(QString variableName,QString variableType,QJsonArray choices, bool hasOther, QStringList extraColumns) { + bool multiSelect = false; + if (isSelect(variableType) == 3) + multiSelect = true; QList res; for (int nrow = 0; nrow < choices.count(); nrow++) { @@ -3999,7 +4163,7 @@ QList getSelectValues(QString variableName, QJsonArray choices, bool value.code = JSONValue.toObject().value("name").toString(); QJsonValue JSONlabel = JSONValue.toObject().value("label"); value.desc = getLabels(JSONlabel); - checkSelectValue(variableName,res,value.code); + checkSelectValue(variableName,res,value.code,true,multiSelect); for (int ex=0; ex < extraColumns.count(); ex++) { TotherLkpValue other_value; @@ -4014,7 +4178,7 @@ QList getSelectValues(QString variableName, QJsonArray choices, bool } if (hasOther) { - bool duplicated = checkSelectValue(variableName,res,"other",false); + bool duplicated = checkSelectValue(variableName,res,"other",false,multiSelect); if (!duplicated) { TlkpValue value; @@ -4110,7 +4274,7 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) QList values; QStringList extra_columns; extra_columns = getExtraColumns(fieldObject.value("choices").toArray()); - values.append(getSelectValues(variableName,fieldObject.value("choices").toArray(),false,extra_columns)); + values.append(getSelectValues(variableName,"NONE",fieldObject.value("choices").toArray(),false,extra_columns)); TfieldDef aField; aField.selectSource = "NONE"; aField.name = fixField(variableName.toLower()); @@ -4136,7 +4300,7 @@ void parseOSMField(TtableDef &OSMTable, QJsonObject fieldObject) { //Creating the lookp table if its neccesary QString table_name = "lkp" + fixField(variableName.toLower(),true); - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values,false,""); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -4434,7 +4598,9 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q QString codeColumn="name"; QString descColumn="label"; QStringList propertyList; - QStringList propertyTypes; + QStringList propertyTypes; + bool select_from_file = false; + QString fileName; if ((isSelect(variableType) == 1) || (isSelect(variableType) == 3) || (isSelect(variableType) == 4)) { bool fromSearchCSV; @@ -4453,7 +4619,6 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q { if (fieldObject.value("itemset").toString("").toLower().trimmed().indexOf(".csv") > 0) { - QString fileName; fileName = fieldObject.value("itemset").toString("").toLower().trimmed(); if (!fieldObject.value("parameters").isUndefined()) { @@ -4464,7 +4629,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q int result; select_type = 3; external_file = fileName; - values.append(getSelectValuesFromCSV2(fixField(variableName, true),fileName,selectHasOrOther(variableType),result,dir,database,"",codeColumn,descColumn)); + select_from_file = true; + values.append(getSelectValuesFromCSV2(fixField(variableName, true),variableType,fileName,selectHasOrOther(variableType),result,dir,database,"",codeColumn,descColumn)); if (result != 0) { if (!justCheck) @@ -4473,7 +4639,6 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q } else { - QString fileName; fileName = fieldObject.value("itemset").toString("").toLower().trimmed(); if (fileName.toLower().trimmed().indexOf(".xml") >= 0) { @@ -4486,7 +4651,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q int result; select_type = 4; external_file = fileName; - values.append(getSelectValuesFromXML(fixField(variableName, true),fileName,selectHasOrOther(variableType),result,dir,codeColumn,descColumn)); + select_from_file = true; + values.append(getSelectValuesFromXML(fixField(variableName, true),variableType,fileName,selectHasOrOther(variableType),result,dir,codeColumn,descColumn)); if (result != 0) { if (!justCheck) @@ -4509,7 +4675,8 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q select_type = 6; external_file = fileName; //qDebug() << fileName; - values.append(getSelectValuesFromGeoJSON(fixField(variableName, true),fileName,result,dir,codeColumn,descColumn,propertyList,propertyTypes)); + select_from_file = true; + values.append(getSelectValuesFromGeoJSON(fixField(variableName, true),variableType,fileName,result,dir,codeColumn,descColumn,propertyList,propertyTypes)); //qDebug() << values.count(); if (result != 0) { @@ -4526,14 +4693,15 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q { propertyList = getExtraColumns(fieldObject.value("choices").toArray()); propertyTypes = getExtraColumnsTypes(fieldObject.value("choices").toArray(), propertyList); - values.append(getSelectValues(fixField(variableName, true),fieldObject.value("choices").toArray(),selectHasOrOther(variableType),propertyList)); + values.append(getSelectValues(fixField(variableName, true),variableType,fieldObject.value("choices").toArray(),selectHasOrOther(variableType),propertyList)); select_type = 1; + hasOnlyExternalSelects = false; } else { int result; - QString fileName; - values.append(getSelectValuesFromCSV(variableApperance,fieldObject.value("choices").toArray(),fixField(variableName, true),selectHasOrOther(variableType),result,dir,database,fileName,codeColumn,descColumn)); + select_from_file = true; + values.append(getSelectValuesFromCSV(variableApperance,fieldObject.value("choices").toArray(),fixField(variableName, true),variableType,selectHasOrOther(variableType),result,dir,database,fileName,codeColumn,descColumn)); select_type = 2; external_file = fileName; if (result != 0) @@ -4553,7 +4721,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q if (queryField != "") { int result; - values.append(getSelectValuesFromCSV2(fixField(variableName, true),"itemsets.csv",selectHasOrOther(variableType),result,dir,database,queryField)); + values.append(getSelectValuesFromCSV2(fixField(variableName, true),variableType,"itemsets.csv",selectHasOrOther(variableType),result,dir,database,queryField)); select_type = 5; external_file = "itemsets.csv"; if (result != 0) @@ -4663,8 +4831,9 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q listName = fixField(variableName.toLower(), true); } QString table_name = "lkp" + listName; - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values,select_from_file,fileName); lkpTable.isLoop = false; + lkpTable.lookupCSV = fileName; lkpTable.isOSM = false; lkpTable.isGroup = false; if (lkpTable.name == "EMPTY") @@ -4942,7 +5111,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q } QString table_name = "lkp" + listName; - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values,select_from_file,fileName); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -4958,6 +5127,7 @@ void parseField(QJsonObject fieldObject, QString mainTable, QString mainField, Q } lkpTable.pos = -1; lkpTable.islookup = true; + lkpTable.lookupCSV = fileName; lkpTable.isOneToOne = false; lkpTable.hasOther = selectHasOrOther(variableType); //Creates the field for code in the lookup @@ -5251,7 +5421,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f QList values; QStringList extra_columns; extra_columns = getExtraColumns(tableObject.value("columns").toArray()); - values.append(getSelectValues(aTable.name,tableObject.value("columns").toArray(),false,extra_columns)); + values.append(getSelectValues(aTable.name,"select one",tableObject.value("columns").toArray(),false,extra_columns)); for (int litem = 0; litem < values.count(); litem++) { aTable.loopItems.append(values[litem].code); @@ -5303,7 +5473,7 @@ void parseTable(QJsonObject tableObject, QString tableType, bool repeatOfOne = f listName = fixField(variableName.toLower(), true); } QString table_name = "lkp" + listName; - TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values); + TtableDef lkpTable = checkDuplicatedLkpTable(table_name,values,false,""); lkpTable.isLoop = false; lkpTable.isOSM = false; lkpTable.isGroup = false; @@ -5747,6 +5917,31 @@ void reportSelectDuplicates() log(XMLResult.toString()); } +void reportSelectInvalid() +{ + QDomElement XMLRoot; + XMLRoot = XMLResult.createElement("XMLInvalidSelects"); + XMLDocRoot.appendChild(XMLRoot); + if (outputType != "m") + { + log("The following variables are multi-select with options that have spaces in the column \"name\" "); + } + for (int pos = 0; pos <= invalidSelectValues.count()-1; pos++) + { + if (outputType != "m") + { + log("\tVariable: " + invalidSelectValues[pos].variableName + " - Option: " + invalidSelectValues[pos].selectValue); + } + QDomElement eDuplicatedItem; + eDuplicatedItem = XMLResult.createElement("invalidItem"); + eDuplicatedItem.setAttribute("variableName",invalidSelectValues[pos].variableName); + eDuplicatedItem.setAttribute("invalidValue",invalidSelectValues[pos].selectValue); + XMLRoot.appendChild(eDuplicatedItem); + } + if (outputType == "m") + log(XMLResult.toString()); +} + QList getValuesFromInsertFile(QString tableName, QDomNode startNode, QString &clmCode, QString &cmlDesc) { QDomNode lkptable = startNode; @@ -6459,7 +6654,7 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di parseJSONObject(firstObject, mainTable, mainField, dir, database); getLanguages(firstObject, ODKLanguages, num_labels); - if (num_labels == 0 && hasSelects) + if (num_labels == 0 && hasSelects && !hasOnlyExternalSelects) exit(8); if (duplicatedTables.count() > 0) @@ -6477,6 +6672,11 @@ int processJSON(QString inputFile, QString mainTable, QString mainField, QDir di reportSelectDuplicates(); exit(9); } + if (invalidSelectValues.count() > 0) + { + reportSelectInvalid(); + exit(36); + } for (int itable = 0; itable < tables.count(); itable++) { @@ -6738,6 +6938,7 @@ void protect_sensitive() int main(int argc, char *argv[]) { + QCoreApplication app(argc, argv); QString title; title = title + "********************************************************************* \n"; title = title + " * JSON XForm To MySQL * \n"; @@ -6778,8 +6979,10 @@ int main(int argc, char *argv[]) title = title + " * 33: Extra survey or choice columns cannot have spaces. * \n"; title = title + " * 34: One or more tables have too many columns (>65000 bytes). * \n"; title = title + " * 35: Select or multi-select uses \"or other\". * \n"; + title = title + " * 36: Multi-selects have spaces in column name. * \n"; title = title + " * * \n"; title = title + " * XML = XML oputput is available. * \n"; + title = title + " * Note: This tool requires CSVKit (sudo apt-get install csvkit) * \n"; title = title + " ********************************************************************* \n"; TCLAP::CmdLine cmd(title.toUtf8().constData(), ' ', "2.0"); @@ -6842,6 +7045,7 @@ int main(int argc, char *argv[]) //Parsing the command lines cmd.parse( argc, argv ); hasSelects = false; + hasOnlyExternalSelects = true; //Get the support files std::vector v = suppFiles.getValue(); for (int i = 0; static_cast(i) < v.size(); i++) diff --git a/LICENSE_AI b/LICENSE_AI new file mode 100644 index 00000000..f8d831f7 --- /dev/null +++ b/LICENSE_AI @@ -0,0 +1,75 @@ +**Note on AI-Assisted Refactoring and LGPL/GPL Derivative Works** + +This note clarifies how software licensed under the GNU Lesser General Public License (LGPL) and GNU General Public License (GPL) applies to code generated through AI-assisted refactoring, translation, or reimplementation. + +**1. Derivative Works and Non-Literal Transformation** + +Both LGPL and GPL are based on copyright law, which protects not only literal code copying but also the **structure, sequence, and organization** of a program. + +A work may be considered **derivative** even if: + +* No original lines of code are copied +* The code is rewritten in a different programming language (e.g., C++ to Python) +* The implementation is newly generated (including by AI) + +If the resulting code reproduces the original program’s **architecture, logic, workflows, or design**, it still qualify as a derivative work. + +--- + +**2. AI-Assisted Refactoring** + +When an AI system is used to: + +* Analyze LGPL/GPL-licensed source code +* Understand its functionality and internal design +* Generate a new implementation based on that understanding + +the resulting code is considered a **translation or transformation** of the original work. + +Under prevailing interpretations of LGPL/GPL: + +* AI-assisted rewrites are treated similarly to **human rewrites** +* The method of creation (AI vs human) does **not change** the derivative work analysis + +--- + +**3. Licensing Implications** + +If the AI-generated code is determined to be a derivative work: + +* Under the GPL: + The resulting code must be licensed under GPL if distributed. + +* Under the LGPL: + The resulting code may need to comply with LGPL requirements, particularly if it incorporates or replicates protected elements of the original library implementation. + +--- + +**4. Independent Implementations** + +AI-generated code may **not** be considered derivative if it is created from: + +* High-level specifications +* Public documentation +* Functional descriptions +* Observed behavior (without access to source code) + +and does **not replicate the protected structure or implementation details** of the original program. + +This is analogous to a **clean-room implementation**, where: + +* One process defines the specification +* Another independently implements it without exposure to the original code + +In such cases, LGPL/GPL obligations typically do **not** apply. + +--- + +**5. Key Determining Factor** + +The central question is: + +> Was the new implementation derived from the original code’s expression, or independently created from its underlying ideas? + +If the AI system had access to and relied on the original LGPL/GPL code to generate the new implementation, it is considered **derivative**. + diff --git a/XMLtoJSON/main.cpp b/XMLtoJSON/main.cpp index 4f70cb70..278c386f 100644 --- a/XMLtoJSON/main.cpp +++ b/XMLtoJSON/main.cpp @@ -136,11 +136,11 @@ pt::ptree processNodeBoost(QStringList repeatArray,bool group,QDomNode node,pt:: //If its a group then we need to extract the elements recursively because they don't generate an array pt::ptree elems;//QJsonArray elems; elems = processNodeBoost(repeatArray,true,start,json); - int count; - count = 0; + //int count; + //count = 0; BOOST_FOREACH(boost::property_tree::ptree::value_type const&v, elems.get_child("")) { - count++; + //count++; const boost::property_tree::ptree &subtree = v.second; BOOST_FOREACH( boost::property_tree::ptree::value_type const&v2, subtree ) { @@ -331,7 +331,11 @@ int main(int argc, char *argv[]) QString nodeset; QStringList nodeArray; nodeset = repeats.item(pos).toElement().attribute("nodeset"); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) nodeArray = nodeset.split("/",QString::SkipEmptyParts); +#else + nodeArray = nodeset.split("/",Qt::SkipEmptyParts); +#endif if (nodeArray.length() > 0) repeatArray.append(nodeArray[nodeArray.length()-1]); } diff --git a/utilities/DCFToODK/DCFToODK.pro b/utilities/DCFToODK/DCFToODK.pro index 740f62af..58fde38e 100644 --- a/utilities/DCFToODK/DCFToODK.pro +++ b/utilities/DCFToODK/DCFToODK.pro @@ -17,3 +17,7 @@ SOURCES += main.cpp \ HEADERS += \ dcftoxml.h \ xmltoyml.h + +greaterThan(QT_MAJOR_VERSION, 5) { +QT += core5compat +} diff --git a/utilities/DCFToODK/dcftoxml.cpp b/utilities/DCFToODK/dcftoxml.cpp index 5291f149..66475a5a 100644 --- a/utilities/DCFToODK/dcftoxml.cpp +++ b/utilities/DCFToODK/dcftoxml.cpp @@ -244,13 +244,13 @@ void DCFToXML::convertToXML(QString dcfFile) exit(1); } QTextStream in(&file); - int pos; - pos = 1; + //int pos; + //pos = 1; while (!in.atEnd()) { QString line = in.readLine(); if (!line.isEmpty()) addToXML(line); //Parse the line - pos++; + //pos++; } } diff --git a/utilities/DCFToODK/xmltoyml.cpp b/utilities/DCFToODK/xmltoyml.cpp index 3efd0b13..17d08326 100644 --- a/utilities/DCFToODK/xmltoyml.cpp +++ b/utilities/DCFToODK/xmltoyml.cpp @@ -5,11 +5,15 @@ #include #include #include -#include +//#include #include #include #include #include +#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) +#include +#include +#endif namespace pt = boost::property_tree; @@ -288,7 +292,12 @@ void XMLToYML::generateYML(QString file, QString mainRecord, QString tempDir) pt::ptree settings_sheet; pt::ptree settings_object; settings_object.put("form_title",fileName.toStdString()); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) settings_object.put("form_id",fileName.remove(QRegExp("[^a-zA-Z\\d\\s]")).toStdString()); +#else + static QRegularExpression re("[^a-zA-Z\\d\\s]"); + settings_object.put("form_id",fileName.remove(re).toStdString()); +#endif settings_sheet.push_back(std::make_pair("", settings_object)); JSONRoot.add_child("settings",settings_sheet); diff --git a/utilities/MySQLDenormalize/MySQLDenormalize.pro b/utilities/MySQLDenormalize/MySQLDenormalize.pro index 5ffba1d8..b5a6ccaf 100644 --- a/utilities/MySQLDenormalize/MySQLDenormalize.pro +++ b/utilities/MySQLDenormalize/MySQLDenormalize.pro @@ -16,10 +16,7 @@ DEFINES += QT_DEPRECATED_WARNINGS # You can also select to disable deprecated APIs only up to a certain version of Qt. #DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0 -unix:INCLUDEPATH += ../../3rdparty /usr/include/mongocxx/v_noabi /usr/include/bsoncxx/v_noabi \ -/usr/local/include/mongocxx/v_noabi /usr/local/include/bsoncxx/v_noabi - -unix:LIBS += -L/lib64 -L/usr/lib64 -L/usr/local/lib -lmongocxx -lbsoncxx +unix:INCLUDEPATH += ../../3rdparty SOURCES += main.cpp \ mainclass.cpp diff --git a/utilities/MySQLDenormalize/main.cpp b/utilities/MySQLDenormalize/main.cpp index f4ffc9e0..71caac30 100644 --- a/utilities/MySQLDenormalize/main.cpp +++ b/utilities/MySQLDenormalize/main.cpp @@ -52,10 +52,10 @@ int main(int argc, char *argv[]) QString title; title = title + " *********************************************************************** \n"; title = title + " * MySQLDenormalize * \n"; - title = title + " * This tool denormalize data from a MySQL Database into JSON * \n"; - title = title + " * files starting from the main table. It relies on the Map XML files * \n"; + title = title + " * This tool denormalize a submission from a MySQL Database into JSON * \n"; + title = title + " * starting from the main table. It relies on the Map XML file * \n"; title = title + " * created by JSONToMySQL. It is useful if you need to go back * \n"; - title = title + " * to a JSON representation of relational data. * \n"; + title = title + " * to a JSON representation from relational data. * \n"; title = title + " *********************************************************************** \n"; TCLAP::CmdLine cmd(title.toUtf8().data(), ' ', "2.0"); @@ -65,19 +65,10 @@ int main(int argc, char *argv[]) TCLAP::ValueArg userArg("u","user","User to connect to MySQL",true,"","string"); TCLAP::ValueArg passArg("p","password","Password to connect to MySQL",true,"","string"); TCLAP::ValueArg schemaArg("s","schema","Schema in MySQL",true,"","string"); - TCLAP::ValueArg createArg("x","createxml","Input create XML file",true,"","string"); - TCLAP::ValueArg tmpArg("T","tempdir","Temporary directory (./tmp by default)",false,"./tmp","string"); - TCLAP::ValueArg encryptArg("e","encrypt","32 char hex encryption key. Auto generate if empty",false,"","string"); - TCLAP::ValueArg tableArg("t","maintable","Main table name",true,"","string"); - TCLAP::ValueArg mapArg("m","mapdirectory","Directory containing the map XML files",true,"","string"); - TCLAP::ValueArg outArg("o","output","Output directory to store the JSON result files",true,"","string"); - TCLAP::ValueArg keyArg("k","key","Specific primary key to use",false,"","string"); - TCLAP::ValueArg valueArg("v","value","Specific primary key value to use",false,"","string"); - TCLAP::ValueArg separatorArg("S","separator","Separator to use in multi-selects. Pipe (|) is default",false,"|","string"); - TCLAP::ValueArg resolveArg("r","resolve","Resolve lookup values: 1=Codes only (default), 2=Descriptions, 3=Codes and descriptions",false,"1","string"); + TCLAP::ValueArg mapArg("m","mapfile","XML map file to use",true,"","string"); + TCLAP::ValueArg outArg("o","output","Output JSON file",true,"","string"); + TCLAP::ValueArg createArg("c","create","Create XML file",true,"","string"); - TCLAP::SwitchArg protectSwitch("c","protect","Protect sensitive fields. False by default", cmd, false); - TCLAP::SwitchArg ODKFormatSwitch("f","odkformat","Format like ODK Collect. Keys will be the same as if data was collected by ODK Collect", cmd, false); cmd.add(hostArg); @@ -85,75 +76,28 @@ int main(int argc, char *argv[]) cmd.add(userArg); cmd.add(passArg); cmd.add(schemaArg); - cmd.add(tmpArg); cmd.add(createArg); - cmd.add(encryptArg); - cmd.add(keyArg); - cmd.add(valueArg); - cmd.add(separatorArg); - cmd.add(tableArg); cmd.add(mapArg); cmd.add(outArg); - cmd.add(resolveArg); + //Parsing the command lines cmd.parse( argc, argv ); - //Getting the variables from the command - bool protectSensitive; - protectSensitive = protectSwitch.getValue(); - - bool likeODKCollect; - likeODKCollect = ODKFormatSwitch.getValue(); - - QString host = QString::fromUtf8(hostArg.getValue().c_str()); QString port = QString::fromUtf8(portArg.getValue().c_str()); QString user = QString::fromUtf8(userArg.getValue().c_str()); QString pass = QString::fromUtf8(passArg.getValue().c_str()); QString schema = QString::fromUtf8(schemaArg.getValue().c_str()); - QString key = QString::fromUtf8(keyArg.getValue().c_str()); - QString value = QString::fromUtf8(valueArg.getValue().c_str()); - QString separator = QString::fromUtf8(separatorArg.getValue().c_str()); - QString tmpDir = QString::fromUtf8(tmpArg.getValue().c_str()); - QString createXML = QString::fromUtf8(createArg.getValue().c_str()); - QString resolve_type = QString::fromUtf8(resolveArg.getValue().c_str()); - QString encryption_key = QString::fromUtf8(encryptArg.getValue().c_str()); - if (encryption_key == "") - { - encryption_key = getRandomHex(32); - } - QString mainTable = QString::fromUtf8(tableArg.getValue().c_str()); - QString mapDir = QString::fromUtf8(mapArg.getValue().c_str()); - QString outputDir = QString::fromUtf8(outArg.getValue().c_str()); + QString mapFile = QString::fromUtf8(mapArg.getValue().c_str()); + QString createFile = QString::fromUtf8(createArg.getValue().c_str()); + QString outputFile = QString::fromUtf8(outArg.getValue().c_str()); - if (key != "" && value == "") - { - log_error("You need to specify key and value"); - exit(1); - } - if (key == "" && value != "") - { - log_error("You need to specify key and value"); - exit(1); - } - - if (likeODKCollect && (key == "")) - { - log_error("You cannot use ODK format with more than one result. You need to specify key and value"); - exit(1); - } - if (likeODKCollect && resolve_type != "1") - { - log_error("You cannot use ODK format with resolving labels"); - exit(1); - } - mainClass *task = new mainClass(&app); - task->setParameters(host,port,user,pass,schema,createXML,protectSensitive,tmpDir,encryption_key,mapDir,outputDir,mainTable, resolve_type, key, value, separator, likeODKCollect); + task->setParameters(host,port,user,pass,schema,mapFile,outputFile,createFile); QObject::connect(task, SIGNAL(finished()), &app, SLOT(quit())); QTimer::singleShot(0, task, SLOT(run())); app.exec(); diff --git a/utilities/MySQLDenormalize/mainclass.cpp b/utilities/MySQLDenormalize/mainclass.cpp index ea9cbb0b..a6c71dda 100644 --- a/utilities/MySQLDenormalize/mainclass.cpp +++ b/utilities/MySQLDenormalize/mainclass.cpp @@ -1,28 +1,4 @@ #include "mainclass.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef Q_MOC_RUN -#include -#include -#include -#include -#include -#include -#include -#include -#endif mainClass::mainClass(QObject *parent) : QObject(parent) @@ -37,1093 +13,24 @@ void mainClass::log(QString message) printf("%s", temp.toUtf8().data()); } -void mainClass::setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, bool protectSensitive, QString tempDir, QString encryption_key, QString mapDir, QString outputDir, QString mainTable, QString resolve_type, QString primaryKey, QString primaryKeyValue, QString separator, bool useODKFormat) +void mainClass::setParameters(QString host,QString port,QString user,QString pass,QString schema,QString mapFile,QString outputFile,QString createFile) { this->host = host; this->port = port; this->user = user; this->pass = pass; this->schema = schema; - this->protectSensitive = protectSensitive; - this->tempDir = tempDir; - this->createXML = createXML; - this->encryption_key = encryption_key; - this->mapDir = mapDir; - this->outputDir = outputDir; - this->mainTable = mainTable; - this->resolve_type = resolve_type.toInt(); - this->primaryKey = primaryKey; - this->primaryKeyValue = primaryKeyValue; - this->separator = separator; - this->useODKFormat = useODKFormat; -} - -void mainClass::getMultiSelectInfo(QDomNode table, QString table_name, QString &multiSelect_field, QStringList &keys, QString &rel_table, QString &rel_field) -{ - QDomNode child = table.firstChild(); - while (!child.isNull()) - { - if (child.toElement().tagName() == "table") - { - if (child.toElement().attribute("name") == table_name) - { - QDomNode field = child.firstChild(); - while (!field.isNull()) - { - if (field.toElement().attribute("rlookup","false") == "true") - { - multiSelect_field = field.toElement().attribute("name"); - rel_table = field.toElement().attribute("rtable"); - rel_field = field.toElement().attribute("rfield"); - } - else - { - if (field.toElement().attribute("key","false") == "true") - { - keys.append(field.toElement().attribute("name")); - } - } - field = field.nextSibling(); - } - } - } - child = child.nextSibling(); - } -} - -void mainClass::loadTable(QDomNode table) -{ - QDomElement eTable; - eTable = table.toElement(); - - TtableDef aTable; - aTable.islookup = false; - aTable.name = eTable.attribute("name",""); - aTable.ODKname = eTable.attribute("xmlcode","NONE"); - aTable.desc = eTable.attribute("name",""); - - QDomNode field = table.firstChild(); - while (!field.isNull()) - { - QDomElement eField; - eField = field.toElement(); - if (eField.tagName() == "field") - { - TfieldDef aField; - aField.name = eField.attribute("name",""); - aField.ODKname = eField.attribute("xmlcode","NONE"); - aField.desc = eField.attribute("desc",""); - aField.type = eField.attribute("type",""); - aField.size = eField.attribute("size","").toInt(); - aField.decSize = eField.attribute("decsize","").toInt(); - - if (eField.attribute("rlookup","false") == "true") - { - aField.isLookUp = true; - aField.lookupRelTable = eField.attribute("rtable"); - aField.lookupRelField = eField.attribute("rfield"); - } - - if (eField.attribute("sensitive","false") == "true") - { - aField.sensitive = true; - aField.protection = eField.attribute("protection","exclude"); - } - else - aField.sensitive = false; - if (eField.attribute("key","false") == "true") - { - TfieldDef keyField; - keyField.name = aField.name; - keyField.replace_value = ""; - aField.isKey = true; - if (aField.sensitive == true) - { - if (protectedKeys.indexOf(aField.name) < 0) - protectedKeys.append(aField.name); - } - } - else - aField.isKey = false; - // NOTE ON Rank. Rank is basically a multiselect with order and handled as a multiselect by ODK Tools. However - // we cannot pull the data from the database because the records may not be stored in the same order the user placed them in Collect - if ((eField.attribute("isMultiSelect","false") == "true") && (eField.attribute("odktype","") != "rank")) - { - aField.isMultiSelect = true; - aField.multiSelectTable = eField.attribute("multiSelectTable"); - QString multiSelect_field; - QStringList keys; - QString multiSelectRelTable; - QString multiSelectRelField; - getMultiSelectInfo(table, aField.multiSelectTable, multiSelect_field, keys, multiSelectRelTable, multiSelectRelField); - aField.multiSelectField = multiSelect_field; - aField.multiSelectRelTable = multiSelectRelTable; - aField.multiSelectRelField = multiSelectRelField; - aField.multiSelectKeys.append(keys); - } - aTable.fields.append(aField); - } - else - { - loadTable(field); - } - field = field.nextSibling(); - } - mainTables.append(aTable); - -} - -int mainClass::generateXLSX() -{ - - QDomDocument docA("input"); - QFile fileA(createXML); - if (!fileA.open(QIODevice::ReadOnly)) - { - log("Cannot open input create XML file"); - returnCode = 1; - return returnCode; - } - if (!docA.setContent(&fileA)) - { - log("Cannot parse input create XML file"); - fileA.close(); - returnCode = 1; - return returnCode; - } - fileA.close(); - - //Load the lookup tables if asked - QDomElement rootA = docA.documentElement(); - - if (rootA.tagName() == "XMLSchemaStructure") - { - - //Getting the fields to export from tables - QDomNode table = rootA.firstChild().nextSibling().firstChild(); - - //Load the data tables recursively - loadTable(table); - for (int nt =mainTables.count()-1; nt >= 0;nt--) - { - if (mainTables[nt].name.indexOf("_msel_") < 0) - tables.append(mainTables[nt]); - } - - QDir currDir(tempDir); - QStringList arguments; - QProcess *mySQLDumpProcess = new QProcess(); - QElapsedTimer procTime; - procTime.start(); - QString sql; - QStringList fields; - QString uri = user + ":" + pass + "@" + host + "/" + schema; - - QStringList jsonFiles; - QString leftjoin; - QStringList leftjoins; - - QVector linked_tables; - QVector multiSelectTables; - for (int pos = 0; pos <= tables.count()-1; pos++) - { - //qDebug() << "Creating temp table " + tables[pos].name; - linked_tables.clear(); - - fields.clear(); - for (int fld = 0; fld < tables[pos].fields.count(); fld++) - { - if (this->protectSensitive) - { - if (tables[pos].fields[fld].sensitive == false) - { - if (tables[pos].fields[fld].isMultiSelect == false) - { - if (tables[pos].fields[fld].isKey == false) - { - if (tables[pos].fields[fld].isLookUp == false) - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - else - { - if (this->resolve_type != 1) - { - TlinkedTable a_linked_table; - a_linked_table.field = tables[pos].fields[fld].name; - a_linked_table.related_table = tables[pos].fields[fld].lookupRelTable; - a_linked_table.related_field = tables[pos].fields[fld].lookupRelField; - linked_tables.append(a_linked_table); - } - if (this->resolve_type == 3) - { - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - else - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - } - else - { - if (protectedKeys.indexOf(tables[pos].fields[fld].name) < 0) - { - - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - else - fields.append("HEX(AES_ENCRYPT(" + tables[pos].name + "." + tables[pos].fields[fld].name + ",UNHEX('" + this->encryption_key + "'))) as " + tables[pos].fields[fld].name); - } - } - else - { - fields.append(tables[pos].fields[fld].name); - TmultiSelectTable a_multiSelectTable; - a_multiSelectTable.field = tables[pos].fields[fld].name; - a_multiSelectTable.multiSelectTable = tables[pos].fields[fld].multiSelectTable; - a_multiSelectTable.multiSelectField = tables[pos].fields[fld].multiSelectField; - a_multiSelectTable.multiSelectRelTable = tables[pos].fields[fld].multiSelectRelTable; - a_multiSelectTable.multiSelectRelField = tables[pos].fields[fld].multiSelectRelField; - a_multiSelectTable.multiSelectKeys.append(tables[pos].fields[fld].multiSelectKeys); - multiSelectTables.append(a_multiSelectTable); - if (this->resolve_type == 3) - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - } - else - { - if (tables[pos].fields[fld].protection != "exclude") - fields.append("HEX(AES_ENCRYPT(" + tables[pos].name + "." + tables[pos].fields[fld].name + ",UNHEX('" + this->encryption_key + "'))) as " + tables[pos].fields[fld].name); - } - } - else - { - if (tables[pos].fields[fld].isMultiSelect == false) - { - if (tables[pos].fields[fld].isLookUp == false) - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - else - { - if (this->resolve_type != 1) - { - TlinkedTable a_linked_table; - a_linked_table.field = tables[pos].fields[fld].name; - a_linked_table.related_table = tables[pos].fields[fld].lookupRelTable; - a_linked_table.related_field = tables[pos].fields[fld].lookupRelField; - linked_tables.append(a_linked_table); - } - if (this->resolve_type == 3) - { - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - else - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - } - else - { - fields.append(tables[pos].fields[fld].name); - TmultiSelectTable a_multiSelectTable; - a_multiSelectTable.field = tables[pos].fields[fld].name; - a_multiSelectTable.multiSelectTable = tables[pos].fields[fld].multiSelectTable; - a_multiSelectTable.multiSelectField = tables[pos].fields[fld].multiSelectField; - a_multiSelectTable.multiSelectRelTable = tables[pos].fields[fld].multiSelectRelTable; - a_multiSelectTable.multiSelectRelField = tables[pos].fields[fld].multiSelectRelField; - a_multiSelectTable.multiSelectKeys.append(tables[pos].fields[fld].multiSelectKeys); - multiSelectTables.append(a_multiSelectTable); - if (this->resolve_type == 3) - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - } - } - QString temp_table; - sql = "SET SQL_MODE = '';\n"; - - QUuid recordUUID=QUuid::createUuid(); - temp_table = "TMP_" + recordUUID.toString().replace("{","").replace("}","").replace("-","_"); - if (primaryKey == "") - sql = sql + "CREATE TABLE " + temp_table + " ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci AS SELECT " + fields.join(",") + " FROM " + tables[pos].name + ";"; - else - { - if (primaryKey != "" && primaryKeyValue != "") - sql = sql + "CREATE TABLE " + temp_table + " ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci AS SELECT " + fields.join(",") + " FROM " + tables[pos].name + " WHERE " + primaryKey + " = '" + primaryKeyValue + "';"; - else - sql = sql + "CREATE TABLE " + temp_table + " ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci AS SELECT " + fields.join(",") + " FROM " + tables[pos].name + ";"; - } - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - - QFile tempfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - if (!tempfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - delete mySQLDumpProcess; - return 1; - } - QTextStream temout(&tempfile); - temout << sql; - tempfile.close(); - - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) - { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; - return 1; - } - arguments.clear(); - - QStringList sqls; - //qDebug() << "Performing Alters on temp table"; - for (int fld = 0; fld < tables[pos].fields.count(); fld++) - { - if (tables[pos].fields[fld].isKey) - sqls.append("ALTER TABLE " + temp_table + " MODIFY COLUMN " + tables[pos].fields[fld].name + " VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\n"); - } - - if (multiSelectTables.count() > 0) - { - QStringList modifies; - for (int i_table=0; i_table < multiSelectTables.count(); i_table++) - { - modifies.append("MODIFY COLUMN " + multiSelectTables[i_table].field + " TEXT"); - if (this->resolve_type == 3) - modifies.append("MODIFY COLUMN `" + multiSelectTables[i_table].field + "-desc` TEXT"); - } - sql = "ALTER TABLE " + temp_table + " " + modifies.join(",") + ";\n"; - sqls.append(sql); - } - - if (linked_tables.count() > 0) - { - QStringList modifies; - for (int i_table=0; i_table < linked_tables.count(); i_table++) - { - sql = "MODIFY COLUMN "; - if (this->resolve_type == 2) - sql = sql + linked_tables[i_table].field; - else - sql = sql + "`" + linked_tables[i_table].field + "-desc`"; - sql = sql + " TEXT"; - modifies.append(sql); - } - sql = "ALTER TABLE " + temp_table + " " + modifies.join(",") + ";\n"; - sqls.append(sql); - - - for (int i_table=0; i_table < linked_tables.count(); i_table++) - { - sql = "UPDATE " + temp_table + "," + linked_tables[i_table].related_table + " AS T" + QString::number(i_table) + " SET "; - QString relfield = linked_tables[i_table].related_field; - if (this->resolve_type == 2) - sql = sql + temp_table + "." + linked_tables[i_table].field + " = T" + QString::number(i_table) + "." + relfield.replace("_cod","_des"); - else - sql = sql + temp_table + ".`" + linked_tables[i_table].field + "-desc` = T" + QString::number(i_table) + "." + relfield.replace("_cod","_des"); - sql = sql + " WHERE " + temp_table + "." + linked_tables[i_table].field + " = T" + QString::number(i_table) + "." + linked_tables[i_table].related_field + ";\n"; - sqls.append(sql); - } - } - if (multiSelectTables.count() > 0) - { - for (int i_table=0; i_table < multiSelectTables.count(); i_table++) - { - if (this->resolve_type == 1 || this->resolve_type == 3) - { - sql = "UPDATE " + temp_table + " AS TA SET TA." + multiSelectTables[i_table].field + " = (SELECT GROUP_CONCAT(TB." + multiSelectTables[i_table].multiSelectField + " SEPARATOR '" + separator + "') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB"; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " WHERE " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - if (this->resolve_type == 2) - { - QString desc_field = multiSelectTables[i_table].multiSelectRelField; - desc_field = desc_field.replace("_cod","_des"); - sql = "UPDATE " + temp_table + " AS TA SET TA." + multiSelectTables[i_table].field + " = (SELECT GROUP_CONCAT(TC." + desc_field + " SEPARATOR '" + separator + "') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB," + multiSelectTables[i_table].multiSelectRelTable + " as TC"; - sql = sql + " WHERE TB." + multiSelectTables[i_table].multiSelectField + " = TC." + multiSelectTables[i_table].multiSelectRelField; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " AND " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - if (this->resolve_type == 3) - { - QString desc_field = multiSelectTables[i_table].multiSelectRelField; - desc_field = desc_field.replace("_cod","_des"); - sql = "UPDATE " + temp_table + " AS TA SET TA.`" + multiSelectTables[i_table].field + "-desc` = (SELECT GROUP_CONCAT(TC." + desc_field + " SEPARATOR '" + separator + "') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB," + multiSelectTables[i_table].multiSelectRelTable + " as TC"; - sql = sql + " WHERE TB." + multiSelectTables[i_table].multiSelectField + " = TC." + multiSelectTables[i_table].multiSelectRelField; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " AND " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - } - } -// qDebug() << "*****************" + tables[pos].name + "**********************" ; -// for (int p=0; p < sqls.count(); p++) -// qDebug() << sqls[p]; - - if (sqls.count() > 0) - { - QFile modfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - if (!modfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - delete mySQLDumpProcess; - return 1; - } - QTextStream modOut(&modfile); - for (int isql=0; isql < sqls.count(); isql++) - modOut << sqls[isql]; - modfile.close(); - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) - { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; - return 1; - } - } - linked_tables.clear(); - multiSelectTables.clear(); - //qDebug() << "Quering table " + tables[pos].name; - if (primaryKey == "") - sql = "SELECT * FROM " + temp_table + ";"; - else - { - if (primaryKey != "" && primaryKeyValue != "") - { - sql = "SELECT * FROM " + temp_table + " WHERE " + primaryKey + " = '" + primaryKeyValue + "';"; - } - else - sql = "SELECT * FROM " + temp_table + ";"; - } - //qDebug() << sql; - arguments.clear(); - arguments << "--sql"; - arguments << "--result-format=json/raw"; - arguments << "--uri=" + uri; - QFile sqlfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - if (!sqlfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - delete mySQLDumpProcess; - return 1; - } - QTextStream out(&sqlfile); - out << sql; - sqlfile.close(); - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".txt"); - mySQLDumpProcess->start("mysqlsh", arguments); - mySQLDumpProcess->waitForFinished(-1); - if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) - { - if (mySQLDumpProcess->error() == QProcess::FailedToStart) - { - log("Error: Command mysqlsh not found"); - } - else - { - log("Running mysqlsh returned error"); - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - log("Running paremeters:" + arguments.join(" ")); - } - delete mySQLDumpProcess; - return 1; - } - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - arguments.append("--execute=DROP TABLE " + temp_table ); - - mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) - { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; - return 1; - } - - jsonFiles.append(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".txt"); - } - - QString DataFile = currDir.absolutePath() + currDir.separator() + "data.txt"; - if (jsonFiles.count() > 1) - { - arguments.clear(); - for (int pos = 0; pos < jsonFiles.count(); pos++) - { - arguments.append(jsonFiles[pos]); - } - mySQLDumpProcess->setStandardOutputFile(DataFile); - mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("cat", arguments); - mySQLDumpProcess->waitForFinished(-1); - if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) - { - if (mySQLDumpProcess->error() == QProcess::FailedToStart) - { - log("Error: Command xsv not found"); - } - else - { - log("Running xsv returned error"); - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - log("Running paremeters:" + arguments.join(" ")); - } - delete mySQLDumpProcess; - return 1; - } - } - else - { - if (jsonFiles.count() > 0) - QFile::rename(jsonFiles[0],DataFile); - } - - QFileInfo dataInfo(DataFile); - if (dataInfo.size() == 0) - { - qDebug() << "There is no data to process"; - delete mySQLDumpProcess; - return 1; - } - - QFileInfo outputDir(currDir.absolutePath() + currDir.separator() + "jsons"); - if (!outputDir.exists()) - currDir.mkdir(currDir.absolutePath() + currDir.separator() + "jsons"); - currDir.setPath(currDir.absolutePath() + currDir.separator() + "jsons"); - - mongocxx::instance instance{}; // This should be done only once. - mongocxx::uri mongo_uri("mongodb://localhost:27017"); - mongocxx::client client(mongo_uri); - mongocxx::database mongoDB = client["mysqldenormalize"]; - //We create a collection based on the last 12 digits of a UUID - QUuid collectionUUID=QUuid::createUuid(); - QString strCollectionUUID=collectionUUID.toString().replace("{","").replace("}",""); - strCollectionUUID = "C" + strCollectionUUID.right(12); - mongocxx::collection coll = mongoDB[strCollectionUUID.toUtf8().constData()]; - - auto index_specification = bsoncxx::builder::stream::document{} << "rowuuid" << 1 << bsoncxx::builder::stream::finalize; - coll.create_index(std::move(index_specification)); - - arguments.clear(); - arguments.append("--db"); - arguments.append("mysqldenormalize"); - arguments.append("--collection"); - arguments.append(strCollectionUUID); - arguments.append("--file"); - arguments.append(DataFile); - - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mongoimport", arguments); - mySQLDumpProcess->waitForFinished(-1); - if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) - { - if (mySQLDumpProcess->error() == QProcess::FailedToStart) - { - log("Error: Command mongoimport not found"); - } - else - { - log("Running mongoimport returned error"); - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - log("Running paremeters:" + arguments.join(" ")); - } - delete mySQLDumpProcess; - return 1; - } - - delete mySQLDumpProcess; - - QSqlDatabase db = QSqlDatabase::addDatabase("QMYSQL"); - db.setHostName(host); - db.setPort(port.toInt()); - db.setDatabaseName(schema); - db.setUserName(user); - db.setPassword(pass); - //db.setConnectOptions("MYSQL_OPT_SSL_MODE=SSL_MODE_DISABLED"); - if (db.open()) - { - mongo_collection = coll; - if (primaryKey == "") - sql = "SELECT surveyid FROM " + mainTable; - else - { - if (primaryKey != "" && primaryKeyValue != "") - sql = "SELECT surveyid FROM " + mainTable + " WHERE " + primaryKey + " = '" + primaryKeyValue + "'"; - else - sql = "SELECT surveyid FROM " + mainTable; - } - QStringList lstIds; - QSqlQuery qryIds(db); - qryIds.exec(sql); - while (qryIds.next()) - lstIds << qryIds.value(0).toString(); - - //log("Generating trees"); - for (int pos = 0; pos <= lstIds.count()-1; pos++) - { - processMapFile(lstIds[pos]); - } -// if (lstIds.count() >= 100000) -// { -// // If we are working with more than 1000 submissions -// // then separate the creation of map files into 20 threads -// int size = lstIds.count() / 2; -// int pos = 0, arrsize = lstIds.size(), sizeInArray = size; -// QList arrays; -// while(pos> threads; -// for( int i=0; imapFile = mapFile; + this->outputFile = outputFile; + this->createFile = createFile; } void mainClass::run() { - if (QFile::exists(createXML)) - { - QDir tDir; - if (!tDir.exists(tempDir)) - { - if (!tDir.mkdir(tempDir)) - { - log("Cannot create temporary directory"); - returnCode = 1; - emit finished(); - } - } - - if (generateXLSX() == 0) - { - returnCode = 0; - emit finished(); - } - else - { - returnCode = 1; - emit finished(); - } - } - else - { - log("The create XML file does not exists"); - returnCode = 1; - emit finished(); - } + returnCode = 0; + emit finished(); } -QList mainClass::getDataByRowUUID4(QVector dataList, QString UUIDToSearch) -{ - QList records; - - for (int pos = 0; pos <= dataList.count()-1;pos++) - { - if ((dataList[pos].UUID == UUIDToSearch)) - { - return dataList[pos].fields; - } - } - - log("Empty result for UUID " + UUIDToSearch); - - return records; -} - - -void mainClass::parseMapFileWithBoost(QVector dataList, QDomNode node, pt::ptree &json, pt::ptree &parent) -{ - QDomElement elem; - elem = node.toElement(); - QString tableName; - QString UUID; - tableName = elem.attribute("table"); - UUID = elem.attribute("uuid"); - //Get the data for a UUID in a table and add it to the JSON object - QList records; - if (tableName.indexOf("_msel_") == -1) - records = getDataByRowUUID4(dataList,UUID); - for (int pos = 0; pos <= records.count()-1; pos++) - { - json.put(records[pos].name.toStdString(), records[pos].value.toStdString()); - } - //If the current node has a child record then process the child - //by recursively call this process. The subtable is a JSON array - if (!node.firstChild().isNull()) - { - elem = node.firstChild().toElement(); - tableName = elem.attribute("table"); - - pt::ptree childObject; - parseMapFileWithBoost(dataList,node.firstChild(),childObject,json); //RECURSIVE!!! - pt::ptree array; - pt::ptree::const_assoc_iterator it; - it = json.find(tableName.toStdString()); - if (it != json.not_found()) - array = json.get_child(tableName.toStdString()); - array.push_back(std::make_pair("", childObject)); - if (tableName.indexOf("_msel_") == -1) - json.put_child(tableName.toStdString(), array); - - } - //If the current node has siblings. - if (!node.nextSibling().isNull()) - { - QDomNode nextSibling; - nextSibling = node.nextSibling(); - //Go trhough each sibbling - while (!nextSibling.isNull()) - { - elem = nextSibling.toElement(); - tableName = elem.attribute("table"); - UUID = elem.attribute("uuid"); - //New siblings usually refer to records in the same table - //We only create an JSON Array if the table changes from - //one sibling to another - //Each sibling table is stored as a JSON Array - pt::ptree array2; - pt::ptree::const_assoc_iterator it2; - it2= parent.find(tableName.toStdString()); - if (it2 != parent.not_found()) - array2 = parent.get_child(tableName.toStdString()); - pt::ptree childObject2; - QList records2; - if (tableName.indexOf("_msel_") == -1) - records2 = getDataByRowUUID4(dataList,UUID); - for (int pos = 0; pos <= records2.count()-1; pos++) - { - childObject2.put(records2[pos].name.toStdString(),records2[pos].value.toStdString()); - } - //If the sibling has a child then recursively call - //this function. - if (!nextSibling.firstChild().isNull()) - { - QString tableName2; - QDomElement elem2; - elem2 = nextSibling.firstChild().toElement(); - tableName2 = elem2.attribute("table"); - pt::ptree childObject3; - parseMapFileWithBoost(dataList,nextSibling.firstChild(),childObject3,childObject2); //!!RECURSIVE - pt::ptree array3; - - pt::ptree::const_assoc_iterator it3; - it3 = childObject2.find(tableName2.toStdString()); - if (it3 != childObject2.not_found()) - array3 = childObject2.get_child(tableName2.toStdString()); - array3.push_back(std::make_pair("", childObject3)); - if (tableName2.indexOf("_msel_") == -1) - childObject2.put_child(tableName2.toStdString(), array3); - } - array2.push_back(std::make_pair("", childObject2)); - if (tableName.indexOf("_msel_") == -1) - parent.put_child(tableName.toStdString(), array2); - - nextSibling = nextSibling.nextSibling(); - } - } -} - - -void mainClass::getAllUUIDs(QDomNode node,QStringList &UUIDs) -{ - QDomNode sibling; - sibling = node; - while (!sibling.isNull()) - { - QDomElement eSibling; - eSibling = sibling.toElement(); - if (eSibling.attribute("table","None") != "None" && eSibling.attribute("uuid","None") != "None") - { - QString table = eSibling.attribute("table","None"); - if (table.indexOf("_msel_") < 0) - { - UUIDs.append(eSibling.attribute("uuid","None")); - } - } - if (!sibling.firstChild().isNull()) - getAllUUIDs(sibling.firstChild(),UUIDs); - sibling = sibling.nextSibling(); - } -} - -void mainClass::processMapFile(QString fileName) -{ - QDir mapPath(mapDir); - QString mapFile; - mapFile = mapPath.absolutePath() + mapPath.separator() + fileName + ".xml"; - - QDomDocument doc("mapfile"); - QFile file(mapFile); - if (!file.open(QIODevice::ReadOnly)) - { - log("Map file " + mapFile + " not found"); - return; - } - if (!doc.setContent(&file)) - { - file.close(); - log("Cannot parse map file " + mapFile); - return; - } - file.close(); - QDomNode root; - root = doc.firstChild().nextSibling().firstChild(); - - QStringList UUIDs; - getAllUUIDs(root,UUIDs); - QString mongoQry; - mongoQry = "{ \"$or\" :["; - for (int pos = 0; pos <= UUIDs.count()-1;pos++) - { - mongoQry = mongoQry + "{\"rowuuid\":\"" + UUIDs[pos] + "\"},"; - } - mongoQry = mongoQry.left(mongoQry.length()-1); - mongoQry = mongoQry + "]}"; - std::string utf8_text = mongoQry.toUtf8().constData(); - bsoncxx::string::view_or_value qry(utf8_text); - bsoncxx::document::value bsondoc = bsoncxx::from_json(qry.view()); - mongocxx::cursor cursor = mongo_collection.find(bsondoc.view()); - - QVector dataList; - for (bsoncxx::document::view doc : cursor) - { - TUUIDDef aRowUUID; - for (bsoncxx::document::element ele : doc) - { - mongocxx::stdx::string_view field_key{ele.key()}; - QString key = QString::fromUtf8(field_key.data()); - QString value; - if (key != "_id") - { - if (ele.type() == bsoncxx::type::k_utf8) - { - value = QString::fromUtf8(ele.get_utf8().value.data()); - if (key == "rowuuid") - aRowUUID.UUID = value; - } - if (ele.type() == bsoncxx::type::k_null) - { - value = ""; - } - if (ele.type() == bsoncxx::type::k_double) - { - value = QString::number(ele.get_double()); - } - if (ele.type() == bsoncxx::type::k_int32) - { - value = QString::number(ele.get_int32()); - } - if (ele.type() == bsoncxx::type::k_int64) - { - value = QString::number(ele.get_int64()); - } - TUUIDFieldDef aField; - aField.name = key; - aField.value = value; - aRowUUID.fields.append(aField); - } - } - dataList.append(aRowUUID); - } - - QDomElement elem; - elem = root.toElement(); - pt::ptree JSONRootBoost; - parseMapFileWithBoost(dataList,root,JSONRootBoost,JSONRootBoost); - QDir outputPath(outputDir); - - QString JSONFileBoost; - JSONFileBoost = outputPath.absolutePath() + mapPath.separator() + fileName + ".json"; - pt::write_json(JSONFileBoost.toStdString(),JSONRootBoost); - - if (useODKFormat) - { - QString BKFile; - BKFile = outputPath.absolutePath() + mapPath.separator() + fileName + ".bk"; - - QProcess *mySQLDumpProcess = new QProcess(); - - //Rename the primary key - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq 'with_entries(if .key == \"" + primaryKey + "\" then .key = \"PRIMARY\" else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - for (int t=0; t < tables.count(); t++) - { - if (tables[t].ODKname != "main" && tables[t].ODKname != "NONE") - { - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq '(.. | select(has(\"" + tables[t].name + "\")?)) |= with_entries(if .key == \"" + tables[t].name + "\" then .key = \"" + tables[t].ODKname + "\" else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - } - for (int f=0; f < tables[t].fields.count(); f++) - { - if (tables[t].fields[f].ODKname != "NONE") - { - if (tables[t].fields[f].isKey == false) - { - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq '(.. | select(has(\"" + tables[t].fields[f].name + "\")?)) |= with_entries(if .key == \"" + tables[t].fields[f].name + "\" then .key = \"" + tables[t].fields[f].ODKname + "\" else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - } - else - { - // Remove all keys - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq 'walk(if type == \"object\" then del(." + tables[t].fields[f].name + ") else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - } - } - else - { - // Remove all interal columns like rowuuid - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq 'walk(if type == \"object\" then del(." + tables[t].fields[f].name + ") else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - } - } - } - // Rename the primary key back - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq 'with_entries(if .key == \"PRIMARY\" then .key = \"" + primaryKey + "\" else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - - for (int t=0; t < tables.count(); t++) - { - for (int f=0; f < tables[t].fields.count(); f++) - { - if (tables[t].fields[f].ODKname != "NONE") - { - if (tables[t].fields[f].isKey == true) - { - mySQLDumpProcess->start("bash", QStringList() << "-c" << "jq '(.. | select(has(\"" + tables[t].fields[f].name + "\")?)) |= with_entries(if .key == \"" + tables[t].fields[f].name + "\" then .key = \"" + tables[t].fields[f].ODKname + "\" else . end)' " + JSONFileBoost + " > " + BKFile + " && mv " + BKFile + " " + JSONFileBoost); - mySQLDumpProcess->waitForFinished(-1); - } - } - } - } - - QDomDocument XMLResult; - XMLResult = QDomDocument("XMLOutputFile"); - QDomElement XMLRoot; - XMLRoot = XMLResult.createElement("XMLOutputFile"); - XMLResult.appendChild(XMLRoot); - QDomElement eDuplicatedItem; - eDuplicatedItem = XMLResult.createElement("OutputFile"); - eDuplicatedItem.setAttribute("fileName",JSONFileBoost); - XMLRoot.appendChild(eDuplicatedItem); - log(XMLResult.toString()); - - delete mySQLDumpProcess; - } - -} diff --git a/utilities/MySQLDenormalize/mainclass.h b/utilities/MySQLDenormalize/mainclass.h index 03565e3d..b4d4c9fe 100644 --- a/utilities/MySQLDenormalize/mainclass.h +++ b/utilities/MySQLDenormalize/mainclass.h @@ -2,88 +2,13 @@ #define MAINCLASS_H #include -#include -#ifndef Q_MOC_RUN -#include -#include -#endif - -namespace pt = boost::property_tree; - -struct fieldDef -{ - QString name; //Field Name - QString ODKname; //Field name in ODK - QString type; //Variable type in MySQL - QString desc; //Variable description - int size; //Variable size - int decSize; //Variable decimal size - bool isMultiSelect = false; - bool isKey = false; - bool isLookUp = false; - QString multiSelectTable; - QString multiSelectField; - QStringList multiSelectKeys; - QString multiSelectRelTable; - QString multiSelectRelField; - QString lookupRelTable; - QString lookupRelField; - QString replace_value; - QString value; - bool sensitive; - QString protection; -}; -typedef fieldDef TfieldDef; - -struct tableDef -{ - QString name; - QString desc; - QString ODKname; - QList fields; //List of fields - bool islookup; //Whether the table is a lookup table -}; -typedef tableDef TtableDef; - -struct UUIDFieldDef -{ - QString name; - QString value; -}; -typedef UUIDFieldDef TUUIDFieldDef; - -struct UUIDDef -{ - QString UUID; - QList fields; -}; -typedef UUIDDef TUUIDDef; - -struct linkedTable -{ - QString field; - QString related_table; - QString related_field; -}; -typedef linkedTable TlinkedTable; - -struct multiSelectTable -{ - QString field; - QString multiSelectTable; - QString multiSelectField; - QString multiSelectRelTable; - QString multiSelectRelField; - QStringList multiSelectKeys; -}; -typedef multiSelectTable TmultiSelectTable; class mainClass : public QObject { Q_OBJECT public: explicit mainClass(QObject *parent = nullptr); - void setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, bool protectSensitive, QString tempDir, QString encryption_key, QString mapDir, QString outputDir, QString mainTable, QString resolve_type, QString primaryKey, QString primaryKeyValue, QString separator, bool useODKFormat); + void setParameters(QString host,QString port,QString user,QString pass,QString schema,QString mapFile,QString outputFile,QString createFile); int returnCode; signals: void finished(); @@ -91,36 +16,14 @@ public slots: void run(); private: void log(QString message); - int generateXLSX(); - void loadTable(QDomNode node); - void getMultiSelectInfo(QDomNode table, QString table_name, QString &multiSelect_field, QStringList &keys, QString &rel_table, QString &rel_field); QString host; QString port; QString user; QString pass; - QString schema; - QString tempDir; - QString createXML; - QString encryption_key; - bool protectSensitive; - QList tables; - QList mainTables; - QStringList tableNames; - QStringList protectedKeys; - void processMapFile(QString fileName); - QString mapDir; - QString outputDir; - QString mainTable; - int resolve_type; - QString primaryKey; - QString primaryKeyValue; - bool useODKFormat; - QString separator; - mongocxx::collection mongo_collection; - void getAllUUIDs(QDomNode node,QStringList &UUIDs); - void parseMapFileWithBoost(QVector dataList, QDomNode node, pt::ptree &json, pt::ptree &parent); - QList getDataByRowUUID4(QVector dataList, QString UUIDToSearch); - void processSection(QStringList section); + QString schema; + QString mapFile; + QString outputFile; + QString createFile; }; #endif // MAINCLASS_H diff --git a/utilities/MySQLToCSV/main.cpp b/utilities/MySQLToCSV/main.cpp index b0684f80..e9ccbfc7 100644 --- a/utilities/MySQLToCSV/main.cpp +++ b/utilities/MySQLToCSV/main.cpp @@ -113,7 +113,7 @@ int main(int argc, char *argv[]) QString resolve_type = QString::fromUtf8(resolveArg.getValue().c_str()); bool ok; - if (resolve_type <= 0 && resolve_type > 3) + if (resolve_type.toInt() <= 0 && resolve_type.toInt() > 3) { log_out("Resolving code must be 1, 2, or 3"); exit(1); diff --git a/utilities/MySQLToSQLite/main.cpp b/utilities/MySQLToSQLite/main.cpp index d8ff5eec..628fbcb5 100644 --- a/utilities/MySQLToSQLite/main.cpp +++ b/utilities/MySQLToSQLite/main.cpp @@ -534,7 +534,9 @@ int main(int argc, char *argv[]) if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) return 1; QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif out << "BEGIN;\n"; for (int pos = 0; pos < lst_tables.count(); pos++) { diff --git a/utilities/MySQLToSTATA/MySQLToSTATA.pro b/utilities/MySQLToSTATA/MySQLToSTATA.pro index 3187c2bf..defb674d 100644 --- a/utilities/MySQLToSTATA/MySQLToSTATA.pro +++ b/utilities/MySQLToSTATA/MySQLToSTATA.pro @@ -1,7 +1,7 @@ QT -= gui -QT += core xml +QT += core xml sql -CONFIG += console +CONFIG += c++11 console CONFIG -= app_bundle TARGET = mysqltostata @@ -19,7 +19,11 @@ DEFINES += QT_DEPRECATED_WARNINGS unix:INCLUDEPATH += ../../3rdparty SOURCES += main.cpp \ + jsonworker.cpp \ + listmutex.cpp \ mainclass.cpp HEADERS += \ + jsonworker.h \ + listmutex.h \ mainclass.h diff --git a/utilities/MySQLToSTATA/jsonworker.cpp b/utilities/MySQLToSTATA/jsonworker.cpp new file mode 100644 index 00000000..f3361e99 --- /dev/null +++ b/utilities/MySQLToSTATA/jsonworker.cpp @@ -0,0 +1,255 @@ +#include "jsonworker.h" +#include +#include +#include +#include +#include +#include +#include +#include + +JSONWorker::JSONWorker(QObject *parent) + : QThread{parent} +{ + +} + +void JSONWorker::setParameters(QString host, QString port, QString user, QString pass, QString schema, QDir currDir, QString outputDirectory) +{ + this->host = host; + this->port = port; + this->user = user; + this->pass = pass; + this->schema = schema; + this->currDir = currDir; + this->outputDirectory = outputDirectory; +} + +void JSONWorker::setName(QString name) +{ + this->name = name; +} + +void JSONWorker::log(QString message) +{ + QString temp; + temp = message + "\n"; + printf("%s", temp.toUtf8().data()); +} + +void JSONWorker::run() +{ + QProcess *mySQLDumpProcess = new QProcess(); + QStringList arguments; + QString fixed_pass(QUrl::toPercentEncoding(pass)); + QString fixed_user(QUrl::toPercentEncoding(user)); + QString uri = fixed_user + ":" + fixed_pass + "@" + host + ":" + port + "/" + schema; + + int index = mutex->get_index(); + //qDebug() << name + " - Processing index: " + QString::number(index); + while (index >= 0) + { + //qDebug() << name + " - Processing index: " + QString::number(index); + // Create the table + if (task_list[index].task_type == 1) + { + //qDebug() << name + " - Processing file: " + task_list[index].sql_file; + arguments.clear(); + arguments.append("--host=" + this->host); + arguments.append("--port=" + this->port); + arguments.append("--password=" + this->pass); + arguments.append("--user=" + this->user); + arguments.append("--database=" + this->schema); + + mySQLDumpProcess->setStandardInputFile(task_list[index].sql_file); + mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); + mySQLDumpProcess->start("mysql", arguments); + mySQLDumpProcess->waitForFinished(-1); + if (mySQLDumpProcess->exitCode() > 0) + { + QString serror = mySQLDumpProcess->readAllStandardError(); + log(serror); + delete mySQLDumpProcess; + this->status = 1; + return; + } + } + if (task_list[index].task_type == 2) + { + QUuid home_id=QUuid::createUuid(); + QString user_home_id=home_id.toString().replace("{","").replace("}",""); + QFileInfo finfo(task_list[index].sql_file); + QString user_home = finfo.canonicalPath() + "/" + user_home_id; + QDir().mkdir(user_home); + +// QFile options_file(user_home + "/options.json"); +// options_file.open(QIODevice::WriteOnly); +// options_file.close(); + + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + env.insert("MYSQLSH_USER_CONFIG_HOME", user_home); + + arguments.clear(); + arguments << "--sql"; + arguments << "--result-format=json/array"; + arguments << "--uri=" + uri; + + mySQLDumpProcess->setProcessEnvironment(env); + mySQLDumpProcess->setStandardInputFile(task_list[index].sql_file); + mySQLDumpProcess->setStandardOutputFile(task_list[index].json_file); + mySQLDumpProcess->start("mysqlsh", arguments); + mySQLDumpProcess->waitForFinished(-1); + if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) + { + if (mySQLDumpProcess->error() == QProcess::FailedToStart) + { + log("Error: Command mysqlsh not found"); + } + else + { + log("Running mysqlsh returned error"); + QString serror = mySQLDumpProcess->readAllStandardError(); + log(serror); + log("Running paremeters:" + arguments.join(" ")); + } + delete mySQLDumpProcess; + this->status = 1; + return; + } + else + { + this->msleep(500); + } + } + if (task_list[index].task_type == 3) + { + if (task_list[index].json_files.count() > 1) + { + //qDebug() << name + " - Creating final file: " + task_list[index].final_file; + arguments.clear(); + arguments << "-s"; + arguments << "reduce .[] as $x ([]; . + $x)"; + for (int f=0; f < task_list[index].json_files.count(); f++) + { + arguments << task_list[index].json_files[f]; + } + mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); + mySQLDumpProcess->setStandardOutputFile(task_list[index].final_file); + + mySQLDumpProcess->start("jq", arguments); + mySQLDumpProcess->waitForFinished(-1); + if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) + { + if (mySQLDumpProcess->error() == QProcess::FailedToStart) + { + log("Error: Command jq not found"); + } + else + { + log("Running jq returned error"); + QString serror = mySQLDumpProcess->readAllStandardError(); + log(serror); + log("Running paremeters:" + arguments.join(" ")); + } + delete mySQLDumpProcess; + this->status = 1; + return; + } + } + else + { + QFile::copy(task_list[index].json_files[0],task_list[index].final_file); + } + + QFileInfo info(task_list[index].final_file); + if (info.size() > 0) + { + QString final_csv_file = task_list[index].final_file; + final_csv_file = final_csv_file.replace(".json",".raw"); + arguments.clear(); + arguments << "-d"; + arguments << " "; + arguments << "-i"; + arguments << task_list[index].final_file; + arguments << "-o"; + arguments << final_csv_file; + + mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); + mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); + mySQLDumpProcess->start("json2csv", arguments); + mySQLDumpProcess->waitForFinished(-1); + if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) + { + if (mySQLDumpProcess->error() == QProcess::FailedToStart) + { + log("Error: Command jq not found"); + } + else + { + log("Running jq returned error"); + QString serror = mySQLDumpProcess->readAllStandardError(); + log(serror); + log("Running paremeters:" + arguments.join(" ")); + } + delete mySQLDumpProcess; + this->status = 1; + return; + } + } + QFile::remove(task_list[index].final_file); + } + if (task_list[index].task_type == 4) + { + QFileInfo info(task_list[index].json_files[0]); + if (info.size() > 0) + { + QString final_csv_file = task_list[index].final_file; + final_csv_file = final_csv_file.replace(".json",".raw"); + arguments.clear(); + arguments << "-d"; + arguments << " "; + arguments << "-i"; + arguments << task_list[index].json_files[0]; + arguments << "-o"; + arguments << final_csv_file; + + mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); + mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); + mySQLDumpProcess->start("json2csv", arguments); + mySQLDumpProcess->waitForFinished(-1); + if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) + { + if (mySQLDumpProcess->error() == QProcess::FailedToStart) + { + log("Error: Command json2csv not found"); + } + else + { + log("Running json2csv returned error"); + QString serror = mySQLDumpProcess->readAllStandardError(); + log(serror); + log("Running paremeters:" + arguments.join(" ")); + } + delete mySQLDumpProcess; + this->status = 1; + return; + } + } + QFile::remove(task_list[index].final_file); + } + index = mutex->get_index(); + } + + delete mySQLDumpProcess; + this->status = 0; +} + +void JSONWorker::setTasks(QList< TtaskItem> task_list) +{ + this->task_list = task_list; +} + +void JSONWorker::setMutex(ListMutex *mutex) +{ + this->mutex = mutex; +} diff --git a/utilities/MySQLToSTATA/jsonworker.h b/utilities/MySQLToSTATA/jsonworker.h new file mode 100644 index 00000000..40d79141 --- /dev/null +++ b/utilities/MySQLToSTATA/jsonworker.h @@ -0,0 +1,44 @@ +#ifndef JSONWORKER_H +#define JSONWORKER_H + +#include +#include +#include "listmutex.h" +#include "mainclass.h" + +//struct taskItem +//{ +// QString table; +// QString create_sql; +// QString alter_sql; +// QString query_sql; +// QString json_file; +//}; +//typedef taskItem TtaskItem; + +class JSONWorker : public QThread +{ + Q_OBJECT +public: + explicit JSONWorker(QObject *parent = nullptr); + void run(); + void setTasks(QList< TtaskItem> task_list); + void setMutex(ListMutex *mutex); + void setName(QString name); + void setParameters(QString host, QString port, QString user, QString pass, QString schema, QDir currDir, QString outputDirectory); + int status; +private: + void log(QString message); + QList< TtaskItem> task_list; + ListMutex *mutex; + QString host; + QString port; + QString pass; + QString user; + QString schema; + QDir currDir; + QString outputDirectory; + QString name; +}; + +#endif // JSONWORKER_H diff --git a/utilities/MySQLToSTATA/listmutex.cpp b/utilities/MySQLToSTATA/listmutex.cpp new file mode 100644 index 00000000..71a36c7a --- /dev/null +++ b/utilities/MySQLToSTATA/listmutex.cpp @@ -0,0 +1,23 @@ +#include "listmutex.h" + + +ListMutex::ListMutex(QObject *parent) + : QObject{parent} +{ + +} + +void ListMutex::set_total(int total) +{ + this->total = total-1; +} + +int ListMutex::get_index() +{ + int res = -1; + mutex.lock(); + res = total; + total = total -1; + mutex.unlock(); + return res; +} diff --git a/utilities/MySQLToSTATA/listmutex.h b/utilities/MySQLToSTATA/listmutex.h new file mode 100644 index 00000000..f843eefe --- /dev/null +++ b/utilities/MySQLToSTATA/listmutex.h @@ -0,0 +1,19 @@ +#ifndef LISTMUTEX_H +#define LISTMUTEX_H + +#include +#include + +class ListMutex : public QObject +{ + Q_OBJECT +public: + explicit ListMutex(QObject *parent = nullptr); + void set_total(int total); + int get_index(); +private: + int total; + QMutex mutex; +}; + +#endif // LISTMUTEX_H diff --git a/utilities/MySQLToSTATA/main.cpp b/utilities/MySQLToSTATA/main.cpp index 792ef51a..c3e13fdf 100644 --- a/utilities/MySQLToSTATA/main.cpp +++ b/utilities/MySQLToSTATA/main.cpp @@ -3,9 +3,10 @@ #include #include "mainclass.h" #include +#include /* -MySQLToXLSX +MySQLToCSV Copyright (C) 2022 QLands Technology Consultants. Author: Carlos Quiros (cquiros_at_qlands.com / c.f.quiros_at_cgiar.org) @@ -27,24 +28,32 @@ License along with MySQLToXLSX. If not, see passArg("p","password","Password to connect to MySQL",true,"","string"); TCLAP::ValueArg schemaArg("s","schema","Schema in MySQL",true,"","string"); TCLAP::ValueArg createArg("x","createxml","Input create XML file",true,"","string"); - TCLAP::ValueArg outArg("o","output","Output directory for the STATA files",true,"","string"); - TCLAP::ValueArg tmpArg("T","tempdir","Temporary directory (./tmp by default)",false,"./tmp","string"); - TCLAP::ValueArg firstArg("f","firstsheetname","Name for the first sheet",false,"","string"); - TCLAP::ValueArg encryptArg("e","encrypt","32 char hex encryption key. Auto generate if empty",false,"","string"); - TCLAP::ValueArg resolveArg("r","resolve","Resolve lookup values: 1=Codes only (default), 2=Descriptions, 3=Codes and descriptions",false,"1","string"); - TCLAP::SwitchArg lookupSwitch("l","includelookups","Include lookup tables. False by default", cmd, false); - TCLAP::SwitchArg mselSwitch("m","includemultiselects","Include multi-select tables as sheets. False by default", cmd, false); + TCLAP::ValueArg outArg("o","output","Output directory for the CSV files",true,"","string"); + TCLAP::ValueArg tmpArg("T","tempdir","Temporary directory (./tmp by default)",false,"./tmp","string"); + TCLAP::ValueArg encryptArg("e","encrypt","32 char hex encryption key. Auto generate if empty",false,"","string"); TCLAP::SwitchArg protectSwitch("c","protect","Protect sensitive fields. False by default", cmd, false); + TCLAP::ValueArg numWorkers("w","workers","Number of workers. 1 by default",false,"1","string"); cmd.add(hostArg); cmd.add(portArg); + cmd.add(numWorkers); cmd.add(userArg); cmd.add(passArg); cmd.add(schemaArg); cmd.add(outArg); cmd.add(tmpArg); - cmd.add(createArg); - cmd.add(firstArg); - cmd.add(encryptArg); - cmd.add(resolveArg); + cmd.add(createArg); + cmd.add(encryptArg); //Parsing the command lines cmd.parse( argc, argv ); @@ -85,12 +90,6 @@ int main(int argc, char *argv[]) bool protectSensitive; protectSensitive = protectSwitch.getValue(); - bool includeLookUps; - includeLookUps = lookupSwitch.getValue(); - - bool includeMSels; - includeMSels = mselSwitch.getValue(); - QString host = QString::fromUtf8(hostArg.getValue().c_str()); QString port = QString::fromUtf8(portArg.getValue().c_str()); @@ -99,19 +98,22 @@ int main(int argc, char *argv[]) QString schema = QString::fromUtf8(schemaArg.getValue().c_str()); QString outputFile = QString::fromUtf8(outArg.getValue().c_str()); QString tmpDir = QString::fromUtf8(tmpArg.getValue().c_str()); - QString createXML = QString::fromUtf8(createArg.getValue().c_str()); - QString firstSheetName = QString::fromUtf8(firstArg.getValue().c_str()); - QString encryption_key = QString::fromUtf8(encryptArg.getValue().c_str()); - QString resolve_type = QString::fromUtf8(resolveArg.getValue().c_str()); + QString createXML = QString::fromUtf8(createArg.getValue().c_str()); + QString encryption_key = QString::fromUtf8(encryptArg.getValue().c_str()); + bool ok; + + + int workers = QString::fromUtf8(numWorkers.getValue().c_str()).toInt(&ok); + if (!ok) + workers = 1; if (encryption_key == "") { - QTime time = QTime::currentTime(); - qsrand((uint)time.msec()); encryption_key = getRandomHex(32); + log_out(encryption_key); } mainClass *task = new mainClass(&app); - task->setParameters(host,port,user,pass,schema,createXML,outputFile,protectSensitive,tmpDir, includeLookUps, includeMSels, firstSheetName, encryption_key, resolve_type); + task->setParameters(host,port,user,pass,schema,createXML,outputFile,protectSensitive,tmpDir, encryption_key, workers); QObject::connect(task, SIGNAL(finished()), &app, SLOT(quit())); QTimer::singleShot(0, task, SLOT(run())); app.exec(); diff --git a/utilities/MySQLToSTATA/mainclass.cpp b/utilities/MySQLToSTATA/mainclass.cpp index 50c71dd2..68636c2a 100644 --- a/utilities/MySQLToSTATA/mainclass.cpp +++ b/utilities/MySQLToSTATA/mainclass.cpp @@ -6,16 +6,15 @@ #include #include #include -#include +#include #include #include #include #include #include -#include -#include -#include -#include +#include "jsonworker.h" +#include "listmutex.h" +#include namespace pt = boost::property_tree; @@ -32,22 +31,30 @@ void mainClass::log(QString message) printf("%s", temp.toUtf8().data()); } -void mainClass::setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, QString outputDir, bool protectSensitive, QString tempDir, bool incLookups, bool incmsels, QString firstSheetName, QString encryption_key, QString resolve_type) +void mainClass::setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, QString outputDir, bool protectSensitive, QString tempDir, QString encryption_key, int num_workers) { this->host = host; this->port = port; this->user = user; this->pass = pass; + this->num_workers = num_workers; this->schema = schema; this->outputDirectory = outputDir; this->protectSensitive = protectSensitive; this->tempDir = tempDir; this->createXML = createXML; - this->incLookups = incLookups; - this->incmsels = incmsels; - this->firstSheetName = firstSheetName; this->encryption_key = encryption_key; - this->resolve_type = resolve_type.toInt(); + this->db = QSqlDatabase::addDatabase("QMYSQL","repository"); + db.setHostName(host); + db.setPort(port.toInt()); + db.setDatabaseName(schema); + db.setUserName(user); + db.setPassword(pass); + if (!db.open()) + { + log("Error while conneting to MySQL"); + exit(1); + } } QString mainClass::getSheetDescription(QString name) @@ -105,7 +112,7 @@ void mainClass::getMultiSelectInfo(QDomNode table, QString table_name, QString & } } child = child.nextSibling(); - } + } } void mainClass::loadTable(QDomNode table) @@ -117,6 +124,10 @@ void mainClass::loadTable(QDomNode table) aTable.islookup = false; aTable.name = eTable.attribute("name",""); aTable.desc = eTable.attribute("name",""); + if (aTable.name.indexOf("_msel_") >= 0) + aTable.ismultiselect = true; + else + aTable.ismultiselect = false; QDomNode field = table.firstChild(); while (!field.isNull()) @@ -186,6 +197,152 @@ void mainClass::loadTable(QDomNode table) } +int mainClass::processTasks(QDir currDir) +{ + ListMutex *mutex = new ListMutex(this); + + QList< JSONWorker*> workers; + for (int w=1; w <= num_workers; w++) + { + JSONWorker *a_worker = new JSONWorker(this); + a_worker->setName("Worker" + QString::number(w)); + a_worker->setParameters(host, port, user, pass, schema, currDir, outputDirectory); + workers.append(a_worker); + } + + //qDebug() << "Working on separation: " + QString::number(separate_task_list.count()); + //Work on separation + mutex->set_total(separate_task_list.count()); + for (int w=0; w < workers.count(); w++) + { + workers[w]->setTasks(separate_task_list); + workers[w]->setMutex(mutex); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->start(); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->wait(); + } + for (int w=0; w < workers.count(); w++) + { + if (workers[w]->status != 0) + return -1; + } + + //qDebug() << "Working on applying updates: " + QString::number(update_task_list.count()); + //Work on the updates + mutex->set_total(update_task_list.count()); + for (int w=0; w < workers.count(); w++) + { + workers[w]->setTasks(update_task_list); + workers[w]->setMutex(mutex); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->start(); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->wait(); + } + for (int w=0; w < workers.count(); w++) + { + if (workers[w]->status != 0) + return -1; + } + + //qDebug() << "Working on generating JSON files"; + //Work on the JSON extract + mutex->set_total(json_task_list.count()); + for (int w=0; w < workers.count(); w++) + { + workers[w]->setTasks(json_task_list); + workers[w]->setMutex(mutex); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->start(); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->wait(); + } + for (int w=0; w < workers.count(); w++) + { + if (workers[w]->status != 0) + return -1; + } + + //qDebug() << "Working on merging the final files"; + //Work in the merging + mutex->set_total(merge_task_list.count()); + for (int w=0; w < workers.count(); w++) + { + workers[w]->setTasks(merge_task_list); + workers[w]->setMutex(mutex); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->start(); + } + for (int w=0; w < workers.count(); w++) + { + workers[w]->wait(); + } + for (int w=0; w < workers.count(); w++) + { + if (workers[w]->status != 0) + return -1; + } + return 0; +} + +QStringList mainClass::get_parts(int total, int parts) +{ + QStringList res; + int div = total / parts; + int start = 1; + int end = div; + if (parts >= total) + { + res.append("1|" + QString::number(total)); + return res; + } + if (parts == 1) + { + res.append("1|" + QString::number(total)); + return res; + } + if (total >= 500) + { + for (int pos = 1; pos < parts; pos++) + { + res.append(QString::number(start) + "|" + QString::number(end)); + start = end + 1 ; + end = start + div; + } + res.append(QString::number(start) + "|" + QString::number(total)); + } + else + { + res.append("1|" + QString::number(total)); + } + return res; +} + +bool mainClass::is_lookup_int(QString table) +{ + for (int pos=0; pos < lookupTables.count(); pos++) + { + if (lookupTables[pos].name == table) + return lookupTables[pos].is_numeric; + } + return false; +} + int mainClass::generateXLSX() { @@ -206,9 +363,43 @@ int mainClass::generateXLSX() } fileA.close(); - //Load the lookup tables if asked + QDomElement rootA = docA.documentElement(); + //Load the lookup tables + QDomNode lkpTable = rootA.firstChild().firstChild(); + //Getting the fields to export from Lookup tables + while (!lkpTable.isNull()) + { + QDomElement eTable; + eTable = lkpTable.toElement(); + + TtableDef aTable; + aTable.islookup = true; + aTable.name = eTable.attribute("name",""); + aTable.desc = eTable.attribute("name",""); + + QDomNode field = lkpTable.firstChild(); + while (!field.isNull()) + { + QDomElement eField; + eField = field.toElement(); + + TfieldDef aField; + if (eField.attribute("name","").right(4) == "_cod") + aTable.lkp_code_field = eField.attribute("name",""); + if (eField.attribute("name","").right(4) == "_des") + aTable.lkp_desc_field = eField.attribute("name",""); + if (eField.attribute("type","").right(4) == "int") + aTable.is_numeric = true; + + field = field.nextSibling(); + } + lookupTables.append(aTable); + + lkpTable = lkpTable.nextSibling(); + } + //Getting the fields to export from tables QDomNode table = rootA.firstChild().nextSibling().firstChild(); @@ -216,139 +407,62 @@ int mainClass::generateXLSX() loadTable(table); for (int nt =mainTables.count()-1; nt >= 0;nt--) { - if (mainTables[nt].name.indexOf("_msel_") < 0) - tables.append(mainTables[nt]); - else - { - if (incmsels) - tables.append(mainTables[nt]); - } + tables.append(mainTables[nt]); } - if (firstSheetName != "") - tables[0].desc = firstSheetName; - if (rootA.tagName() == "XMLSchemaStructure") { - if (this->incLookups) - { - QDomNode lkpTable = rootA.firstChild().firstChild(); - //Getting the fields to export from Lookup tables - while (!lkpTable.isNull()) - { - QDomElement eTable; - eTable = lkpTable.toElement(); - TtableDef aTable; - aTable.islookup = true; - aTable.name = eTable.attribute("name",""); - aTable.desc = eTable.attribute("name",""); + QDir currDir(tempDir); - QDomNode field = lkpTable.firstChild(); - while (!field.isNull()) - { - QDomElement eField; - eField = field.toElement(); - - TfieldDef aField; - aField.name = eField.attribute("name",""); - aField.desc = eField.attribute("desc",""); - aField.type = eField.attribute("type",""); - if (eField.attribute("sensitive","false") == "true") - { - aField.sensitive = true; - aField.protection = eField.attribute("protection","exclude"); - } - else - aField.sensitive = false; - aField.size = eField.attribute("size","").toInt(); - aField.decSize = eField.attribute("decsize","").toInt(); - aTable.fields.append(aField); + QElapsedTimer procTime; + procTime.start(); + QString sql; + QStringList fields; + QStringList fields_for_select; + QStringList descfields; + QStringList fields_for_create; + QStringList drops; - field = field.nextSibling(); - } - tables.append(aTable); + QDir finalDir(outputDirectory); - lkpTable = lkpTable.nextSibling(); + if (!finalDir.exists(outputDirectory)) + { + if (!finalDir.mkdir(outputDirectory)) + { + log("Cannot create temporary output directory"); + returnCode = 1; + emit finished(); } } - QDir currDir(tempDir); - QStringList arguments; - QProcess *mySQLDumpProcess = new QProcess(); - QTime procTime; - procTime.start(); - QString sql; - QStringList fields; - QString uri = user + ":" + pass + "@" + host + "/" + schema; - QStringList sheets; - QStringList csvs; - - QVector linked_tables; - QVector multiSelectTables; - - for (int pos = 0; pos <= tables.count()-1; pos++) - { - qDebug() << "Creating temp table " + tables[pos].name; - linked_tables.clear(); - sheets.append(getSheetDescription(tables[pos].desc)); + for (int pos = 0; pos < tables.count(); pos++) + { + TtaskItem a_merge_task; + a_merge_task.task_type = 4; + a_merge_task.table = tables[pos].name; + a_merge_task.final_file = finalDir.absolutePath() + currDir.separator() + tables[pos].name + ".json"; + + + TtaskItem a_json_task; + a_json_task.task_type = 2; + a_json_task.table = tables[pos].name; + + a_json_task.sql_file = currDir.absolutePath() + currDir.separator() + tables[pos].name + "_query.sql"; + a_json_task.json_file = currDir.absolutePath() + currDir.separator() + tables[pos].name + ".json"; + + a_merge_task.json_files.append(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".json"); + + fields.clear(); + descfields.clear(); for (int fld = 0; fld < tables[pos].fields.count(); fld++) { if (this->protectSensitive) { if (tables[pos].fields[fld].sensitive == false) { - if (tables[pos].fields[fld].isMultiSelect == false) - { - if (tables[pos].fields[fld].isKey == false) - { - if (tables[pos].fields[fld].isLookUp == false) - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - else - { - if (this->resolve_type != 1) - { - TlinkedTable a_linked_table; - a_linked_table.field = tables[pos].fields[fld].name; - a_linked_table.related_table = tables[pos].fields[fld].lookupRelTable; - a_linked_table.related_field = tables[pos].fields[fld].lookupRelField; - linked_tables.append(a_linked_table); - } - if (this->resolve_type == 3) - { - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - else - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - } - else - { - if (protectedKeys.indexOf(tables[pos].fields[fld].name) < 0) - { - - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - else - fields.append("HEX(AES_ENCRYPT(" + tables[pos].name + "." + tables[pos].fields[fld].name + ",UNHEX('" + this->encryption_key + "'))) as " + tables[pos].fields[fld].name); - } - } - else - { - fields.append(tables[pos].fields[fld].name); - TmultiSelectTable a_multiSelectTable; - a_multiSelectTable.field = tables[pos].fields[fld].name; - a_multiSelectTable.multiSelectTable = tables[pos].fields[fld].multiSelectTable; - a_multiSelectTable.multiSelectField = tables[pos].fields[fld].multiSelectField; - a_multiSelectTable.multiSelectRelTable = tables[pos].fields[fld].multiSelectRelTable; - a_multiSelectTable.multiSelectRelField = tables[pos].fields[fld].multiSelectRelField; - a_multiSelectTable.multiSelectKeys.append(tables[pos].fields[fld].multiSelectKeys); - multiSelectTables.append(a_multiSelectTable); - if (this->resolve_type == 3) - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } + fields.append(tables[pos].fields[fld].name); } else { @@ -357,392 +471,166 @@ int mainClass::generateXLSX() } } else - { - if (tables[pos].fields[fld].isMultiSelect == false) - { - if (tables[pos].fields[fld].isLookUp == false) - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - else - { - if (this->resolve_type != 1) - { - TlinkedTable a_linked_table; - a_linked_table.field = tables[pos].fields[fld].name; - a_linked_table.related_table = tables[pos].fields[fld].lookupRelTable; - a_linked_table.related_field = tables[pos].fields[fld].lookupRelField; - linked_tables.append(a_linked_table); - } - if (this->resolve_type == 3) - { - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - else - fields.append(tables[pos].name + "." + tables[pos].fields[fld].name); - } - } - else - { - fields.append(tables[pos].fields[fld].name); - TmultiSelectTable a_multiSelectTable; - a_multiSelectTable.field = tables[pos].fields[fld].name; - a_multiSelectTable.multiSelectTable = tables[pos].fields[fld].multiSelectTable; - a_multiSelectTable.multiSelectField = tables[pos].fields[fld].multiSelectField; - a_multiSelectTable.multiSelectRelTable = tables[pos].fields[fld].multiSelectRelTable; - a_multiSelectTable.multiSelectRelField = tables[pos].fields[fld].multiSelectRelField; - a_multiSelectTable.multiSelectKeys.append(tables[pos].fields[fld].multiSelectKeys); - multiSelectTables.append(a_multiSelectTable); - if (this->resolve_type == 3) - fields.append("'' as '" + tables[pos].fields[fld].name + "-desc'"); - } - } - } - QString temp_table; - sql = "SET SQL_MODE = '';\nSTART TRANSACTION;\n"; - - QUuid recordUUID=QUuid::createUuid(); - temp_table = "TMP_" + recordUUID.toString().replace("{","").replace("}","").replace("-","_"); - sql = sql + "CREATE TABLE " + temp_table + " ENGINE=MyISAM DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci AS SELECT " + fields.join(",") + " FROM " + tables[pos].name + ";\nCOMMIT;\n"; - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - - QFile tempfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - if (!tempfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - delete mySQLDumpProcess; - return 1; - } - QTextStream temout(&tempfile); - temout << sql; - tempfile.close(); - - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) - { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; - return 1; - } - arguments.clear(); - - QStringList sqls; - qDebug() << "Performing Alters on temp table"; - sqls.append("START TRANSACTION;\n"); - - for (int fld = 0; fld < tables[pos].fields.count(); fld++) - { - if (tables[pos].fields[fld].isKey) - sqls.append("ALTER TABLE " + temp_table + " MODIFY COLUMN " + tables[pos].fields[fld].name + " VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;\n"); - } - - if (multiSelectTables.count() > 0) - { - QStringList modifies; - for (int i_table=0; i_table < multiSelectTables.count(); i_table++) - { - modifies.append("MODIFY COLUMN " + multiSelectTables[i_table].field + " TEXT"); - if (this->resolve_type == 3) - modifies.append("MODIFY COLUMN `" + multiSelectTables[i_table].field + "-desc` TEXT"); - } - sql = "ALTER TABLE " + temp_table + " " + modifies.join(",") + ";\n"; - sqls.append(sql); - } - - if (linked_tables.count() > 0) - { - QStringList modifies; - for (int i_table=0; i_table < linked_tables.count(); i_table++) - { - sql = "MODIFY COLUMN "; - if (this->resolve_type == 2) - sql = sql + linked_tables[i_table].field; - else - sql = sql + "`" + linked_tables[i_table].field + "-desc`"; - sql = sql + " TEXT"; - modifies.append(sql); - } - sql = "ALTER TABLE " + temp_table + " " + modifies.join(",") + ";\n"; - sqls.append(sql); - - - for (int i_table=0; i_table < linked_tables.count(); i_table++) - { - sql = "UPDATE " + temp_table + "," + linked_tables[i_table].related_table + " AS T" + QString::number(i_table) + " SET "; - QString relfield = linked_tables[i_table].related_field; - if (this->resolve_type == 2) - sql = sql + temp_table + "." + linked_tables[i_table].field + " = T" + QString::number(i_table) + "." + relfield.replace("_cod","_des"); - else - sql = sql + temp_table + ".`" + linked_tables[i_table].field + "-desc` = T" + QString::number(i_table) + "." + relfield.replace("_cod","_des"); - sql = sql + " WHERE " + temp_table + "." + linked_tables[i_table].field + " = T" + QString::number(i_table) + "." + linked_tables[i_table].related_field + ";\n"; - sqls.append(sql); - } - } - if (multiSelectTables.count() > 0) - { - for (int i_table=0; i_table < multiSelectTables.count(); i_table++) - { - if (this->resolve_type == 1 || this->resolve_type == 3) - { - sql = "UPDATE " + temp_table + " AS TA SET TA." + multiSelectTables[i_table].field + " = (SELECT GROUP_CONCAT(TB." + multiSelectTables[i_table].multiSelectField + " SEPARATOR ',') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB"; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " WHERE " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - if (this->resolve_type == 2) - { - QString desc_field = multiSelectTables[i_table].multiSelectRelField; - desc_field = desc_field.replace("_cod","_des"); - sql = "UPDATE " + temp_table + " AS TA SET TA." + multiSelectTables[i_table].field + " = (SELECT GROUP_CONCAT(TC." + desc_field + " SEPARATOR ',') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB," + multiSelectTables[i_table].multiSelectRelTable + " as TC"; - sql = sql + " WHERE TB." + multiSelectTables[i_table].multiSelectField + " = TC." + multiSelectTables[i_table].multiSelectRelField; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " AND " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - if (this->resolve_type == 3) - { - QString desc_field = multiSelectTables[i_table].multiSelectRelField; - desc_field = desc_field.replace("_cod","_des"); - sql = "UPDATE " + temp_table + " AS TA SET TA.`" + multiSelectTables[i_table].field + "-desc` = (SELECT GROUP_CONCAT(TC." + desc_field + " SEPARATOR ',') FROM " + multiSelectTables[i_table].multiSelectTable + " as TB," + multiSelectTables[i_table].multiSelectRelTable + " as TC"; - sql = sql + " WHERE TB." + multiSelectTables[i_table].multiSelectField + " = TC." + multiSelectTables[i_table].multiSelectRelField; - QStringList wheres; - QStringList groups; - for (int a_key = 0; a_key < multiSelectTables[i_table].multiSelectKeys.count(); a_key++) - { - wheres.append("TA." + multiSelectTables[i_table].multiSelectKeys[a_key] + " = TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - groups.append("TB." + multiSelectTables[i_table].multiSelectKeys[a_key]); - } - sql = sql + " AND " + wheres.join(" AND "); - sql = sql + " GROUP BY " + groups.join(",") + ");\n"; - sqls.append(sql); - } - } - } -// for (int p=0; p < sqls.count(); p++) -// qDebug() << sqls[p]; - sqls.append("COMMIT;\n"); - if (sqls.count() > 0) - { - QFile modfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - if (!modfile.open(QIODevice::WriteOnly | QIODevice::Text)) - { - delete mySQLDumpProcess; - return 1; - } - QTextStream modOut(&modfile); - for (int isql=0; isql < sqls.count(); isql++) - modOut << sqls[isql]; - modfile.close(); - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) - { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; - return 1; - } - } - - // Export the structure of the temporary table JSON format - arguments.clear(); - arguments << "--sql"; - arguments << "--result-format=json/array"; - arguments << "--uri=" + uri; - arguments << "--execute=desc " + temp_table; - mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); - mySQLDumpProcess->setStandardOutputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".json"); - - mySQLDumpProcess->start("mysqlsh", arguments); - mySQLDumpProcess->waitForFinished(-1); - if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) - { - if (mySQLDumpProcess->error() == QProcess::FailedToStart) - { - log("Error: Command mysqlsh not found"); - } - else - { - log("Running mysqlsh returned error"); - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - log("Running paremeters:" + arguments.join(" ")); - } - delete mySQLDumpProcess; - return 1; + fields.append(tables[pos].fields[fld].name); } - QFile loadFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".json"); - - if (!loadFile.open(QIODevice::ReadOnly)) { - log("Couldn't open json structure file."); - delete mySQLDumpProcess; - return 1; - } - QByteArray jsonData = loadFile.readAll(); - QJsonDocument loadDoc(QJsonDocument::fromJson(jsonData)); - QJsonArray fields = loadDoc.array(); QStringList fields_to_select; - for (auto a_field : fields) + for (int fld =0; fld < fields.count(); fld++) { - QJsonObject element = a_field.toObject(); - fields_to_select << "ifnull(`" + element["Field"].toString() + "`,'') as `" + element["Field"].toString() + "`"; + if (fields[fld].indexOf(" as ") < 0) + fields_to_select.append("ifnull(`" + fields[fld] + "`,'') as `" + fields[fld] + "`"); + else + fields_to_select.append(fields[fld]); } - - linked_tables.clear(); - multiSelectTables.clear(); - qDebug() << "Parsing table " + tables[pos].name + " to TAB format"; - sql = "SELECT " + fields_to_select.join(",") + " FROM " + temp_table + ";"; - - arguments.clear(); - arguments << "--sql"; - arguments << "--result-format=tabbed"; - arguments << "--uri=" + uri; - QFile sqlfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); + sql = "SELECT " + fields_to_select.join(",") + " FROM " + tables[pos].name + ";\n"; + QFile sqlfile(currDir.absolutePath() + currDir.separator() + tables[pos].name + "_query.sql"); if (!sqlfile.open(QIODevice::WriteOnly | QIODevice::Text)) { - delete mySQLDumpProcess; return 1; } QTextStream out(&sqlfile); out << sql; sqlfile.close(); - mySQLDumpProcess->setStandardInputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".sql"); - mySQLDumpProcess->setStandardOutputFile(currDir.absolutePath() + currDir.separator() + tables[pos].name + ".raw"); - mySQLDumpProcess->start("mysqlsh", arguments); - mySQLDumpProcess->waitForFinished(-1); - if ((mySQLDumpProcess->exitCode() > 0) || (mySQLDumpProcess->error() == QProcess::FailedToStart)) - { - if (mySQLDumpProcess->error() == QProcess::FailedToStart) - { - log("Error: Command mysqlsh not found"); - } - else - { - log("Running mysqlsh returned error"); - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - log("Running paremeters:" + arguments.join(" ")); - } - delete mySQLDumpProcess; - return 1; - } + json_task_list.append(a_json_task); + merge_task_list.append(a_merge_task); - QString do_file = currDir.absolutePath() + currDir.separator() + tables[pos].name + ".do"; + } + int result = processTasks(currDir); + + // Generating DO files for each table + for (int pos = 0; pos < tables.count(); pos++) + { + QString do_file = finalDir.absolutePath() + currDir.separator() + tables[pos].name + ".do"; QFile dofile(do_file); if (!dofile.open(QIODevice::WriteOnly | QIODevice::Text)) { - log("Cannot open DCF file for writing"); - delete mySQLDumpProcess; + log("Cannot open DO file for writing"); return 1; } QTextStream do_out(&dofile); do_out << "/* This script will generate a STATA " + tables[pos].name + ".dta file from a " + tables[pos].name + ".raw tab delimited file*/\n"; do_out << "/* You only need to run this once.*/\n"; do_out << "/* Change c:\\my_working_dir to the directory holding this file.*/\n"; + do_out << "clear\n"; do_out << "cd \"c:\\my_working_dir\"\n"; do_out << "import delimited \"" + tables[pos].name + ".raw\", delimiter(tab) varnames(1) encoding(UTF-8) stringcols(_all)\n"; - - - for (auto a_field : fields) + for (int fld=0; fld < tables[pos].fields.count(); fld++) { QString line_data; - QJsonObject element = a_field.toObject(); - if (element["Type"].toString() == "double") - line_data = "destring " + element["Field"].toString() + ", replace float force\n"; - if (element["Type"].toString().indexOf("decimal") >= 0) - line_data = "destring " + element["Field"].toString() + ", replace float force\n"; - if (element["Type"].toString().indexOf("int") >= 0) - line_data = "destring " + element["Field"].toString() + ", replace force\n"; - if (element["Type"].toString().indexOf("varchar") >= 0) + + if (tables[pos].fields[fld].type == "double") + line_data = "destring " + tables[pos].fields[fld].name + ", replace float force\n"; + if (tables[pos].fields[fld].type.indexOf("decimal") >= 0) + line_data = "destring " + tables[pos].fields[fld].name + ", replace float force\n"; + if (tables[pos].fields[fld].type.indexOf("int") >= 0) + line_data = "destring " + tables[pos].fields[fld].name + ", replace force\n"; + if (tables[pos].fields[fld].type.indexOf("varchar") >= 0) { - QRegExp exp("\\(([^()]*)\\)"); - exp.indexIn(element["Type"].toString()); - QString zize = exp.cap(1); - line_data = "recast str"+ zize + " " + element["Field"].toString() + ", force\n"; + QString zize = QString::number(tables[pos].fields[fld].size); + line_data = "recast str"+ zize + " " + tables[pos].fields[fld].name + ", force\n"; } if (line_data != "") do_out << line_data; - } do_out << "save " + tables[pos].name + "\n"; - do_out << "clear"; + dofile.close(); - loadFile.close(); - - arguments.clear(); - arguments.append("--host=" + this->host); - arguments.append("--port=" + this->port); - arguments.append("--password=" + this->pass); - arguments.append("--user=" + this->user); - arguments.append("--database=" + this->schema); - arguments.append("--execute=DROP TABLE " + temp_table ); - - mySQLDumpProcess->setStandardInputFile(QProcess::nullDevice()); - mySQLDumpProcess->setStandardOutputFile(QProcess::nullDevice()); - mySQLDumpProcess->start("mysql", arguments); - mySQLDumpProcess->waitForFinished(-1); - if (mySQLDumpProcess->exitCode() > 0) + } + + // Generate labels do files + for (int pos = 0; pos < tables.count(); pos++) + { + QString do_file = finalDir.absolutePath() + currDir.separator() + tables[pos].name + "_labels.do"; + + QFile dofile(do_file); + if (!dofile.open(QIODevice::WriteOnly | QIODevice::Text)) { - QString serror = mySQLDumpProcess->readAllStandardError(); - log(serror); - delete mySQLDumpProcess; + log("Cannot open DO labels file for writing"); return 1; } + QTextStream do_out(&dofile); - QString tab_file = currDir.absolutePath() + currDir.separator() + tables[pos].name + ".raw"; + for (int fld=0; fld < tables[pos].fields.count(); fld++) + { + do_out << "label variable " + tables[pos].fields[fld].name + " \"" + tables[pos].fields[fld].desc.replace("\"","") + "\"\n"; + } + do_out << "\n"; - csvs.append(tab_file); - csvs.append(do_file); + for (int fld=0; fld < tables[pos].fields.count(); fld++) + { + if (tables[pos].fields[fld].isLookUp == true) + { + if (is_lookup_int(tables[pos].fields[fld].lookupRelTable)) + do_out << "label values " + tables[pos].fields[fld].name + " " + tables[pos].fields[fld].lookupRelTable + "\n"; + else + do_out << "* label values " + tables[pos].fields[fld].name + " " + tables[pos].fields[fld].lookupRelTable + "\n"; + } + } + + dofile.close(); } + QString do_file_lbl = finalDir.absolutePath() + currDir.separator() + "lookups.do"; + QFile dofile_lbl(do_file_lbl); + if (!dofile_lbl.open(QIODevice::WriteOnly | QIODevice::Text)) + { + log("Cannot open DO lookup file for writing"); + return 1; + } + QTextStream do_out_lbl(&dofile_lbl); - for (int pos = 0; pos < csvs.count(); pos++) + QSqlQuery lkps(db); + do_out_lbl << "#delimit ;\n"; + + for (int pos = 0; pos < lookupTables.count(); pos++) { - QString newpath; - newpath = csvs[pos]; - newpath.replace(currDir.absolutePath() + currDir.separator(), outputDirectory + currDir.separator()); - if (QFile::copy(csvs[pos], newpath)) - QFile::remove(csvs[pos]); + if (lookupTables[pos].is_numeric) + { + do_out_lbl << "label define " + lookupTables[pos].name + "\n"; + QString sql = "SELECT " + lookupTables[pos].lkp_code_field + "," + lookupTables[pos].lkp_desc_field + " FROM " + lookupTables[pos].name; + //qDebug() << sql; + lkps.exec(sql); + while(lkps.next()) + { + do_out_lbl << lkps.value(0).toString() + " \"" + lkps.value(1).toString() + "\"\n"; + } + do_out_lbl << ";\n"; + } + else + { + do_out_lbl << "* label define " + lookupTables[pos].name + "\n"; + QString sql = "SELECT " + lookupTables[pos].lkp_code_field + "," + lookupTables[pos].lkp_desc_field + " FROM " + lookupTables[pos].name; + //qDebug() << sql; + lkps.exec(sql); + while(lkps.next()) + { + do_out_lbl << "* " + lkps.value(0).toString() + " \"" + lkps.value(1).toString() + "\"\n"; + } + do_out_lbl << "* ;\n"; + } } + do_out_lbl << "#delimit cr\n"; + //exit(1); - delete mySQLDumpProcess; + + QSqlQuery drop(db); + for (int d=0; d < drops.count(); d++) + { + drop.exec(drops[d]); + } + db.close(); + QDir tdir(tempDir); + tdir.removeRecursively(); + + if (result != 0) + { + returnCode = 1; + return returnCode; + } int Hours; int Minutes; diff --git a/utilities/MySQLToSTATA/mainclass.h b/utilities/MySQLToSTATA/mainclass.h index d32cbd15..c851c4f5 100644 --- a/utilities/MySQLToSTATA/mainclass.h +++ b/utilities/MySQLToSTATA/mainclass.h @@ -3,6 +3,8 @@ #include #include +#include +#include struct fieldDef { @@ -34,6 +36,10 @@ struct tableDef QString desc; QList fields; //List of fields bool islookup; //Whether the table is a lookup table + bool ismultiselect; //Whether the table is a lookup table + QString lkp_code_field; + QString lkp_desc_field; + bool is_numeric = false; }; typedef tableDef TtableDef; @@ -56,12 +62,23 @@ struct multiSelectTable }; typedef multiSelectTable TmultiSelectTable; +struct taskItem +{ + QString table; + int task_type; + QString sql_file; + QString json_file; + QStringList json_files; + QString final_file; +}; +typedef taskItem TtaskItem; + class mainClass : public QObject { Q_OBJECT public: explicit mainClass(QObject *parent = nullptr); - void setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, QString outputDir, bool protectSensitive, QString tempDir, bool incLookups, bool incmsels, QString firstSheetName, QString encryption_key, QString resolve_type); + void setParameters(QString host, QString port, QString user, QString pass, QString schema, QString createXML, QString outputDir, bool protectSensitive, QString tempDir, QString encryption_key, int num_workers); int returnCode; signals: void finished(); @@ -69,10 +86,13 @@ public slots: void run(); private: void log(QString message); - int generateXLSX(); + int generateXLSX(); + int processTasks(QDir currDir); QString getSheetDescription(QString name); void loadTable(QDomNode node); + bool is_lookup_int(QString table); void getMultiSelectInfo(QDomNode table, QString table_name, QString &multiSelect_field, QStringList &keys, QString &rel_table, QString &rel_field); + QStringList get_parts(int total, int parts); QString host; QString port; QString user; @@ -82,16 +102,19 @@ public slots: QString tempDir; QString createXML; QString encryption_key; - int resolve_type = 1; bool protectSensitive; QList tables; QList mainTables; + QList lookupTables; QStringList tableNames; + int num_workers; int letterIndex; - bool incLookups; - bool incmsels; - QString firstSheetName; - QStringList protectedKeys; + QStringList protectedKeys; + QList< TtaskItem> separate_task_list; + QList< TtaskItem> update_task_list; + QList< TtaskItem> json_task_list; + QList< TtaskItem> merge_task_list; + QSqlDatabase db; }; #endif // MAINCLASS_H diff --git a/utilities/MySQLToXLSX/main.cpp b/utilities/MySQLToXLSX/main.cpp index 8ab5087c..7da132b0 100644 --- a/utilities/MySQLToXLSX/main.cpp +++ b/utilities/MySQLToXLSX/main.cpp @@ -112,7 +112,7 @@ int main(int argc, char *argv[]) QString resolve_type = QString::fromUtf8(resolveArg.getValue().c_str()); bool ok; - if (resolve_type <= 0 && resolve_type > 3) + if (resolve_type.toInt() <= 0 && resolve_type.toInt() > 3) { log_out("Resolving code must be 1, 2, or 3"); exit(1); diff --git a/utilities/MySQLToXLSX/mainclass.cpp b/utilities/MySQLToXLSX/mainclass.cpp index 171e4e83..1108c5c6 100644 --- a/utilities/MySQLToXLSX/mainclass.cpp +++ b/utilities/MySQLToXLSX/mainclass.cpp @@ -437,7 +437,10 @@ int mainClass::generateXLSX() { for (int pos = 0; pos <= tables.count()-1; pos++) { - sheets.append(getSheetDescription(tables[pos].desc)); + TsheetItem a_sheet; + a_sheet.tableName = tables[pos].desc; + a_sheet.sheetName = getSheetDescription(tables[pos].desc); + sheets.append(a_sheet); linked_tables.clear(); fields.clear(); fields_for_select.clear(); @@ -927,7 +930,10 @@ int mainClass::generateXLSX() QDir finalDir(tempDir); for (int pos = 0; pos < tables.count(); pos++) { - sheets.append(getSheetDescription(tables[pos].desc)); + TsheetItem a_sheet; + a_sheet.tableName = tables[pos].desc; + a_sheet.sheetName = getSheetDescription(tables[pos].desc); + sheets.append(a_sheet); TtaskItem a_merge_task; a_merge_task.task_type = 4; @@ -996,6 +1002,50 @@ int mainClass::generateXLSX() if (result == 0) { + //Create an XML file to report the name of each table and its sheet + //We can use this later on to open the excel file and apply formats + //to colunmns + QString sheetFile = outputFile + ".xml"; + QDomDocument XMLSheetStructure; + XMLSheetStructure = QDomDocument("XMLSheetFile"); + QDomElement XMLRoot; + XMLRoot = XMLSheetStructure.createElement("XMLSheetFile"); + XMLSheetStructure.appendChild(XMLRoot); + + QDomElement XMLSheets; + XMLSheets = XMLSheetStructure.createElement("sheets"); + XMLRoot.appendChild(XMLSheets); + + + for (int pos = 0; pos < sheets.count(); pos++) + { + QDomElement XMLSheet; + XMLSheet = XMLSheetStructure.createElement("sheet"); + XMLSheet.setAttribute("name", sheets[pos].sheetName); + XMLSheet.setAttribute("table", sheets[pos].tableName); + XMLSheets.appendChild(XMLSheet); + } + + if (QFile::exists(sheetFile)) + QFile::remove(sheetFile); + QFile XMLSheetFile(sheetFile); + if (XMLSheetFile.open(QIODevice::WriteOnly | QIODevice::Text)) + { + QTextStream outXMLSheet(&XMLSheetFile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) + outXMLSheet.setCodec("UTF-8"); +#endif + XMLSheetStructure.save(outXMLSheet,1,QDomNode::EncodingFromTextStream); + XMLSheetFile.close(); + } + else + { + log("Error: Cannot create sheet file file"); + returnCode = 1; + return returnCode; + } + + // Convert the CSVS into one Excel file QProcess *mySQLDumpProcess = new QProcess(); QStringList arguments; @@ -1004,7 +1054,7 @@ int mainClass::generateXLSX() if (QFile::exists(finalCSVS[pos])) { arguments.append("-s"); - arguments.append(sheets[pos]); + arguments.append(sheets[pos].sheetName); } } arguments.append("--output"); diff --git a/utilities/MySQLToXLSX/mainclass.h b/utilities/MySQLToXLSX/mainclass.h index dbf7fc08..bb57bffc 100644 --- a/utilities/MySQLToXLSX/mainclass.h +++ b/utilities/MySQLToXLSX/mainclass.h @@ -70,6 +70,13 @@ struct taskItem }; typedef taskItem TtaskItem; +struct sheetItem +{ + QString tableName; + QString sheetName; +}; +typedef sheetItem TsheetItem; + class mainClass : public QObject { Q_OBJECT @@ -104,7 +111,7 @@ public slots: QList mainTables; QList lookupTables; QStringList tableNames; - QStringList sheets; + QList sheets; QStringList finalCSVS; int num_workers; int letterIndex; diff --git a/utilities/createAuditTriggers/main.cpp b/utilities/createAuditTriggers/main.cpp index 89a74ee1..6004cf68 100644 --- a/utilities/createAuditTriggers/main.cpp +++ b/utilities/createAuditTriggers/main.cpp @@ -50,9 +50,11 @@ int main(int argc, char *argv[]) QString tables = QString::fromUtf8(tablesArg.getValue().c_str()); mainClass *task = new mainClass(&app); - +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) task->setParameters(host,port,user,password,schema,output,tables.split(",",QString::SkipEmptyParts)); - +#else + task->setParameters(host,port,user,password,schema,output,tables.split(",",Qt::SkipEmptyParts)); +#endif QObject::connect(task, SIGNAL(finished()), &app, SLOT(quit())); QTimer::singleShot(0, task, SLOT(run())); diff --git a/utilities/createDummyJSON/main.cpp b/utilities/createDummyJSON/main.cpp index fff09fd4..97fbe49f 100644 --- a/utilities/createDummyJSON/main.cpp +++ b/utilities/createDummyJSON/main.cpp @@ -240,10 +240,18 @@ int main(int argc, char *argv[]) if (sarrays != "") { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) QStringList items = sarrays.split(",",QString::SkipEmptyParts); +#else + QStringList items = sarrays.split(",",Qt::SkipEmptyParts); +#endif for (int pos = 0; pos < items.count(); pos++) { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) QStringList parts = items[pos].split(":",QString::SkipEmptyParts); +#else + QStringList parts = items[pos].split(":",Qt::SkipEmptyParts); +#endif if (parts.length() == 2) { TarraySizeItem item; diff --git a/utilities/createFromXML/main.cpp b/utilities/createFromXML/main.cpp index 9bc42053..7979d5e9 100644 --- a/utilities/createFromXML/main.cpp +++ b/utilities/createFromXML/main.cpp @@ -35,6 +35,7 @@ struct relatedTable { QString name; QString relName; + QString on_delete = "CASCADE"; QList< TrelatedField> fields; }; typedef relatedTable TrelatedTable; @@ -163,6 +164,7 @@ void createTable(QString tableName,QList fields,QTextStream &outstrm, TrelatedTable relTable; relTable.name = efield.attribute("rtable",""); relTable.relName = efield.attribute("rname",""); + relTable.on_delete = efield.attribute("on_delete","CASCADE"); TrelatedField relfield; relfield.name = efield.attribute("name",""); @@ -185,7 +187,7 @@ void createTable(QString tableName,QList fields,QTextStream &outstrm, rels << "FOREIGN KEY (" + getFields(relTables[pos]) + ")" << "\n"; rels << "REFERENCES " + relTables[pos].name + " (" + getRelatedFields(relTables[pos]) + ")" << "\n"; - rels << "ON DELETE CASCADE " << "\n"; + rels << "ON DELETE " + relTables[pos].on_delete << "\n"; rels << "ON UPDATE NO ACTION," << "\n"; } diff --git a/utilities/createTemporaryTable/main.cpp b/utilities/createTemporaryTable/main.cpp index 44b28f5d..a19c70ad 100644 --- a/utilities/createTemporaryTable/main.cpp +++ b/utilities/createTemporaryTable/main.cpp @@ -3,6 +3,9 @@ #include #include "mainclass.h" #include +#if QT_VERSION >= QT_VERSION_CHECK(6, 0, 0) +#include +#endif /* createTemporaryTable @@ -29,7 +32,11 @@ QString getRandomHex(const int &length) QString randomHex; for(int i = 0; i < length; i++) { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) int n = qrand() % 16; +#else + int n = QRandomGenerator::global()->generate() % 16; +#endif randomHex.append(QString::number(n,16)); } diff --git a/utilities/createTemporaryTable/mainclass.cpp b/utilities/createTemporaryTable/mainclass.cpp index 84439603..5ab80fa1 100644 --- a/utilities/createTemporaryTable/mainclass.cpp +++ b/utilities/createTemporaryTable/mainclass.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -215,12 +215,12 @@ int mainClass::generateXLSX() QStringList arguments; QProcess *mySQLDumpProcess = new QProcess(); - QTime procTime; + QElapsedTimer procTime; procTime.start(); QString sql; QStringList fields; QStringList sheets; - QStringList csvs; + //QStringList csvs; QVector linked_tables; QVector multiSelectTables; diff --git a/utilities/mergeVersions/compareinsert.cpp b/utilities/mergeVersions/compareinsert.cpp index 17a6683c..e7c763fd 100644 --- a/utilities/mergeVersions/compareinsert.cpp +++ b/utilities/mergeVersions/compareinsert.cpp @@ -19,7 +19,7 @@ License along with Merge Versions. If not, see +//#include compareInsert::compareInsert(QObject *parent) : QObject(parent) { @@ -85,7 +85,9 @@ int compareInsert::createCFile() if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif docB.save(out,1,QDomNode::EncodingFromTextStream); file.close(); } @@ -107,7 +109,9 @@ int compareInsert::createDiffFile() if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream outD(&dfile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) outD.setCodec("UTF-8"); +#endif for (int tpos = 0; tpos < diff.count();tpos++) { if (diff[tpos].parsed == false) diff --git a/utilities/mergeVersions/main.cpp b/utilities/mergeVersions/main.cpp index 77bd5ef6..ac7dd988 100644 --- a/utilities/mergeVersions/main.cpp +++ b/utilities/mergeVersions/main.cpp @@ -128,12 +128,20 @@ int main(int argc, char *argv[]) QList valuesToIgnore; if (ignoreString != "") { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) ignoreTables = ignoreString.split(";",QString::SkipEmptyParts); +#else + ignoreTables = ignoreString.split(";",Qt::SkipEmptyParts); +#endif for (int pos =0; pos < ignoreTables.count(); pos++) { if (ignoreTables[pos].indexOf(":") > 0) { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) QStringList parts = ignoreTables[pos].split(":",QString::SkipEmptyParts); +#else + QStringList parts = ignoreTables[pos].split(":",Qt::SkipEmptyParts); +#endif if (parts.count() == 2) { int index; @@ -150,12 +158,20 @@ int main(int argc, char *argv[]) { TignoreTableValues aIgnoreValue; aIgnoreValue.table = parts[0]; +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) aIgnoreValue.values = parts[1].split(",",QString::SkipEmptyParts); +#else + aIgnoreValue.values = parts[1].split(",",Qt::SkipEmptyParts); +#endif valuesToIgnore.append(aIgnoreValue); } else { +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) valuesToIgnore[index].values.append(parts[1].split(",",QString::SkipEmptyParts)); +#else + valuesToIgnore[index].values.append(parts[1].split(",",Qt::SkipEmptyParts)); +#endif } } } diff --git a/utilities/mergeVersions/mainclass.cpp b/utilities/mergeVersions/mainclass.cpp index 587a0bae..dfc03233 100644 --- a/utilities/mergeVersions/mainclass.cpp +++ b/utilities/mergeVersions/mainclass.cpp @@ -131,7 +131,9 @@ void mainClass::run() if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif XMLResult.save(out,1,QDomNode::EncodingFromTextStream); file.close(); } diff --git a/utilities/mergeVersions/mergecreate.cpp b/utilities/mergeVersions/mergecreate.cpp index 1702eaa2..02bb6bdd 100644 --- a/utilities/mergeVersions/mergecreate.cpp +++ b/utilities/mergeVersions/mergecreate.cpp @@ -124,7 +124,9 @@ int mergeCreate::compare() if (file.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream out(&file); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) out.setCodec("UTF-8"); +#endif docB.save(out,1,QDomNode::EncodingFromTextStream); file.close(); } @@ -140,7 +142,9 @@ int mergeCreate::compare() if (dfile.open(QIODevice::WriteOnly | QIODevice::Text)) { QTextStream outD(&dfile); +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) outD.setCodec("UTF-8"); +#endif for (int dpos = 0; dpos < diff.count();dpos++) { outD << diff[dpos]; @@ -468,7 +472,7 @@ void mergeCreate::addTableToSDiff(QDomNode table, bool lookUp) { if (field.attribute("type","") != "text") { - if (field.attribute("type","") != "int(9)" && field.attribute("type","") != "decimal(17,3)") + if (field.attribute("type","") != "int(9)" && field.attribute("type","") != "decimal(17,3)" && field.attribute("type","") != "date" && field.attribute("type","") != "datetime") sql = sql + " " + field.attribute("type","") + " (" + field.attribute("size","0") + ")"; else sql = sql + " " + field.attribute("type",""); @@ -478,7 +482,7 @@ void mergeCreate::addTableToSDiff(QDomNode table, bool lookUp) } if (field.attribute("key","") == "true") { - sql = sql + "NOT NULL COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; + sql = sql + " NOT NULL COMMENT \"" + field.attribute("desc","Without description") + "\",\n"; keys.append(field.attribute("name","")); } else @@ -1135,8 +1139,14 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) QString parentTableName; parentTableName = table.parentNode().toElement().attribute("name",""); parentfound = findTable(docB,parentTableName); - if (!parentfound.isNull()) + + //qDebug() << eTable.toElement().attribute("name","") + "-A"; + //qDebug() << parentTableName + "-A"; + //qDebug() << newTables; + // If the parent if found or the parent is also a new table + if (!parentfound.isNull() || newTables.indexOf(parentTableName) >= 0) { + newTables.append(eTable.toElement().attribute("name","")); QDomNode field = table.firstChild(); while (!field.isNull()) { @@ -1158,8 +1168,11 @@ void mergeCreate::compareTables(QDomNode table,QDomDocument &docB) error.to = eTable.toElement().attribute("name",""); errorList.append(error); } - addTableToSDiff(eTable,false); - parentfound.appendChild(table.cloneNode(true)); + if (newTables.indexOf(parentTableName) < 0) + { + addTableToSDiff(eTable,false); + parentfound.appendChild(table.cloneNode(true)); + } } else { diff --git a/utilities/mergeVersions/mergecreate.h b/utilities/mergeVersions/mergecreate.h index ef19b4a5..daacde42 100644 --- a/utilities/mergeVersions/mergecreate.h +++ b/utilities/mergeVersions/mergecreate.h @@ -87,6 +87,7 @@ class mergeCreate : public QObject QList create_lookup_rels; QStringList dropped_rels; QStringList newFields; + QStringList newTables; void addAlterFieldToDiff(QString table, QDomElement eField, int newSize, int newDec, bool islookup); void ddTableToDrop(QString name); void changeLookupRelationship(QString table, QDomElement a, QDomElement b, bool islookup); diff --git a/utilities/utilities.pro b/utilities/utilities.pro index 482d9789..90df426a 100644 --- a/utilities/utilities.pro +++ b/utilities/utilities.pro @@ -3,9 +3,7 @@ TEMPLATE = subdirs CONFIG += debug SUBDIRS = createFromXML \ - DCFToODK \ insertFromXML \ - MySQLDenormalize \ MySQLToXLSX \ MySQLToCSV \ MySQLToJSON \