diff --git a/.gitmodules b/.gitmodules index fbeb6fc9..08350594 100644 --- a/.gitmodules +++ b/.gitmodules @@ -74,3 +74,6 @@ [submodule "src/lib/Clipper2"] path = src/lib/Clipper2 url = https://github.com/AngusJohnson/Clipper2 +[submodule "src/lib/pugixml"] + path = src/lib/pugixml + url = https://github.com/zeux/pugixml.git diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 90bfb74e..dc1ca7c4 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -38,6 +38,7 @@ + diff --git a/CMakeLists.txt b/CMakeLists.txt index 9770ea8d..1c0088fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,6 +307,10 @@ option(CLIPPER2_TESTS "Build tests" OFF) add_subdirectory(src/lib/Clipper2/CPP) target_link_libraries(BrickSimLib PUBLIC Clipper2) +#pugixml +add_subdirectory(src/lib/pugixml) +target_link_libraries(BrickSimLib PUBLIC pugixml-static) + add_subdirectory(src) if (WIN32) diff --git a/src/constant_data/constants.cpp b/src/constant_data/constants.cpp index 7e25e8ba..d9400702 100644 --- a/src/constant_data/constants.cpp +++ b/src/constant_data/constants.cpp @@ -66,5 +66,4 @@ namespace bricksim::constants { ; const char* versionString = NUM(BRICKSIM_VERSION_MAJOR) "." NUM(BRICKSIM_VERSION_MINOR) "." NUM(BRICKSIM_VERSION_PATCH); - const char* LDRAW_LIBRARY_DOWNLOAD_URL = "https://library.ldraw.org/library/updates/complete.zip"; } diff --git a/src/constant_data/constants.h b/src/constant_data/constants.h index 6ed4925a..65060eda 100644 --- a/src/constant_data/constants.h +++ b/src/constant_data/constants.h @@ -37,7 +37,10 @@ namespace bricksim::constants { extern const uint16_t gitCommitCount; extern const char* gitCommitHash; - extern const char* LDRAW_LIBRARY_DOWNLOAD_URL; + static constexpr auto LDRAW_LIBRARY_DOWNLOAD_URL = "https://library.ldraw.org/library/updates/complete.zip"; + static constexpr auto LDRAW_LIBRARY_UPDATES_XML_URL = "https://library.ldraw.org/updates?output=XML"; + + static constexpr auto LDRAW_CONFIG_FILE_NAME = "LDConfig.ldr"; constexpr float pInf = std::numeric_limits::infinity(); constexpr float nInf = -pInf; diff --git a/src/controller.cpp b/src/controller.cpp index 9870ba2d..e7af8c12 100644 --- a/src/controller.cpp +++ b/src/controller.cpp @@ -350,12 +350,12 @@ namespace bricksim::controller { } std::array initSteps = { - Task{"load color definitions", ldr::color_repo::initialize}, Task{"initialize shadow file repo", ldr::file_repo::initializeShadowFileRepo}, Task{"initialize file list", [](float* progress) { ldr::file_repo::get().initialize(progress); spdlog::info("File Repo base path is {}", ldr::file_repo::get().getBasePath().string()); }}, + Task{"load color definitions", ldr::color_repo::initialize}, Task{"initialize price guide provider", info_providers::price_guide::initialize}, Task{"initialize thumbnail generator", []() { thumbnailGenerator = std::make_shared(); }}, Task{"initialize BrickLink constants", info_providers::bricklink_constants::initialize}, diff --git a/src/db.cpp b/src/db.cpp index a8ceda69..df3f798c 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -264,6 +264,9 @@ namespace bricksim::db { } void put(const std::vector& entries) { + if (entries.empty()) { + return; + } std::string command = "INSERT INTO files (name, title, category) VALUES "; std::string name; std::string title; diff --git a/src/db.h b/src/db.h index 01c4204d..17b5dee9 100644 --- a/src/db.h +++ b/src/db.h @@ -42,6 +42,7 @@ namespace bricksim::db { namespace valueCache { constexpr auto LAST_INDEX_LDCONFIG_HASH = "LAST_INDEX_LDCONFIG_HASH"; + constexpr auto CURRENT_PARTS_LIBRARY_VERSION = "CURRENT_PARTS_LIBRARY_VERSION"; template std::optional get(const char* key); diff --git a/src/gui/main_menu_bar.cpp b/src/gui/main_menu_bar.cpp index b535406c..04af1ffe 100644 --- a/src/gui/main_menu_bar.cpp +++ b/src/gui/main_menu_bar.cpp @@ -27,6 +27,7 @@ namespace bricksim::gui { void drawUtilitiesMenu() { if (ImGui::BeginMenu("Utilities")) { ImGui::MenuItem(windows::getName(windows::Id::GEAR_RATIO_CALCULATOR), "", windows::isVisible(windows::Id::GEAR_RATIO_CALCULATOR)); + ImGui::MenuItem(windows::getName(windows::Id::LDRAW_LIBRARY_UPDATER), "", windows::isVisible(windows::Id::LDRAW_LIBRARY_UPDATER)); ImGui::EndMenu(); } } diff --git a/src/gui/windows/utilities/CMakeLists.txt b/src/gui/windows/utilities/CMakeLists.txt index 7913eee3..42f64200 100644 --- a/src/gui/windows/utilities/CMakeLists.txt +++ b/src/gui/windows/utilities/CMakeLists.txt @@ -1,4 +1,6 @@ target_sources(BrickSimLib PRIVATE window_gear_ratio_calculator.cpp window_gear_ratio_calculator.h + window_ldraw_library_updater.cpp + window_ldraw_library_updater.h ) \ No newline at end of file diff --git a/src/gui/windows/utilities/window_gear_ratio_calculator.cpp b/src/gui/windows/utilities/window_gear_ratio_calculator.cpp index 0e8be333..7a3b800e 100644 --- a/src/gui/windows/utilities/window_gear_ratio_calculator.cpp +++ b/src/gui/windows/utilities/window_gear_ratio_calculator.cpp @@ -5,7 +5,7 @@ #include "window_gear_ratio_calculator.h" -namespace bricksim::gui::windows::tools::gear_ratio_calculator { +namespace bricksim::gui::windows::utilities::gear_ratio_calculator { void draw(Data& data) { if (ImGui::Begin(data.name, &data.visible)) { collectWindowInfo(data.id); @@ -91,7 +91,7 @@ namespace bricksim::gui::windows::tools::gear_ratio_calculator { ImGui::Separator(); ImGui::Text("Gear Ratio: %ld:%ld", totalRatio.getA(), totalRatio.getB()); - ImGui::End(); } + ImGui::End(); } } diff --git a/src/gui/windows/utilities/window_gear_ratio_calculator.h b/src/gui/windows/utilities/window_gear_ratio_calculator.h index 3a8aa9de..018e2acf 100644 --- a/src/gui/windows/utilities/window_gear_ratio_calculator.h +++ b/src/gui/windows/utilities/window_gear_ratio_calculator.h @@ -2,6 +2,6 @@ #include "../windows.h" -namespace bricksim::gui::windows::tools::gear_ratio_calculator { +namespace bricksim::gui::windows::utilities::gear_ratio_calculator { void draw(Data& data); } diff --git a/src/gui/windows/utilities/window_ldraw_library_updater.cpp b/src/gui/windows/utilities/window_ldraw_library_updater.cpp new file mode 100644 index 00000000..0a9dd92b --- /dev/null +++ b/src/gui/windows/utilities/window_ldraw_library_updater.cpp @@ -0,0 +1,101 @@ +#include "window_ldraw_library_updater.h" +#include "../../../controller.h" +#include "../../../utilities/ldraw_library_updater.h" +#include "../../gui.h" + +namespace bricksim::gui::windows::utilities::ldraw_library_updater { + using namespace bricksim::ldraw_library_updater; + + void draw(Data& data) { + const auto lastVisible = data.visible; + if (ImGui::Begin(data.name, &data.visible)) { + collectWindowInfo(data.id); + auto& state = getState(); + + switch (state.step) { + case Step::INACTIVE: + controller::addBackgroundTask("Initialize LDraw Library Updater", []() { getState().initialize(); }); + case Step::INITIALIZING: + { + const auto progressMessage = fmt::format("Initializing {}%", state.initializingProgress * 100.f); + ImGui::ProgressBar(state.initializingProgress, ImVec2(-FLT_MIN, 0), progressMessage.c_str()); + break; + } + case Step::CHOOSE_ACTION: + case Step::UPDATE_INCREMENTAL: + case Step::UPDATE_COMPLETE: + ImGui::SeparatorText("Current Status"); + ImGui::Text("Library base path: %s", ldr::file_repo::get().getBasePath().string().c_str()); + ImGui::Text("Current LDConfig date: %d-%02d-%02d", (int)state.currentReleaseDate.year(), (unsigned int)state.currentReleaseDate.month(), (unsigned int)state.currentReleaseDate.day()); + ImGui::Text("Current release ID: %s", state.currentReleaseId.c_str()); + + ImGui::SeparatorText("Incremental Update"); + if (state.incrementalUpdates.empty()) { + ImGui::PushStyleColor(ImGuiCol_Text, color::GREEN); + ImGui::Text(ICON_FA_CHECK " no incremental updates found. This usually means that your library is up to date."); + ImGui::PopStyleColor(); + } else { + if (ImGui::BeginTable("incrementalUpdates", state.step == Step::UPDATE_INCREMENTAL ? 4 : 3)) { + ImGui::TableSetupColumn("ID"); + ImGui::TableSetupColumn("Date"); + ImGui::TableSetupColumn("Size"); + if (state.step == Step::UPDATE_INCREMENTAL) { + ImGui::TableSetupColumn("Progress"); + } + ImGui::TableHeadersRow(); + + for (int i = 0; i < state.incrementalUpdates.size(); ++i) { + const auto& dist = state.incrementalUpdates[i]; + + ImGui::TableNextColumn(); + ImGui::Text("%s", dist.id.c_str()); + + ImGui::TableNextColumn(); + ImGui::Text("%d-%02d-%02d", (int)dist.date.year(), (unsigned int)dist.date.month(), (unsigned int)dist.date.day()); + + ImGui::TableNextColumn(); + ImGui::Text("%s", stringutil::formatBytesValue(dist.size).c_str()); + + if (state.step == Step::UPDATE_INCREMENTAL) { + ImGui::TableNextColumn(); + ImGui::ProgressBar(i < state.incrementalUpdateProgress.size() ? state.incrementalUpdateProgress[i] : 0.f); + } + } + ImGui::EndTable(); + } + ImGui::Text("Total download size for incremental update: %s", stringutil::formatBytesValue(state.getIncrementalUpdateTotalSize()).c_str()); + ImGui::BeginDisabled(state.step != Step::CHOOSE_ACTION); + if (ImGui::Button("Do Incremental Update")) { + controller::addBackgroundTask("Update LDraw Library", []() { getState().doIncrementalUpdate(); }); + } + ImGui::EndDisabled(); + } + + ImGui::SeparatorText("Complete Update"); + if (state.completeDistribution) { + ImGui::Text("Complete update download size: %s", stringutil::formatBytesValue(state.completeDistribution->size).c_str()); + ImGui::BeginDisabled(state.step != Step::CHOOSE_ACTION); + if (ImGui::Button("Do Complete Update")) { + controller::addBackgroundTask("Update LDraw Library", []() { getState().doCompleteUpdate(); }); + } + ImGui::EndDisabled(); + if (state.step == Step::UPDATE_COMPLETE && state.completeUpdateProgress.has_value()) { + ImGui::ProgressBar(*state.completeUpdateProgress); + } + } else { + ImGui::Text("Complete distribution is not available"); + } + + break; + case Step::FINISHED: + ImGui::Text("Update finished."); + ImGui::Text("Your parts library is up to date"); + break; + } + } + if (lastVisible && !data.visible) { + resetState(); + } + ImGui::End(); + } +} \ No newline at end of file diff --git a/src/gui/windows/utilities/window_ldraw_library_updater.h b/src/gui/windows/utilities/window_ldraw_library_updater.h new file mode 100644 index 00000000..681962b9 --- /dev/null +++ b/src/gui/windows/utilities/window_ldraw_library_updater.h @@ -0,0 +1,7 @@ +#pragma once + +#include "../windows.h" + +namespace bricksim::gui::windows::utilities::ldraw_library_updater { + void draw(Data& data); +} \ No newline at end of file diff --git a/src/gui/windows/windows.cpp b/src/gui/windows/windows.cpp index ac73debe..aef5765f 100644 --- a/src/gui/windows/windows.cpp +++ b/src/gui/windows/windows.cpp @@ -8,6 +8,7 @@ #include "../../metrics.h" #include "utilities/window_gear_ratio_calculator.h" +#include "utilities/window_ldraw_library_updater.h" #include "window_about.h" #include "window_connection_visualization.h" #include "window_debug.h" @@ -49,12 +50,13 @@ namespace bricksim::gui::windows { {Id::IMGUI_DEMO, ICON_FA_IMAGE " ImGui Demo", false, drawImGuiDemo, noCleanup}, {Id::ORIENTATION_CUBE, ICON_FA_CUBE " Orientation Cube", true, orientation_cube::draw, noCleanup}, {Id::LOG, ICON_FA_LIST " Log", false, log::draw, noCleanup}, - {Id::GEAR_RATIO_CALCULATOR, ICON_FA_GEARS " Gear Ratio Calculator", false, tools::gear_ratio_calculator::draw, noCleanup}, + {Id::GEAR_RATIO_CALCULATOR, ICON_FA_GEARS " Gear Ratio Calculator", false, utilities::gear_ratio_calculator::draw, noCleanup}, {Id::MODEL_INFO, ICON_FA_INFO " Model Info", false, model_info::draw, noCleanup}, {Id::EDITOR_META_INFO, ICON_FA_RECEIPT " Meta-Info", false, editor_meta_info::draw, noCleanup}, {Id::LDRAW_FILE_INSPECTOR, ICON_FA_EYE " LDraw File Inspector", false, ldraw_file_inspector::draw, noCleanup}, {Id::TOOLBAR, ICON_FA_SCREWDRIVER_WRENCH " Toolbar", true, toolbar::draw, noCleanup}, {Id::CONNECTION_VISUALIZATION, ICON_FA_SHARE_NODES " Connection visualization", false, connection_visualization::draw, connection_visualization::cleanup}, + {Id::LDRAW_LIBRARY_UPDATER, ICON_FA_DOWNLOAD " LDraw Library Updater", false, utilities::ldraw_library_updater::draw, noCleanup}, }}; void drawAll() { diff --git a/src/gui/windows/windows.h b/src/gui/windows/windows.h index 7a1181f0..d71c8422 100644 --- a/src/gui/windows/windows.h +++ b/src/gui/windows/windows.h @@ -22,6 +22,7 @@ namespace bricksim::gui::windows { LDRAW_FILE_INSPECTOR, TOOLBAR, CONNECTION_VISUALIZATION, + LDRAW_LIBRARY_UPDATER, }; struct Data { diff --git a/src/helpers/parts_library_downloader.cpp b/src/helpers/parts_library_downloader.cpp index 77d8b846..da4f7628 100644 --- a/src/helpers/parts_library_downloader.cpp +++ b/src/helpers/parts_library_downloader.cpp @@ -8,8 +8,12 @@ #include "../config/write.h" namespace bricksim::parts_library_downloader { - namespace { - int progressFunc([[maybe_unused]] void* clientp, long downloadTotal, long downloadNow, [[maybe_unused]] long uploadTotal, [[maybe_unused]] long uploadNow) { + + void downloadPartsLibrary() { + status = Status::IN_PROGRESS; + spdlog::info("starting parts library download"); + auto filePath = util::extendHomeDir("~/ldraw.zip"); + auto [statusCode, content] = util::downloadFile(constants::LDRAW_LIBRARY_DOWNLOAD_URL, filePath, [](long downloadTotal, long downloadNow, [[maybe_unused]] long uploadTotal, [[maybe_unused]] long uploadNow) { const auto now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); const auto msDiff = now - lastSpeedCalcTimestampMs; if (msDiff > 1000) { @@ -22,14 +26,7 @@ namespace bricksim::parts_library_downloader { downNow = downloadNow; downTotal = downloadTotal; return shouldStop ? 1 : 0; - } - } - - void downloadPartsLibrary() { - status = Status::IN_PROGRESS; - spdlog::info("starting parts library download"); - auto filePath = util::extendHomeDir("~/ldraw.zip"); - auto [statusCode, content] = util::downloadFile(constants::LDRAW_LIBRARY_DOWNLOAD_URL, filePath, progressFunc); + }); if (statusCode < 200 || statusCode >= 300) { spdlog::error("parts library download failed. Error code: {}", statusCode); errorCode = statusCode; diff --git a/src/helpers/stringutil.cpp b/src/helpers/stringutil.cpp index 840cf544..02e55012 100644 --- a/src/helpers/stringutil.cpp +++ b/src/helpers/stringutil.cpp @@ -1,4 +1,5 @@ #include "stringutil.h" +#include "fast_float/fast_float.h" #include #include #include @@ -333,4 +334,37 @@ namespace bricksim::stringutil { } return result; } + std::size_t findClosingQuote(std::string_view str, std::size_t start) { + std::size_t end = start; + do { + end = str.find('"', end); + } while (end!=std::string_view::npos && (end==0 || str[end-1]=='\\')); + return end; + } + std::chrono::year_month_day parseYYYY_MM_DD(std::string_view str) { + if (str.length() != std::strlen("0000-00-00")) { + throw std::invalid_argument("invalid size"); + } + int year; + int month; + int day; + fast_float::from_chars(str.data()+0, str.data()+4, year); + fast_float::from_chars(str.data()+5, str.data()+7, month); + fast_float::from_chars(str.data()+8, str.data()+10, day); + return std::chrono::year_month_day(std::chrono::year(year), + std::chrono::month(month), + std::chrono::day(day)); + } + std::chrono::year_month_day parseYYYY_MM(std::string_view str, int day) { + if (str.length() != std::strlen("0000-00")) { + throw std::invalid_argument("invalid size"); + } + int year; + int month; + fast_float::from_chars(str.data()+0, str.data()+4, year); + fast_float::from_chars(str.data()+5, str.data()+7, month); + return std::chrono::year_month_day(std::chrono::year(year), + std::chrono::month(month), + std::chrono::day(day)); + } } diff --git a/src/helpers/stringutil.h b/src/helpers/stringutil.h index 25be6215..73c30eab 100644 --- a/src/helpers/stringutil.h +++ b/src/helpers/stringutil.h @@ -2,6 +2,7 @@ #include "../gui/icons.h" #include "../lib/IconFontCppHeaders/IconsFontAwesome6.h" #include +#include #include #include #include @@ -77,4 +78,12 @@ namespace bricksim::stringutil { } std::string removeIcons(std::string_view withIcons); + + /** + * @return the index of the first occurrence of " that is >= start and not preceded by a backslash + */ + std::size_t findClosingQuote(std::string_view str, std::size_t start = 0); + + std::chrono::year_month_day parseYYYY_MM_DD(std::string_view str); + std::chrono::year_month_day parseYYYY_MM(std::string_view str, int day=1); } diff --git a/src/helpers/util.cpp b/src/helpers/util.cpp index 48842a69..f52c9b50 100644 --- a/src/helpers/util.cpp +++ b/src/helpers/util.cpp @@ -15,18 +15,20 @@ #include #include #include +#include #ifdef BRICKSIM_PLATFORM_WINDOWS #include -#ifdef min + #ifdef min #undef min -#endif -#ifdef max + #endif + #ifdef max #undef max -#endif + #endif #elif defined(BRICKSIM_PLATFORM_LINUX) || defined(BRICKSIM_PLATFORM_MACOS) -#include -#include + #include + #include + #include #endif namespace bricksim::util { @@ -70,21 +72,21 @@ namespace bricksim::util { void openDefaultBrowser(const std::string& link) { spdlog::info("openDefaultBrowser(\"{}\")", link); - #ifdef BRICKSIM_PLATFORM_WINDOWS +#ifdef BRICKSIM_PLATFORM_WINDOWS ShellExecute(nullptr, "open", link.c_str(), nullptr, nullptr, SW_SHOWNORMAL);//todo testing - #elif defined(BRICKSIM_PLATFORM_MACOS) +#elif defined(BRICKSIM_PLATFORM_MACOS) std::string command = std::string("open ") + link; - #elif defined(BRICKSIM_PLATFORM_LINUX) +#elif defined(BRICKSIM_PLATFORM_LINUX) std::string command = std::string("xdg-open ") + link; - #else +#else #warning "openDefaultProwser not supported on this platform" - #endif - #if defined(BRICKSIM_PLATFORM_LINUX) || defined(BRICKSIM_PLATFORM_MACOS) +#endif +#if defined(BRICKSIM_PLATFORM_LINUX) || defined(BRICKSIM_PLATFORM_MACOS) int exitCode = system(command.c_str()); if (exitCode != 0) { spdlog::warn("command \"{}\" exited with code {}", command, exitCode); } - #endif +#endif } bool memeqzero(const void* data, size_t length) { @@ -226,27 +228,47 @@ namespace bricksim::util { return written; } - std::pair downloadFile(const std::string& url, const std::filesystem::path targetFile, int (*progressFunc)(void*, long, long, long, long)) { - CURL* curl = curl_easy_init(); + struct CurlActionData { + std::string url; + std::function progressCallback; + }; + + int curl_xferinfo_callback_func(void* clientp, curl_off_t dlTotal, curl_off_t dlNow, curl_off_t ulTotal, curl_off_t ulNow) { + const auto actionData = (CurlActionData*)clientp; + return actionData->progressCallback(dlTotal, dlNow, ulTotal, ulNow); + } + + FileDownloadResult downloadFile(const std::string& url, const std::filesystem::path targetFile, const std::optional> progressFunc) { + auto curl = std::unique_ptr(curl_easy_init(), curl_easy_cleanup); if (curl) { FILE* fp = fopen(targetFile.string().c_str(), "wb"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionToFile); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, writeFunctionToFile); + curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, fp); + std::unique_ptr actionData; if (progressFunc != nullptr) { - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progressFunc); + actionData = std::make_unique(url, *progressFunc); + curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(curl.get(), CURLOPT_XFERINFOFUNCTION, curl_xferinfo_callback_func); + curl_easy_setopt(curl.get(), CURLOPT_XFERINFODATA, actionData.get()); } else { - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L); + actionData = nullptr; + curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1L); } - CURLcode res = curl_easy_perform(curl); + CURLcode res = curl_easy_perform(curl.get()); + long httpCode; + curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &httpCode); + curl_off_t contentSize; + curl_easy_getinfo(curl.get(), CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &contentSize); - curl_easy_cleanup(curl); fclose(fp); + + return {static_cast(httpCode), static_cast(contentSize)}; } - return {234, "TODO"}; + return {-1, 0}; } std::string readFileToString(const std::filesystem::path& path) { @@ -306,14 +328,14 @@ namespace bricksim::util { } void setThreadName(const char* threadName) { - #ifdef BRICKSIM_PLATFORM_LINUX +#ifdef BRICKSIM_PLATFORM_LINUX pthread_setname_np(pthread_self(), threadName); - #elif defined(BRICKSIM_PLATFORM_MACOS) +#elif defined(BRICKSIM_PLATFORM_MACOS) pthread_setname_np(threadName); - #endif - #ifdef USE_PL +#endif +#ifdef USE_PL plDeclareThreadDyn(threadName); - #endif +#endif } UtfType determineUtfTypeFromBom(const std::string_view text) { @@ -345,11 +367,11 @@ namespace bricksim::util { } int64_t getPID() { - #if defined(BRICKSIM_PLATFORM_WINDOWS) +#if defined(BRICKSIM_PLATFORM_WINDOWS) return GetCurrentProcessId(); - #else +#else return getpid(); - #endif +#endif } bool isStbiFlipVertically() { @@ -410,4 +432,36 @@ namespace bricksim::util { glm::mat4 DecomposedTransformation::scaleAsMat4() const { return glm::scale(glm::mat4(1.0f), scale); } + + std::array md5(const std::filesystem::path& filePath) { + auto context = std::unique_ptr(EVP_MD_CTX_new(), EVP_MD_CTX_free); + if (!context) { + throw std::runtime_error("Cannot calculate MD5: failed to create OpenSSL EVP MD context"); + } + + const EVP_MD *md = EVP_md5(); + if (!EVP_DigestInit_ex2(context.get(), md, nullptr)) { + throw std::runtime_error("Cannot calculate MD5: failed to initialize digest"); + } + + std::ifstream file(filePath, std::ios::binary); + if (!file) { + throw std::runtime_error("Cannot calculate MD5: failed to open file " + filePath.string()); + } + + std::vector buffer(4096); + while (file.read(buffer.data(), buffer.size()) || file.gcount() > 0) { + if (!EVP_DigestUpdate(context.get(), buffer.data(), file.gcount())) { + throw std::runtime_error("Cannot calculate MD5: failed to update digest"); + } + } + + std::array md5Hash; + unsigned int mdLen; + if (!EVP_DigestFinal_ex(context.get(), reinterpret_cast(md5Hash.data()), &mdLen) || mdLen != 16) { + throw std::runtime_error("Cannot calculate MD5: failed to finalize digest"); + } + + return md5Hash; + } } diff --git a/src/helpers/util.h b/src/helpers/util.h index 52275f81..3995b611 100644 --- a/src/helpers/util.h +++ b/src/helpers/util.h @@ -154,13 +154,17 @@ namespace bricksim::util { */ std::pair requestGET(const std::string& url, bool useCache = true, size_t sizeLimit = 0, int (*progressFunc)(void*, long, long, long, long) = nullptr); + struct FileDownloadResult { + int httpCode; + std::size_t contentLength; + }; /** * @param url * @param targetFile where the file should be saved * @param progressFunc int(void* clientp, long downloadTotal, long downloadNow, long uploadTotal, long uploadNow) // if return value is != 0, transfer stops * @return (responseCode, responseString) */ - std::pair downloadFile(const std::string& url, std::filesystem::path targetFile, int (*progressFunc)(void*, long, long, long, long) = nullptr); + FileDownloadResult downloadFile(const std::string& url, std::filesystem::path targetFile, const std::optional> progressFunc = std::nullopt); std::string readFileToString(const std::filesystem::path& path); @@ -274,4 +278,6 @@ namespace bricksim::util { UtfType determineUtfTypeFromBom(const std::string_view text); int64_t getPID(); + + std::array md5(const std::filesystem::path& filePath); } diff --git a/src/ldr/CMakeLists.txt b/src/ldr/CMakeLists.txt index 99ab9ec8..29fe2674 100644 --- a/src/ldr/CMakeLists.txt +++ b/src/ldr/CMakeLists.txt @@ -1,6 +1,8 @@ target_sources(BrickSimLib PRIVATE colors.cpp colors.h + config.cpp + config.h file_reader.cpp file_reader.h file_repo.cpp diff --git a/src/ldr/colors.cpp b/src/ldr/colors.cpp index e86bbae9..6e3c9db4 100644 --- a/src/ldr/colors.cpp +++ b/src/ldr/colors.cpp @@ -1,12 +1,13 @@ #include "colors.h" #include "../helpers/stringutil.h" #include "../helpers/util.h" +#include "config.h" #include "file_repo.h" #include namespace bricksim::ldr { - Color::Color(const std::string& line) { - std::stringstream linestream(line);//todo optimize this one day (using strtok instead of stringstream) + Color::Color(const std::string_view line) { + std::stringstream linestream((std::string(line)));//todo optimize this one day (using strtok instead of stringstream) linestream >> name; while (linestream.rdbuf()->in_avail() != 0) { std::string keyword; @@ -121,16 +122,9 @@ namespace bricksim::ldr { void initialize() { static bool initialized = false; if (!initialized) { - std::stringstream inpStream; - std::string contentString = ldr::file_repo::get().getLibraryLdrFileContent("LDConfig.ldr"); - inpStream << contentString; - for (std::string line; getline(inpStream, line);) { - auto trimmed = stringutil::trim(line); - if (!trimmed.empty() && trimmed.rfind("0 !COLOUR", 0) == 0) { - auto col = std::make_shared(line.substr(10)); - colors[col->code] = col; - hueSortedCodes.push_back({color::HSV(col->value).hue, col->code}); - } + for (const auto& col: getConfig().getColors()) { + colors[col->code] = col; + hueSortedCodes.push_back({color::HSV(col->value).hue, col->code}); } instDummyColor = std::make_shared(); colors[INSTANCE_DUMMY_COLOR_CODE] = instDummyColor; diff --git a/src/ldr/colors.h b/src/ldr/colors.h index 0ebacefe..a8bcb8c1 100644 --- a/src/ldr/colors.h +++ b/src/ldr/colors.h @@ -45,7 +45,7 @@ namespace bricksim::ldr { Color() = default; - explicit Color(const std::string& line); + explicit Color(const std::string_view line); std::string name; code_t code; color::RGB value; diff --git a/src/ldr/config.cpp b/src/ldr/config.cpp new file mode 100644 index 00000000..a52fe804 --- /dev/null +++ b/src/ldr/config.cpp @@ -0,0 +1,109 @@ +#include "config.h" +#include "fast_float/fast_float.h" +#include "file_repo.h" +#include "spdlog/spdlog.h" + +namespace bricksim::ldr { + namespace { + std::unique_ptr config; + std::shared_ptr getCurrentFileFromRepo() { + return file_repo::get().getFile((std::shared_ptr)nullptr, constants::LDRAW_CONFIG_FILE_NAME); + } + + } + + const LDConfig& getConfig() { + std::shared_ptr currentFile; + if (config!= nullptr) { + currentFile = getCurrentFileFromRepo(); + if (currentFile != config->getFile()) { + config = nullptr; + } + } + if (config == nullptr) { + if (currentFile == nullptr) { + currentFile = getCurrentFileFromRepo(); + } + config = std::make_unique(currentFile); + } + return *config; + } + LDConfig::LDConfig(const std::shared_ptr& file) : + file(file) { + const std::string_view updateLine = file->metaInfo.fileTypeLine; + const auto startIdx = updateLine.find_first_not_of(LDR_WHITESPACE, updateLine.find("UPDATE")+6); + const auto endIdx = updateLine.find_first_of(LDR_WHITESPACE, startIdx); + updateDate = updateLine.substr(startIdx, endIdx); + + for (const auto& element: file->elements) { + if (element->getType() == 0) { + const auto& content = std::dynamic_pointer_cast(element)->content; + const auto contentTrimmed = stringutil::trim(std::string_view(content)); + if (contentTrimmed.starts_with("!COLOUR")) { + colors.push_back(std::make_shared(stringutil::trim(contentTrimmed.substr(strlen("!COLOUR"))))); + } else if (contentTrimmed.starts_with("!LDRAW_ORG Configuration UPDATE")) { + + } else if (contentTrimmed.starts_with("!AVATAR")) { + try { + avatars.emplace_back(stringutil::trim(contentTrimmed.substr(strlen("!AVATAR")))); + } catch (std::invalid_argument) { + spdlog::warn("invalid avatar found in ldraw config file: {}", contentTrimmed); + } + } + } + } + } + std::shared_ptr LDConfig::getFile() const { + return file.lock(); + } + const std::string& LDConfig::getUpdateDate() const { + return updateDate; + } + const std::vector>& LDConfig::getColors() const { + return colors; + } + const std::vector& LDConfig::getAvatars() const { + return avatars; + } + + Avatar::Avatar(std::string_view line) { + std::size_t start; + std::size_t end = 0; + + if (!line.starts_with("CATEGORY")) { + throw std::invalid_argument(""); + } + start = line.find('"')+1; + end = stringutil::findClosingQuote(line, start); + if (end==std::string_view::npos) { + throw std::invalid_argument(""); + } + category = line.substr(start, end); + + start = line.find_first_not_of(LDR_WHITESPACE, end+1); + if (!line.substr(start).starts_with("DESCRIPTION")) { + throw std::invalid_argument(""); + } + start = line.find('"', start)+1; + end = stringutil::findClosingQuote(line, start); + if (end==std::string_view::npos) { + throw std::invalid_argument(""); + } + description = line.substr(start, end); + + start = line.find_first_not_of(LDR_WHITESPACE, end+1); + if (!line.substr(start).starts_with("PART")) { + throw std::invalid_argument(""); + } + end = line.find_first_of(LDR_WHITESPACE, start); + for (int i = 0; i < 9; ++i) { + start = line.find_first_not_of(LDR_WHITESPACE, end); + end = std::min(line.size(), line.find_first_of(LDR_WHITESPACE, start)); + fast_float::from_chars(line.data()+start, line.data()+end, rotation[i]); + } + + start = line.find('"', end)+1; + end = std::min(line.size(), stringutil::findClosingQuote(line, start)); + fileName = line.substr(start, end); + } +} \ No newline at end of file diff --git a/src/ldr/config.h b/src/ldr/config.h new file mode 100644 index 00000000..12181bb5 --- /dev/null +++ b/src/ldr/config.h @@ -0,0 +1,28 @@ +#pragma once + +#include "files.h" +#include +namespace bricksim::ldr { + struct Avatar { + Avatar(std::string_view line); + std::string category; + std::string description; + std::array rotation; + std::string fileName; + }; + + class LDConfig { + std::weak_ptr file; + std::string updateDate; + std::vector> colors; + std::vector avatars; + public: + explicit LDConfig(const std::shared_ptr& file); + std::shared_ptr getFile() const; + const std::string& getUpdateDate() const; + const std::vector>& getColors() const; + const std::vector& getAvatars() const; + }; + + const LDConfig& getConfig(); +} \ No newline at end of file diff --git a/src/ldr/file_repo.cpp b/src/ldr/file_repo.cpp index 2cc0c0e8..37a97008 100644 --- a/src/ldr/file_repo.cpp +++ b/src/ldr/file_repo.cpp @@ -23,6 +23,7 @@ namespace bricksim::ldr::file_repo { const char* PSEUDO_CATEGORY_MODEL = "__MODEL"; const char* PSEUDO_CATEGORY_HIDDEN_PART = "__HIDDEN_PART"; const char* PSEUDO_CATEGORY_BINARY_FILE = "__BINARY_FILE"; + const char* PSEUDO_CATEGORY_OTHER = "__OTHER"; const char* const PSEUDO_CATEGORIES[] = {PSEUDO_CATEGORY_SUBPART, PSEUDO_CATEGORY_PRIMITIVE, PSEUDO_CATEGORY_MODEL, PSEUDO_CATEGORY_HIDDEN_PART, PSEUDO_CATEGORY_BINARY_FILE}; const char* const PART_SEARCH_PREFIXES[] = {"parts/", "p/", "models/", ""}; @@ -96,8 +97,8 @@ namespace bricksim::ldr::file_repo { std::string getContentOfLdrFile(const std::filesystem::path& path) { return path.extension() == ".io" - ? getContentOfIoFile(path) - : util::readFileToString(path); + ? getContentOfIoFile(path) + : util::readFileToString(path); } } @@ -197,6 +198,8 @@ namespace bricksim::ldr::file_repo { type = FileType::PRIMITIVE; } else if (entryOpt->category == PSEUDO_CATEGORY_MODEL) { type = FileType::MODEL; + } else if (entryOpt->category == PSEUDO_CATEGORY_OTHER) { + type = FileType::OTHER; } else { type = FileType::PART; } @@ -213,8 +216,8 @@ namespace bricksim::ldr::file_repo { } const auto finalPath = contextRelativePath.has_value() - ? fileNamespace->searchPath / *contextRelativePath / name - : fileNamespace->searchPath / name; + ? fileNamespace->searchPath / *contextRelativePath / name + : fileNamespace->searchPath / name; if (std::filesystem::exists(finalPath)) { return addLdrFileWithContent(fileNamespace, name, finalPath, FileType::MODEL, getContentOfLdrFile(finalPath)); } @@ -250,8 +253,8 @@ namespace bricksim::ldr::file_repo { auto filenameWithForwardSlash = stringutil::replaceChar(name, '\\', '/'); std::string nameInLibrary; const auto prePrefixes = searchPath == BinaryFileSearchPath::TEXMAP - ? std::vector({"textures/", ""}) - : std::vector({""}); + ? std::vector({"textures/", ""}) + : std::vector({""}); for (const auto& prePrefix: prePrefixes) { for (const auto& prefix: PART_SEARCH_PREFIXES) { auto fullName = prefix + prePrefix; @@ -333,7 +336,8 @@ namespace bricksim::ldr::file_repo { return (isLdrFilename(filename) || isBinaryFilename(filename)) && (filename.starts_with("parts/") || filename.starts_with("p/") - || filename.starts_with("models/")); + || filename.starts_with("models/") + || filename == constants::LDRAW_CONFIG_FILE_NAME); } bool FileRepo::isLdrFilename(const std::string& filename) { @@ -358,6 +362,7 @@ namespace bricksim::ldr::file_repo { return "parts/" + name; case FileType::PRIMITIVE: return "p/" + name; + case FileType::OTHER: default: return name; } @@ -373,93 +378,116 @@ namespace bricksim::ldr::file_repo { } else if (pathRelativeToBase.starts_with("models/")) { return {ldr::FileType::MODEL, pathRelativeToBase.substr(7)}; } - return {ldr::FileType::MODEL, pathRelativeToBase}; + return {ldr::FileType::OTHER, pathRelativeToBase}; } - constexpr const auto LDCONFIG_FILE_NAME = "LDConfig.ldr"; - void FileRepo::initialize(float* progress) { - auto currentLDConfigContent = getLibraryLdrFileContent(LDCONFIG_FILE_NAME); - const auto currentHash = fmt::format("{:x}", adler32(1, reinterpret_cast(currentLDConfigContent.data()), currentLDConfigContent.size())); + std::string currentHash = getLDConfigContentHash(); const auto lastIndexHash = db::valueCache::get(db::valueCache::LAST_INDEX_LDCONFIG_HASH); bool needFill = false; if (currentHash != lastIndexHash) { needFill = true; - spdlog::info("FileRepo: Hash of {} changed ({}!={}), going to refill file list", LDCONFIG_FILE_NAME, lastIndexHash.value_or("?"), currentHash); - db::fileList::deleteAllEntries(); + spdlog::info("FileRepo: Hash of {} changed ({}!={}), going to refill file list", constants::LDRAW_CONFIG_FILE_NAME, lastIndexHash.value_or("?"), currentHash); } else if (db::fileList::getSize() == 0) { needFill = true; spdlog::info("FileRepo: file list in db is empty, going to fill it"); } if (needFill) { - auto before = std::chrono::high_resolution_clock::now(); - - auto fileNames = listAllFileNames(progress); - const auto numFiles = fileNames.size(); - const auto numCores = std::thread::hardware_concurrency(); - const auto filesPerThread = numFiles / numCores; - std::vector threads; - for (size_t threadNum = 0; threadNum < numCores; ++threadNum) { - const size_t iStart = threadNum * filesPerThread; //inclusive - const size_t iEnd = (threadNum == numCores - 1) ? numFiles : iStart + filesPerThread;//exclusive - threads.emplace_back([this, - iStart, - iEnd, - &threadNum, - &fileNames, - progress]() { - std::string threadName = fmt::format("FileList filler #", threadNum); - util::setThreadName(threadName.c_str()); - std::vector entries; - for (auto fileName = fileNames.cbegin() + iStart; fileName < fileNames.cbegin() + iEnd; ++fileName) { - auto [type, name] = getTypeAndNameFromPathRelativeToBase(*fileName); - if (isBinaryFilename(name)) { - entries.push_back({*fileName, name, PSEUDO_CATEGORY_BINARY_FILE}); - } else { - auto ldrFile = addLdrFileWithContent(nullptr, name, "", type, getLibraryLdrFileContent(*fileName)); - - std::string category; - if (type == ldr::FileType::PART) { - const char& firstChar = ldrFile->metaInfo.title[0]; - if ((firstChar == '~' && ldrFile->metaInfo.title[1] != '|') || firstChar == '=' || firstChar == '_') { - category = PSEUDO_CATEGORY_HIDDEN_PART; - } else { - category = ldrFile->metaInfo.getCategory(); - } - } else if (type == ldr::FileType::SUBPART) { - category = PSEUDO_CATEGORY_SUBPART; - } else if (type == ldr::FileType::PRIMITIVE) { - category = PSEUDO_CATEGORY_PRIMITIVE; - } else if (type == ldr::FileType::MODEL) { - category = PSEUDO_CATEGORY_MODEL; - } - entries.push_back({name, ldrFile->metaInfo.title, category}); - } - if (iStart == 0) { - *progress = .4f * static_cast(entries.size()) / static_cast(iEnd) + .5f; - } + fillFileList([progress](float p) { *progress = p; }, currentHash); + } + } + std::string FileRepo::getLDConfigContentHash() { + auto currentLDConfigContent = getLibraryLdrFileContent(constants::LDRAW_CONFIG_FILE_NAME); + const auto currentHash = fmt::format("{:x}", adler32(1, reinterpret_cast(currentLDConfigContent.data()), currentLDConfigContent.size())); + return currentHash; + } + void FileRepo::fillFileList(std::function progress) { + fillFileList(progress, getLDConfigContentHash()); + } + void FileRepo::fillFileList(std::function progress, const std::string& currentLDConfigHash) { + auto before = std::chrono::high_resolution_clock::now(); + + db::fileList::deleteAllEntries(); + + auto fileNames = listAllFileNames(progress); + const auto numFiles = fileNames.size(); + const auto numCores = std::thread::hardware_concurrency(); + const auto filesPerThread = numFiles / numCores; + std::vector threads; + std::vector latestUpdates; + latestUpdates.resize(numCores); + for (size_t threadNum = 0; threadNum < numCores; ++threadNum) { + const size_t iStart = threadNum * filesPerThread; //inclusive + const size_t iEnd = (threadNum == numCores - 1) ? numFiles : iStart + filesPerThread;//exclusive + threads.emplace_back([this, + iStart, + iEnd, + threadNum, + &fileNames, + progress, + &latestUpdates]() { + std::string threadName = fmt::format("FileList filler #{}", threadNum); + util::setThreadName(threadName.c_str()); + std::vector entries; + std::string latestUpdate; + for (auto fileName = fileNames.cbegin() + iStart; fileName < fileNames.cbegin() + iEnd; ++fileName) { + auto [type, name] = getTypeAndNameFromPathRelativeToBase(*fileName); + if (isBinaryFilename(name)) { + entries.push_back({*fileName, name, PSEUDO_CATEGORY_BINARY_FILE}); + } else { + auto ldrFile = addLdrFileWithContent(nullptr, name, "", type, getLibraryLdrFileContent(*fileName)); + + std::string category; + if (type == FileType::PART) { + const char& firstChar = ldrFile->metaInfo.title[0]; + if ((firstChar == '~' && ldrFile->metaInfo.title[1] != '|') || firstChar == '=' || firstChar == '_') { + category = PSEUDO_CATEGORY_HIDDEN_PART; + } else { + category = ldrFile->metaInfo.getCategory(); } - db::fileList::put(entries); - if (iStart == 0) { - *progress = 1.0f; + } else if (type == FileType::SUBPART) { + category = PSEUDO_CATEGORY_SUBPART; + } else if (type == FileType::PRIMITIVE) { + category = PSEUDO_CATEGORY_PRIMITIVE; + } else if (type == FileType::MODEL) { + category = PSEUDO_CATEGORY_MODEL; + } else if (type == FileType::OTHER) { + category = PSEUDO_CATEGORY_OTHER; + } + entries.push_back({name, ldrFile->metaInfo.title, category}); + if (*fileName != constants::LDRAW_CONFIG_FILE_NAME) { + const auto update = ldrFile->metaInfo.getUpdateId(); + if (update > latestUpdate) { + latestUpdate = update; } - }); - } + } + } + if (iStart == 0) { + progress(.4f * static_cast(entries.size()) / static_cast(iEnd) + .5f); + } + } + db::fileList::put(entries); + if (iStart == 0) { + progress(1.f); + } + latestUpdates[threadNum] = latestUpdate; + }); + } - for (auto& t: threads) { - t.join(); - } + for (auto& t: threads) { + t.join(); + } - db::valueCache::set(db::valueCache::LAST_INDEX_LDCONFIG_HASH, currentHash); + db::valueCache::set(db::valueCache::LAST_INDEX_LDCONFIG_HASH, currentLDConfigHash); + db::valueCache::set(db::valueCache::CURRENT_PARTS_LIBRARY_VERSION, *std::max_element(latestUpdates.cbegin(), latestUpdates.cend())); - auto after = std::chrono::high_resolution_clock::now(); - auto durationMs = static_cast(std::chrono::duration_cast(after - before).count()) / 1000.0; + auto after = std::chrono::high_resolution_clock::now(); + auto durationMs = static_cast(std::chrono::duration_cast(after - before).count()) / 1000.0; - if (numFiles != static_cast(db::fileList::getSize())) { - spdlog::error("had {} fileNames, but only {} are in db", numFiles, db::fileList::getSize()); - } - spdlog::info("filled fileList in {} ms using {} threads. Size: {}", durationMs, numCores, numFiles); + if (numFiles != static_cast(db::fileList::getSize())) { + spdlog::error("had {} fileNames, but only {} are in db", numFiles, db::fileList::getSize()); } + spdlog::info("filled fileList in {} ms using {} threads. Size: {}", durationMs, numCores, numFiles); } oset_t FileRepo::getAllCategories() { @@ -590,6 +618,42 @@ namespace bricksim::ldr::file_repo { } return nullptr; } + void FileRepo::updateLibraryFiles(const std::filesystem::path& updatedFileDirectory, std::function progress, uint64_t estimatedFileCount) { + updateLibraryFilesImpl(updatedFileDirectory, [&progress, estimatedFileCount](int fileNo) { + progress(std::min(.5f, .5f * fileNo / estimatedFileCount)); + }); + refreshAfterUpdateOrReplaceLibrary(progress); + } + + void FileRepo::replaceLibraryFiles(const std::filesystem::path& replacementFileOrDirectory, std::function progress, uint64_t estimatedFileCount) { + replaceLibraryFilesImpl(replacementFileOrDirectory, [&progress, estimatedFileCount](int fileNo) { + progress(std::min(.5f, .5f * fileNo / estimatedFileCount)); + }); + refreshAfterUpdateOrReplaceLibrary(progress); + } + void FileRepo::refreshAfterUpdateOrReplaceLibrary(const std::function& progress) { + { + plLockWait("FileRepo::ldrFilesMtx"); + std::scoped_lock lg(ldrFilesMtx); + plLockScopeState("FileRepo::ldrFilesMtx", true); + ldrFiles.clear(); + } + { + plLockWait("FileRepo::binaryFilesMtx"); + std::scoped_lock lg(binaryFilesMtx); + plLockScopeState("FileRepo::binaryFilesMtx", true); + binaryFiles.clear(); + } + progress(.5f); + fillFileList([&progress](float fillFraction) { + progress(.5f + fillFraction * .5f); + }); + } + + std::string FileRepo::getVersion() const { + return db::valueCache::get(db::valueCache::CURRENT_PARTS_LIBRARY_VERSION).value_or(""); + } + FileRepo::~FileRepo() = default; } diff --git a/src/ldr/file_repo.h b/src/ldr/file_repo.h index 6f20a1ac..90c4b7ee 100644 --- a/src/ldr/file_repo.h +++ b/src/ldr/file_repo.h @@ -69,6 +69,7 @@ namespace bricksim::ldr::file_repo { std::filesystem::path& getBasePath(); static oset_t getAllCategories(); std::shared_ptr getNamespace(const std::string& name); + std::string getVersion() const; /** * @param type @@ -85,7 +86,7 @@ namespace bricksim::ldr::file_repo { * @param progress range from 0.0f to 0.5f * @return vector of file names relative to root of library */ - virtual std::vector listAllFileNames(float* progress) = 0; + virtual std::vector listAllFileNames(std::function progress) = 0; virtual std::string getLibraryLdrFileContent(FileType type, const std::string& name) = 0; virtual std::string getLibraryLdrFileContent(const std::string& nameRelativeToRoot) = 0; virtual std::shared_ptr getLibraryBinaryFileContent(const std::string& nameRelativeToRoot) = 0; @@ -98,6 +99,10 @@ namespace bricksim::ldr::file_repo { const std::shared_ptr& newNamespace, const std::string& newName); + void updateLibraryFiles(const std::filesystem::path& updatedFileDirectory, std::function progress, uint64_t estimatedFileCount); + virtual bool replaceLibraryFilesDirectlyFromZip() = 0; + void replaceLibraryFiles(const std::filesystem::path& replacementFileOrDirectory, std::function progress, uint64_t estimatedFileCount); + protected: static bool shouldFileBeSavedInList(const std::string& filename); /** @@ -106,8 +111,12 @@ namespace bricksim::ldr::file_repo { * @return */ static std::pair getTypeAndNameFromPathRelativeToBase(const std::string& pathRelativeToBase); + virtual void updateLibraryFilesImpl(const std::filesystem::path& updatedFileDirectory, std::function progress) = 0; + virtual void replaceLibraryFilesImpl(const std::filesystem::path& replacementFileOrDirectory, std::function progress) = 0; std::filesystem::path basePath; + void fillFileList(std::function progress); + void fillFileList(std::function progress, const std::string& currentLDConfigHash); private: uomap_t, uomap_t>>> ldrFiles; std::mutex ldrFilesMtx; @@ -118,6 +127,8 @@ namespace bricksim::ldr::file_repo { omap_t>> partsByCategory; static bool isLdrFilename(const std::string& filename); static bool isBinaryFilename(const std::string& filename); + std::string getLDConfigContentHash(); + void refreshAfterUpdateOrReplaceLibrary(const std::function& progress); }; FileRepo& get(); diff --git a/src/ldr/files.cpp b/src/ldr/files.cpp index 1d243882..178b3938 100644 --- a/src/ldr/files.cpp +++ b/src/ldr/files.cpp @@ -475,6 +475,20 @@ namespace bricksim::ldr { } return headerCategory.value(); } + const std::string_view FileMetaInfo::getUpdateId() const{ + auto pos = fileTypeLine.find("UPDATE"); + if (pos != std::string::npos) { + pos += strlen("UPDATE"); + pos = fileTypeLine.find_first_not_of(LDR_WHITESPACE, pos); + auto pos2 = fileTypeLine.find_first_of(LDR_WHITESPACE, pos+1); + if (pos2==std::string::npos) { + pos2 = fileTypeLine.length(); + } + return std::string_view(fileTypeLine).substr(pos, pos2); + } else { + return {}; + } + } namespace { const char* getFileTypeStr(const FileType type) { diff --git a/src/ldr/files.h b/src/ldr/files.h index 1455d607..7bcecca9 100644 --- a/src/ldr/files.h +++ b/src/ldr/files.h @@ -26,7 +26,8 @@ namespace bricksim::ldr { MPD_SUBFILE, PART, SUBPART, - PRIMITIVE + PRIMITIVE, + OTHER, }; enum class WindingOrder { @@ -60,6 +61,7 @@ namespace bricksim::ldr { bool addLine(const std::string& line); [[nodiscard]] const std::string& getCategory(); + [[nodiscard]] const std::string_view getUpdateId() const; private: bool firstLine = true; diff --git a/src/ldr/regular_file_repo.cpp b/src/ldr/regular_file_repo.cpp index b9e5973b..5d1b3997 100644 --- a/src/ldr/regular_file_repo.cpp +++ b/src/ldr/regular_file_repo.cpp @@ -8,14 +8,14 @@ namespace bricksim::ldr::file_repo { spdlog::warn("{} not found or not a directory", basePath.string()); return false; } - if (!std::filesystem::exists(basePath / "LDConfig.ldr")) { - spdlog::warn("LDConfig.ldr not found in {}, therefore it's not a valid ldraw library directory", basePath.string()); + if (!std::filesystem::exists(basePath / constants::LDRAW_CONFIG_FILE_NAME)) { + spdlog::warn("{} not found in {}, therefore it's not a valid ldraw library directory", constants::LDRAW_CONFIG_FILE_NAME, basePath.string()); return false; } return true; } - std::vector RegularFileRepo::listAllFileNames(float* progress) { + std::vector RegularFileRepo::listAllFileNames(std::function progress) { std::vector files; for (const auto& entry: std::filesystem::recursive_directory_iterator(basePath)) { auto path = util::withoutBasePath(entry.path(), basePath).string(); @@ -23,7 +23,7 @@ namespace bricksim::ldr::file_repo { if (shouldFileBeSavedInList(pathWithForwardSlash)) { files.push_back(pathWithForwardSlash); - *progress = std::min(1.f, .5f * static_cast(files.size()) / ESTIMATE_PART_LIBRARY_FILE_COUNT); + progress(std::min(1.f, .5f * static_cast(files.size()) / ESTIMATE_PART_LIBRARY_FILE_COUNT)); } } return files; @@ -49,4 +49,19 @@ namespace bricksim::ldr::file_repo { std::shared_ptr RegularFileRepo::getLibraryBinaryFileContent(const std::string& nameRelativeToRoot) { return std::make_shared(basePath / nameRelativeToRoot); } + void RegularFileRepo::updateLibraryFilesImpl(const std::filesystem::path& updatedFileDirectory, std::function progress) { + std::filesystem::copy(updatedFileDirectory, basePath, std::filesystem::copy_options::update_existing|std::filesystem::copy_options::recursive); + } + bool RegularFileRepo::replaceLibraryFilesDirectlyFromZip() { + return false; + } + void RegularFileRepo::replaceLibraryFilesImpl(const std::filesystem::path& replacementFileOrDirectory, std::function progress) { + if (!std::filesystem::is_directory(replacementFileOrDirectory)) { + throw std::invalid_argument("expected a directory as replacement path"); + } + std::filesystem::path tmpPath = basePath.string()+"_old"; + std::filesystem::rename(basePath, tmpPath); + std::filesystem::copy(replacementFileOrDirectory, basePath, std::filesystem::copy_options::recursive); + std::filesystem::remove_all(tmpPath); + } } diff --git a/src/ldr/regular_file_repo.h b/src/ldr/regular_file_repo.h index 5b881182..4db5e620 100644 --- a/src/ldr/regular_file_repo.h +++ b/src/ldr/regular_file_repo.h @@ -7,10 +7,15 @@ namespace bricksim::ldr::file_repo { public: explicit RegularFileRepo(const std::filesystem::path& basePath); static bool isValidBasePath(const std::filesystem::path& basePath); - std::vector listAllFileNames(float* progress) override; + std::vector listAllFileNames(std::function progress) override; ~RegularFileRepo() override; std::string getLibraryLdrFileContent(ldr::FileType type, const std::string& name) override; std::string getLibraryLdrFileContent(const std::string& nameRelativeToRoot) override; std::shared_ptr getLibraryBinaryFileContent(const std::string& nameRelativeToRoot) override; + bool replaceLibraryFilesDirectlyFromZip() override; + + protected: + void updateLibraryFilesImpl(const std::filesystem::path& updatedFileDirectory, std::function progress) override; + void replaceLibraryFilesImpl(const std::filesystem::path& replacementFileOrDirectory, std::function progress) override; }; } diff --git a/src/ldr/zip_file_repo.cpp b/src/ldr/zip_file_repo.cpp index 1e90ba5a..49b2b232 100644 --- a/src/ldr/zip_file_repo.cpp +++ b/src/ldr/zip_file_repo.cpp @@ -1,4 +1,5 @@ #include "zip_file_repo.h" +#include #include #include @@ -11,7 +12,7 @@ namespace bricksim::ldr::file_repo { return false; } int err; - zip_t* za = zip_open(basePath.string().c_str(), 0, &err); + zip_t* za = zip_open(basePath.string().c_str(), ZIP_RDONLY, &err); bool valid; if (za == nullptr) { zip_error_t zipError; @@ -21,9 +22,9 @@ namespace bricksim::ldr::file_repo { valid = false; } else if (zip_get_num_entries(za, 0) > 0) { const std::string rootFolder = getZipRootFolder(za); - const auto ldConfigPath = rootFolder + "LDConfig.ldr"; + const auto ldConfigPath = rootFolder + constants::LDRAW_CONFIG_FILE_NAME; if (zip_name_locate(za, ldConfigPath.c_str(), ZIP_FL_ENC_GUESS) == -1) { - spdlog::warn("LDConfig.ldr not in {}", basePath.string()); + spdlog::warn("{} not in {}", constants::LDRAW_CONFIG_FILE_NAME, basePath.string()); valid = false; } else { spdlog::debug("{} is a valid zip library.", basePath.string()); @@ -37,7 +38,7 @@ namespace bricksim::ldr::file_repo { return valid; } - std::vector ZipFileRepo::listAllFileNames(float* progress) { + std::vector ZipFileRepo::listAllFileNames(std::function progress) { std::scoped_lock lg(libzipLock); std::vector result; struct zip_stat fileStat{}; @@ -48,7 +49,7 @@ namespace bricksim::ldr::file_repo { const std::string nameString(fileStat.name + nameCutOff); if (shouldFileBeSavedInList(nameString)) { result.emplace_back(nameString); - *progress = std::min(1.0f, 0.5f * i / numEntries); + progress(std::min(1.0f, 0.5f * i / numEntries)); } } return result; @@ -59,23 +60,32 @@ namespace bricksim::ldr::file_repo { if (!isValidBasePath(basePath)) { throw std::invalid_argument("invalid basePath: " + basePath.string()); } - int errorCode; - zipArchive = zip_open(basePath.string().c_str(), 0, &errorCode); + openZipArchive(); + + if (zipArchive== nullptr) { + return; + } rootFolderName = getZipRootFolder(zipArchive); + } + void ZipFileRepo::openZipArchive() { + int errorCode; + zipArchive = zip_open(basePath.string().c_str(), 0, &errorCode); if (zipArchive == nullptr) { zip_error_t zipError; zip_error_init_with_code(&zipError, errorCode); spdlog::error("can't open zip library with path {}: {} {}", basePath.string(), errorCode, zip_error_strerror(&zipError)); zip_error_fini(&zipError); - return; } } - ZipFileRepo::~ZipFileRepo() { + void ZipFileRepo::closeZipArchive() const { zip_close(zipArchive); } + ZipFileRepo::~ZipFileRepo() { + closeZipArchive(); + } std::string ZipFileRepo::getLibraryLdrFileContent(ldr::FileType type, const std::string& name) { return getLibraryLdrFileContent(getPathRelativeToBase(type, name)); @@ -147,4 +157,48 @@ namespace bricksim::ldr::file_repo { return result; } + + void ZipFileRepo::updateLibraryFilesImpl(const std::filesystem::path& updatedFileDirectory, std::function progress) { + std::scoped_lock lg(libzipLock); + int currentFileNr = 0; + for (const auto& entry: std::filesystem::recursive_directory_iterator(updatedFileDirectory)) { + if (!entry.is_regular_file()) { + continue; + } + const auto relativePath = std::filesystem::relative(entry.path(), updatedFileDirectory).string(); + const auto zipFileName = rootFolderName + relativePath; + zip_int64_t existingFileIndex = zip_name_locate(zipArchive, zipFileName.c_str(), ZIP_FL_NOCASE); + if (existingFileIndex >= 0) { + if (zip_delete(zipArchive, existingFileIndex) != 0) { + throw std::invalid_argument("cannot delete existing file from zip"); + } + } + + auto* file = std::fopen(entry.path().c_str(), "rb"); + if (!file) { + throw std::invalid_argument(fmt::format("cannot read {} from file system", entry.path().string())); + } + + zip_error_t err; + auto* sourceFile = zip_source_filep_create(file, 0, ZIP_LENGTH_TO_END, &err); + if (sourceFile == nullptr || zip_file_add(zipArchive, zipFileName.c_str(), sourceFile, ZIP_FL_OVERWRITE | ZIP_FL_ENC_UTF_8) < 0) { + throw std::invalid_argument(fmt::format("cannot copy {} into zip file", entry.path().string())); + } + + progress(currentFileNr++); + } + closeZipArchive(); + openZipArchive(); + } + bool ZipFileRepo::replaceLibraryFilesDirectlyFromZip() { + return true; + } + void ZipFileRepo::replaceLibraryFilesImpl(const std::filesystem::path& replacementFileOrDirectory, std::function progress) { + if (!std::filesystem::is_regular_file(replacementFileOrDirectory)) { + throw std::invalid_argument("replacement file is not a regular file"); + } + closeZipArchive(); + std::filesystem::copy(replacementFileOrDirectory, basePath, std::filesystem::copy_options::overwrite_existing); + openZipArchive(); + } } diff --git a/src/ldr/zip_file_repo.h b/src/ldr/zip_file_repo.h index 9a97478f..d08a236b 100644 --- a/src/ldr/zip_file_repo.h +++ b/src/ldr/zip_file_repo.h @@ -9,11 +9,16 @@ namespace bricksim::ldr::file_repo { public: explicit ZipFileRepo(const std::filesystem::path& basePath); static bool isValidBasePath(const std::filesystem::path& basePath); - std::vector listAllFileNames(float* progress) override; + std::vector listAllFileNames(std::function progress) override; virtual ~ZipFileRepo(); std::string getLibraryLdrFileContent(ldr::FileType type, const std::string& name) override; std::string getLibraryLdrFileContent(const std::string& nameRelativeToRoot) override; std::shared_ptr getLibraryBinaryFileContent(const std::string& nameRelativeToRoot) override; + bool replaceLibraryFilesDirectlyFromZip() override; + + protected: + void updateLibraryFilesImpl(const std::filesystem::path& updatedFileDirectory, std::function progress) override; + void replaceLibraryFilesImpl(const std::filesystem::path& replacementFileOrDirectory, std::function progress) override; private: struct zip* zipArchive; @@ -21,5 +26,7 @@ namespace bricksim::ldr::file_repo { std::mutex libzipLock; static std::string getZipRootFolder(zip_t* archive);//including / at the end std::pair openFileByName(const std::string& nameRelativeToRoot); + void openZipArchive(); + void closeZipArchive() const; }; } diff --git a/src/lib/pugixml b/src/lib/pugixml new file mode 160000 index 00000000..30cc354f --- /dev/null +++ b/src/lib/pugixml @@ -0,0 +1 @@ +Subproject commit 30cc354fe37114ec7a0a4ed2192951690357c2ed diff --git a/src/utilities/CMakeLists.txt b/src/utilities/CMakeLists.txt index f7ea674a..788ab4dd 100644 --- a/src/utilities/CMakeLists.txt +++ b/src/utilities/CMakeLists.txt @@ -1,4 +1,6 @@ target_sources(BrickSimLib PRIVATE gears.cpp gears.h + ldraw_library_updater.cpp + ldraw_library_updater.h ) \ No newline at end of file diff --git a/src/utilities/ldraw_library_updater.cpp b/src/utilities/ldraw_library_updater.cpp new file mode 100644 index 00000000..36ea2f66 --- /dev/null +++ b/src/utilities/ldraw_library_updater.cpp @@ -0,0 +1,273 @@ +#include "ldraw_library_updater.h" +#include "../helpers/util.h" +#include "../ldr/config.h" +#include "../ldr/file_repo.h" +#include "../ldr/zip_file_repo.h" +#include "pugixml.hpp" +#include +#include +#include +#include + +namespace bricksim::ldraw_library_updater { + namespace { + std::unique_ptr state; + + static constexpr std::size_t ESTIMATED_UPDATE_FILE_SIZE = 45000; + + int updatesListDownloadProgress([[maybe_unused]] void* ptr, [[maybe_unused]] long downTotal, long downNow, [[maybe_unused]] long upTotal, [[maybe_unused]] long upNow) { + if (state != nullptr) { + state->initializingProgress = .1f + std::max(.9f, .9f * downNow / ESTIMATED_UPDATE_FILE_SIZE); + } + return 0; + } + Distribution parseDistribution(const pugi::xml_node& distNode) { + const auto* releaseDate = distNode.child("release_date").child_value(); + const auto releaseDateParsed = std::strlen(releaseDate) == 10 + ? stringutil::parseYYYY_MM_DD(std::string_view(releaseDate)) + : std::chrono::year_month_day(); + const auto* releaseId = distNode.child("release_id").child_value(); + const auto* url = distNode.child("url").child_value(); + std::size_t size; + sscanf(distNode.child("size").child_value(), "%zu", &size); + const auto* md5hex = distNode.child("md5_fingerprint").child_value(); + std::array md5{static_cast(0)}; + for (int i = 0; i < 16; ++i) { + for (int j = 0; j < 2; ++j) { + const auto c = md5hex[i * 2 + j]; + const std::byte nibble = static_cast(c <= '9' ? c - '0' : c - 'a' + 10); + md5[i] = (md5[i] << 4) | nibble; + } + } + return {releaseId, releaseDateParsed, url, size, md5}; + } + } + + UpdateState& getState() { + if (state == nullptr) { + state = std::make_unique(); + } + return *state; + } + void resetState() { + state = nullptr; + } + + UpdateFailedException::UpdateFailedException(const std::string& message, const std::source_location location) : + TaskFailedException(message, location) {} + + void UpdateState::initialize() { + if (step != Step::INACTIVE) { + throw std::invalid_argument("wrong time to call this method"); + } + step = Step::INITIALIZING; + initializingProgress = .05f; + findCurrentRelease(); + initializingProgress = .1f; + readUpdatesList(); + initializingProgress = 1.f; + step = Step::CHOOSE_ACTION; + } + void UpdateState::findCurrentRelease() { + currentReleaseDate = stringutil::parseYYYY_MM_DD(ldr::getConfig().getUpdateDate()); + currentReleaseId = ldr::file_repo::get().getVersion(); + } + void UpdateState::readUpdatesList() { + const auto [statusCode, xmlContent] = util::requestGET(constants::LDRAW_LIBRARY_UPDATES_XML_URL, false, 0, updatesListDownloadProgress); + if (statusCode < 200 | statusCode >= 300) { + throw UpdateFailedException(fmt::format("download of update info XML failed with status {}", statusCode)); + } + pugi::xml_document doc; + const auto parseResult = doc.load_buffer(xmlContent.c_str(), xmlContent.size()); + if (parseResult.status != pugi::status_ok) { + throw UpdateFailedException(fmt::format("error while parsing update info XML at char {}. {}", parseResult.offset, parseResult.description())); + } + for (auto distNode = doc.root().first_child().child("distribution"); distNode; distNode = distNode.next_sibling("distribution")) { + const auto* fileFormat = distNode.child("file_format").child_value(); + if (std::strcmp(fileFormat, "ZIP") == 0) { + const auto* releaseType = distNode.child("release_type").child_value(); + const auto distribution = parseDistribution(distNode); + + if (std::strcmp(releaseType, "UPDATE") == 0) { + if (distribution.id > currentReleaseId) { + incrementalUpdates.push_back(distribution); + } + } else if (std::strcmp(releaseType, "COMPLETE") == 0) { + completeDistribution = distribution; + } + } + } + } + std::size_t UpdateState::getIncrementalUpdateTotalSize() const { + return std::accumulate(incrementalUpdates.cbegin(), + incrementalUpdates.cend(), + std::size_t(0), + [](std::size_t x, const Distribution& dist) { return x + dist.size; }); + } + void UpdateState::doIncrementalUpdate() { + step = Step::UPDATE_INCREMENTAL; + spdlog::info("starting incremental LDraw library update"); + const auto tmpDirectory = std::filesystem::temp_directory_path() / "BrickSimIncrementalUpdate"; + std::filesystem::create_directory(tmpDirectory); + std::vector tmpZipFiles; + for (int i = 0; i < incrementalUpdates.size(); ++i) { + incrementalUpdateProgress.push_back(0); + auto& currentStepProgress = incrementalUpdateProgress.back(); + currentStepProgress = .01f; + + const auto dist = incrementalUpdates[i]; + tmpZipFiles.push_back(tmpDirectory / (dist.id + ".zip")); + const auto url = dist.url; + const auto downloadResult = util::downloadFile(url, tmpZipFiles.back(), [¤tStepProgress](std::size_t dlTotal, std::size_t dlNow, std::size_t ulTotal, std::size_t ulNow) { + currentStepProgress = dlTotal > 0 ? .5f * dlNow / dlTotal : .5f; + return 0; + }); + spdlog::debug("downloaded {} to {}", url, tmpZipFiles.back().string()); + if (downloadResult.httpCode < 200 || downloadResult.httpCode >= 300) { + throw UpdateFailedException(fmt::format("cannot download {}. HTTP Code was {}", url, downloadResult.httpCode)); + } + if (dist.size != downloadResult.contentLength) { + throw UpdateFailedException(fmt::format("Content Length mismatch for {} (expected={}, actual={}", url, dist.size, downloadResult.contentLength)); + } + const auto& expectedMD5 = dist.md5; + if (!std::all_of(expectedMD5.cbegin(), expectedMD5.cend(), [](std::byte x) { return x == std::byte{0}; })) { + const auto actualMD5 = util::md5(tmpZipFiles.back()); + if (expectedMD5 != actualMD5) { + throw UpdateFailedException(fmt::format("MD5 mismatch (expected={:02x}, actual={:02x}) for {}", fmt::join(expectedMD5, ""), fmt::join(actualMD5, ""), dist.url)); + } + } + } + + const auto mergedDirectory = tmpDirectory / "merged"; + std::filesystem::create_directory(mergedDirectory); + uoset_t extractedFiles; + + for (int i = incrementalUpdates.size() - 1; i >= 0; --i) { + const auto tmpZipPath = tmpZipFiles[i]; + const auto distr = incrementalUpdates[i]; + auto& currentStepProgress = incrementalUpdateProgress[i]; + + int err; + const auto archive = std::unique_ptr(zip_open(tmpZipPath.string().c_str(), ZIP_RDONLY, &err), zip_close); + if (archive == nullptr) { + zip_error_t zipError; + zip_error_init_with_code(&zipError, err); + auto msg = fmt::format("Cannot open .zip file: {} (downloaded from {})", zip_error_strerror(&zipError), distr.url); + zip_error_fini(&zipError); + throw UpdateFailedException(msg); + } + const auto numEntries = zip_get_num_entries(archive.get(), ZIP_FL_UNCHANGED); + for (zip_int64_t j = 0; j < numEntries; ++j) { + struct zip_stat stat; + if (zip_stat_index(archive.get(), j, ZIP_FL_UNCHANGED, &stat) != 0) { + throw UpdateFailedException(fmt::format("Cannot stat {}-th file in .zip (downloaded from {})", j, distr.url)); + } + const auto [_, notAlreadyExist] = extractedFiles.insert(stat.name); + if (!notAlreadyExist) { + continue; + } + auto entryPath = mergedDirectory / stat.name; + auto zfile = std::unique_ptr(zip_fopen_index(archive.get(), j, ZIP_FL_UNCHANGED), zip_fclose); + if (zfile == nullptr) { + throw UpdateFailedException(fmt::format("cannot open file {} inside .zip (downloaded from {})", stat.name, distr.url)); + } + std::filesystem::create_directories(entryPath.parent_path()); + std::ofstream outfile(entryPath, std::ios::binary); + if (!outfile) { + throw UpdateFailedException(fmt::format("cannot create temporary file {}", entryPath.string())); + } + + std::array buf; + zip_int64_t bytesRead; + while ((bytesRead = zip_fread(zfile.get(), buf.data(), buf.size())) > 0) { + outfile.write(buf.data(), bytesRead); + } + } + spdlog::debug("extracted {} entries from {}", numEntries, tmpZipPath.string()); + } + + std::filesystem::path sourceDir; + if (std::filesystem::is_regular_file(mergedDirectory / "ldraw" / constants::LDRAW_CONFIG_FILE_NAME)) { + sourceDir = mergedDirectory / "ldraw"; + } else if (std::filesystem::is_regular_file(mergedDirectory / constants::LDRAW_CONFIG_FILE_NAME)) { + sourceDir = mergedDirectory; + } else { + throw UpdateFailedException(fmt::format("cannot find {} in {} or {}", constants::LDRAW_CONFIG_FILE_NAME, (mergedDirectory / "ldraw").string(), mergedDirectory.string())); + } + + auto progressFunc = [this](float progress) { + const auto fraction = .5f * progress + .5f; + std::fill(incrementalUpdateProgress.begin(), incrementalUpdateProgress.end(), fraction); + }; + ldr::file_repo::get().updateLibraryFiles(sourceDir, progressFunc, extractedFiles.size()); + + std::filesystem::remove_all(tmpDirectory); + step = Step::FINISHED; + } + void UpdateState::doCompleteUpdate() { + step = Step::UPDATE_COMPLETE; + const auto tmpDirectory = std::filesystem::temp_directory_path() / "BrickSimCompleteUpdate"; + std::filesystem::create_directory(tmpDirectory); + const auto tmpFile = tmpDirectory / "complete.zip"; + util::downloadFile(completeDistribution->url, tmpFile, [this](std::size_t dlTotal, std::size_t dlNow, std::size_t ulTotal, std::size_t ulNow) { + completeUpdateProgress = dlTotal > 0 ? .5f * dlNow / dlTotal : 0.f; + return 0; + }); + + const auto needsExtract = !ldr::file_repo::get().replaceLibraryFilesDirectlyFromZip(); + + uint64_t numEntries; + std::filesystem::path fileOrDirectoryForReplacement; + { + int err; + const auto archive = std::unique_ptr(zip_open(tmpFile.string().c_str(), ZIP_RDONLY, &err), zip_close); + if (archive == nullptr) { + zip_error_t zipError; + zip_error_init_with_code(&zipError, err); + auto msg = fmt::format("Cannot open .zip file: {} (downloaded from {})", zip_error_strerror(&zipError), completeDistribution->url); + zip_error_fini(&zipError); + throw UpdateFailedException(msg); + } + numEntries = zip_get_num_entries(archive.get(), 0); + + if (needsExtract) { + const auto extractionDirectory = tmpDirectory / "extracted"; + for (zip_int64_t j = 0; j < numEntries; ++j) { + struct zip_stat stat; + if (zip_stat_index(archive.get(), j, ZIP_FL_UNCHANGED, &stat) != 0) { + throw UpdateFailedException(fmt::format("Cannot stat {}-th file in .zip (downloaded from {})", j, completeDistribution->url)); + } + if (stat.size<=0) { + continue; + } + auto entryPath = extractionDirectory / stat.name; + auto zfile = std::unique_ptr(zip_fopen_index(archive.get(), j, ZIP_FL_UNCHANGED), zip_fclose); + if (zfile == nullptr) { + throw UpdateFailedException(fmt::format("cannot open file {} inside .zip (downloaded from {})", stat.name, completeDistribution->url)); + } + std::filesystem::create_directories(entryPath.parent_path()); + std::ofstream outfile(entryPath, std::ios::binary); + if (!outfile) { + throw UpdateFailedException(fmt::format("cannot create temporary file {}", entryPath.string())); + } + + std::array buf; + zip_int64_t bytesRead; + while ((bytesRead = zip_fread(zfile.get(), buf.data(), buf.size())) > 0) { + outfile.write(buf.data(), bytesRead); + } + } + fileOrDirectoryForReplacement = std::filesystem::is_directory(extractionDirectory / "ldraw") + ? extractionDirectory / "ldraw" + : extractionDirectory; + } else { + fileOrDirectoryForReplacement = tmpFile; + } + } + + ldr::file_repo::get().replaceLibraryFiles(fileOrDirectoryForReplacement, [this](float progress){completeUpdateProgress = .5+.5*progress;}, numEntries); + + std::filesystem::remove_all(tmpDirectory); + step = Step::FINISHED; + } +} \ No newline at end of file diff --git a/src/utilities/ldraw_library_updater.h b/src/utilities/ldraw_library_updater.h new file mode 100644 index 00000000..12741424 --- /dev/null +++ b/src/utilities/ldraw_library_updater.h @@ -0,0 +1,61 @@ +#pragma once + +#include "../errors/exceptions.h" +#include "../helpers/stringutil.h" +#include "../ldr/config.h" +#include "pugixml.hpp" +#include + +namespace bricksim::ldraw_library_updater { + class UpdateFailedException : public errors::TaskFailedException { + public: + UpdateFailedException(const std::string& message, const std::source_location location = std::source_location::current()); + }; + + enum class Step { + INACTIVE, + INITIALIZING, + CHOOSE_ACTION, + UPDATE_INCREMENTAL, + UPDATE_COMPLETE, + FINISHED, + }; + struct Distribution { + std::string id; + std::chrono::year_month_day date; + std::string url; + std::size_t size; + std::array md5; + }; + + class UpdateState { + public: + Step step; + + float initializingProgress = 0.f; + + std::string currentReleaseId; + std::chrono::year_month_day currentReleaseDate; + + std::vector incrementalUpdates; + std::vector incrementalUpdateProgress; + + std::optional completeDistribution; + std::optional completeUpdateProgress; + + std::size_t getIncrementalUpdateTotalSize() const; + + void initialize(); + + void doIncrementalUpdate(); + + void doCompleteUpdate(); + + private: + void findCurrentRelease(); + void readUpdatesList(); + }; + + UpdateState& getState(); + void resetState(); +} \ No newline at end of file