Automatic LDraw part library update

.gitmodules

@@ -74,3 +74,6 @@
 [submodule "src/lib/Clipper2"]
 	path = src/lib/Clipper2
 	url = https://github.com/AngusJohnson/Clipper2
+[submodule "src/lib/pugixml"]
+	path = src/lib/pugixml
+	url = https://github.com/zeux/pugixml.git

CMakeLists.txt

@@ -307,6 +307,10 @@ option(CLIPPER2_TESTS "Build tests" OFF)
 add_subdirectory(src/lib/Clipper2/CPP)
 target_link_libraries(BrickSimLib PUBLIC Clipper2)
 
+#pugixml
+add_subdirectory(src/lib/pugixml)
+target_link_libraries(BrickSimLib PUBLIC pugixml-static)
+
 add_subdirectory(src)
 
 if (WIN32)

src/constant_data/constants.cpp

@@ -66,5 +66,4 @@ namespace bricksim::constants {
     ;
     const char* versionString = NUM(BRICKSIM_VERSION_MAJOR) "." NUM(BRICKSIM_VERSION_MINOR) "." NUM(BRICKSIM_VERSION_PATCH);
 
-    const char* LDRAW_LIBRARY_DOWNLOAD_URL = "https://library.ldraw.org/library/updates/complete.zip";
 }

src/constant_data/constants.h

@@ -37,7 +37,10 @@ namespace bricksim::constants {
     extern const uint16_t gitCommitCount;
     extern const char* gitCommitHash;
 
-    extern const char* LDRAW_LIBRARY_DOWNLOAD_URL;
+    static constexpr auto LDRAW_LIBRARY_DOWNLOAD_URL = "https://library.ldraw.org/library/updates/complete.zip";
+    static constexpr auto LDRAW_LIBRARY_UPDATES_XML_URL = "https://library.ldraw.org/updates?output=XML";
+
+    static constexpr auto LDRAW_CONFIG_FILE_NAME = "LDConfig.ldr";
 
     constexpr float pInf = std::numeric_limits<float>::infinity();
     constexpr float nInf = -pInf;

src/controller.cpp

@@ -350,12 +350,12 @@ namespace bricksim::controller {
             }
 
             std::array initSteps = {
-                    Task{"load color definitions", ldr::color_repo::initialize},
                     Task{"initialize shadow file repo", ldr::file_repo::initializeShadowFileRepo},
                     Task{"initialize file list", [](float* progress) {
                         ldr::file_repo::get().initialize(progress);
                         spdlog::info("File Repo base path is {}", ldr::file_repo::get().getBasePath().string());
                     }},
+                    Task{"load color definitions", ldr::color_repo::initialize},
                     Task{"initialize price guide provider", info_providers::price_guide::initialize},
                     Task{"initialize thumbnail generator", []() { thumbnailGenerator = std::make_shared<graphics::ThumbnailGenerator>(); }},
                     Task{"initialize BrickLink constants", info_providers::bricklink_constants::initialize},

src/db.cpp

@@ -264,6 +264,9 @@ namespace bricksim::db {
         }
 
         void put(const std::vector<Entry>& entries) {
+            if (entries.empty()) {
+                return;
+            }
             std::string command = "INSERT INTO files (name, title, category) VALUES ";
             std::string name;
             std::string title;

src/db.h

@@ -42,6 +42,7 @@ namespace bricksim::db {
 
     namespace valueCache {
         constexpr auto LAST_INDEX_LDCONFIG_HASH = "LAST_INDEX_LDCONFIG_HASH";
+        constexpr auto CURRENT_PARTS_LIBRARY_VERSION = "CURRENT_PARTS_LIBRARY_VERSION";
         template<typename T>
         std::optional<T> get(const char* key);
 

src/gui/main_menu_bar.cpp

@@ -27,6 +27,7 @@ namespace bricksim::gui {
         void drawUtilitiesMenu() {
             if (ImGui::BeginMenu("Utilities")) {
                 ImGui::MenuItem(windows::getName(windows::Id::GEAR_RATIO_CALCULATOR), "", windows::isVisible(windows::Id::GEAR_RATIO_CALCULATOR));
+                ImGui::MenuItem(windows::getName(windows::Id::LDRAW_LIBRARY_UPDATER), "", windows::isVisible(windows::Id::LDRAW_LIBRARY_UPDATER));
                 ImGui::EndMenu();
             }
         }

src/gui/windows/utilities/CMakeLists.txt

@@ -1,4 +1,6 @@
 target_sources(BrickSimLib PRIVATE
         window_gear_ratio_calculator.cpp
         window_gear_ratio_calculator.h
+        window_ldraw_library_updater.cpp
+        window_ldraw_library_updater.h
         )

src/gui/windows/utilities/window_gear_ratio_calculator.cpp

@@ -5,7 +5,7 @@
 
 #include "window_gear_ratio_calculator.h"
 
-namespace bricksim::gui::windows::tools::gear_ratio_calculator {
+namespace bricksim::gui::windows::utilities::gear_ratio_calculator {
     void draw(Data& data) {
         if (ImGui::Begin(data.name, &data.visible)) {
             collectWindowInfo(data.id);
@@ -91,7 +91,7 @@ namespace bricksim::gui::windows::tools::gear_ratio_calculator {
             ImGui::Separator();
             ImGui::Text("Gear Ratio: %ld:%ld", totalRatio.getA(), totalRatio.getB());
 
-            ImGui::End();
         }
+        ImGui::End();
     }
 }

src/gui/windows/utilities/window_gear_ratio_calculator.h

@@ -2,6 +2,6 @@
 
 #include "../windows.h"
 
-namespace bricksim::gui::windows::tools::gear_ratio_calculator {
+namespace bricksim::gui::windows::utilities::gear_ratio_calculator {
     void draw(Data& data);
 }

src/gui/windows/utilities/window_ldraw_library_updater.cpp

@@ -0,0 +1,101 @@
+#include "window_ldraw_library_updater.h"
+#include "../../../controller.h"
+#include "../../../utilities/ldraw_library_updater.h"
+#include "../../gui.h"
+
+namespace bricksim::gui::windows::utilities::ldraw_library_updater {
+    using namespace bricksim::ldraw_library_updater;
+
+    void draw(Data& data) {
+        const auto lastVisible = data.visible;
+        if (ImGui::Begin(data.name, &data.visible)) {
+            collectWindowInfo(data.id);
+            auto& state = getState();
+
+            switch (state.step) {
+                case Step::INACTIVE:
+                    controller::addBackgroundTask("Initialize LDraw Library Updater", []() { getState().initialize(); });
+                case Step::INITIALIZING:
+                {
+                    const auto progressMessage = fmt::format("Initializing {}%", state.initializingProgress * 100.f);
+                    ImGui::ProgressBar(state.initializingProgress, ImVec2(-FLT_MIN, 0), progressMessage.c_str());
+                    break;
+                }
+                case Step::CHOOSE_ACTION:
+                case Step::UPDATE_INCREMENTAL:
+                case Step::UPDATE_COMPLETE:
+                    ImGui::SeparatorText("Current Status");
+                    ImGui::Text("Library base path: %s", ldr::file_repo::get().getBasePath().string().c_str());
+                    ImGui::Text("Current LDConfig date: %d-%02d-%02d", (int)state.currentReleaseDate.year(), (unsigned int)state.currentReleaseDate.month(), (unsigned int)state.currentReleaseDate.day());
+                    ImGui::Text("Current release ID: %s", state.currentReleaseId.c_str());
+
+                    ImGui::SeparatorText("Incremental Update");
+                    if (state.incrementalUpdates.empty()) {
+                        ImGui::PushStyleColor(ImGuiCol_Text, color::GREEN);
+                        ImGui::Text(ICON_FA_CHECK " no incremental updates found. This usually means that your library is up to date.");
+                        ImGui::PopStyleColor();
+                    } else {
+                        if (ImGui::BeginTable("incrementalUpdates", state.step == Step::UPDATE_INCREMENTAL ? 4 : 3)) {
+                            ImGui::TableSetupColumn("ID");
+                            ImGui::TableSetupColumn("Date");
+                            ImGui::TableSetupColumn("Size");
+                            if (state.step == Step::UPDATE_INCREMENTAL) {
+                                ImGui::TableSetupColumn("Progress");
+                            }
+                            ImGui::TableHeadersRow();
+
+                            for (int i = 0; i < state.incrementalUpdates.size(); ++i) {
+                                const auto& dist = state.incrementalUpdates[i];
+
+                                ImGui::TableNextColumn();
+                                ImGui::Text("%s", dist.id.c_str());
+
+                                ImGui::TableNextColumn();
+                                ImGui::Text("%d-%02d-%02d", (int)dist.date.year(), (unsigned int)dist.date.month(), (unsigned int)dist.date.day());
+
+                                ImGui::TableNextColumn();
+                                ImGui::Text("%s", stringutil::formatBytesValue(dist.size).c_str());
+
+                                if (state.step == Step::UPDATE_INCREMENTAL) {
+                                    ImGui::TableNextColumn();
+                                    ImGui::ProgressBar(i < state.incrementalUpdateProgress.size() ? state.incrementalUpdateProgress[i] : 0.f);
+                                }
+                            }
+                            ImGui::EndTable();
+                        }
+                        ImGui::Text("Total download size for incremental update: %s", stringutil::formatBytesValue(state.getIncrementalUpdateTotalSize()).c_str());
+                        ImGui::BeginDisabled(state.step != Step::CHOOSE_ACTION);
+                        if (ImGui::Button("Do Incremental Update")) {
+                            controller::addBackgroundTask("Update LDraw Library", []() { getState().doIncrementalUpdate(); });
+                        }
+                        ImGui::EndDisabled();
+                    }
+
+                    ImGui::SeparatorText("Complete Update");
+                    if (state.completeDistribution) {
+                        ImGui::Text("Complete update download size: %s", stringutil::formatBytesValue(state.completeDistribution->size).c_str());
+                        ImGui::BeginDisabled(state.step != Step::CHOOSE_ACTION);
+                        if (ImGui::Button("Do Complete Update")) {
+                            controller::addBackgroundTask("Update LDraw Library", []() { getState().doCompleteUpdate(); });
+                        }
+                        ImGui::EndDisabled();
+                        if (state.step == Step::UPDATE_COMPLETE && state.completeUpdateProgress.has_value()) {
+                            ImGui::ProgressBar(*state.completeUpdateProgress);
+                        }
+                    } else {
+                        ImGui::Text("Complete distribution is not available");
+                    }
+
+                    break;
+                case Step::FINISHED:
+                    ImGui::Text("Update finished.");
+                    ImGui::Text("Your parts library is up to date");
+                    break;
+            }
+        }
+        if (lastVisible && !data.visible) {
+            resetState();
+        }
+        ImGui::End();
+    }
+}

src/gui/windows/utilities/window_ldraw_library_updater.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include "../windows.h"
+
+namespace bricksim::gui::windows::utilities::ldraw_library_updater {
+    void draw(Data& data);
+}

src/gui/windows/windows.cpp

@@ -8,6 +8,7 @@
 
 #include "../../metrics.h"
 #include "utilities/window_gear_ratio_calculator.h"
+#include "utilities/window_ldraw_library_updater.h"
 #include "window_about.h"
 #include "window_connection_visualization.h"
 #include "window_debug.h"
@@ -49,12 +50,13 @@ namespace bricksim::gui::windows {
                     {Id::IMGUI_DEMO, ICON_FA_IMAGE " ImGui Demo", false, drawImGuiDemo, noCleanup},
                     {Id::ORIENTATION_CUBE, ICON_FA_CUBE " Orientation Cube", true, orientation_cube::draw, noCleanup},
                     {Id::LOG, ICON_FA_LIST " Log", false, log::draw, noCleanup},
-                    {Id::GEAR_RATIO_CALCULATOR, ICON_FA_GEARS " Gear Ratio Calculator", false, tools::gear_ratio_calculator::draw, noCleanup},
+                    {Id::GEAR_RATIO_CALCULATOR, ICON_FA_GEARS " Gear Ratio Calculator", false, utilities::gear_ratio_calculator::draw, noCleanup},
                     {Id::MODEL_INFO, ICON_FA_INFO " Model Info", false, model_info::draw, noCleanup},
                     {Id::EDITOR_META_INFO, ICON_FA_RECEIPT " Meta-Info", false, editor_meta_info::draw, noCleanup},
                     {Id::LDRAW_FILE_INSPECTOR, ICON_FA_EYE " LDraw File Inspector", false, ldraw_file_inspector::draw, noCleanup},
                     {Id::TOOLBAR, ICON_FA_SCREWDRIVER_WRENCH " Toolbar", true, toolbar::draw, noCleanup},
                     {Id::CONNECTION_VISUALIZATION, ICON_FA_SHARE_NODES " Connection visualization", false, connection_visualization::draw, connection_visualization::cleanup},
+                    {Id::LDRAW_LIBRARY_UPDATER, ICON_FA_DOWNLOAD " LDraw Library Updater", false, utilities::ldraw_library_updater::draw, noCleanup},
             }};
 
     void drawAll() {

src/gui/windows/windows.h

@@ -22,6 +22,7 @@ namespace bricksim::gui::windows {
         LDRAW_FILE_INSPECTOR,
         TOOLBAR,
         CONNECTION_VISUALIZATION,
+        LDRAW_LIBRARY_UPDATER,
     };
 
     struct Data {

src/helpers/parts_library_downloader.cpp

@@ -8,8 +8,12 @@
 #include "../config/write.h"
 
 namespace bricksim::parts_library_downloader {
-    namespace {
-        int progressFunc([[maybe_unused]] void* clientp, long downloadTotal, long downloadNow, [[maybe_unused]] long uploadTotal, [[maybe_unused]] long uploadNow) {
+
+    void downloadPartsLibrary() {
+        status = Status::IN_PROGRESS;
+        spdlog::info("starting parts library download");
+        auto filePath = util::extendHomeDir("~/ldraw.zip");
+        auto [statusCode, content] = util::downloadFile(constants::LDRAW_LIBRARY_DOWNLOAD_URL, filePath, [](long downloadTotal, long downloadNow, [[maybe_unused]] long uploadTotal, [[maybe_unused]] long uploadNow) {
             const auto now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
             const auto msDiff = now - lastSpeedCalcTimestampMs;
             if (msDiff > 1000) {
@@ -22,14 +26,7 @@ namespace bricksim::parts_library_downloader {
             downNow = downloadNow;
             downTotal = downloadTotal;
             return shouldStop ? 1 : 0;
-        }
-    }
-
-    void downloadPartsLibrary() {
-        status = Status::IN_PROGRESS;
-        spdlog::info("starting parts library download");
-        auto filePath = util::extendHomeDir("~/ldraw.zip");
-        auto [statusCode, content] = util::downloadFile(constants::LDRAW_LIBRARY_DOWNLOAD_URL, filePath, progressFunc);
+        });
         if (statusCode < 200 || statusCode >= 300) {
             spdlog::error("parts library download failed. Error code: {}", statusCode);
             errorCode = statusCode;

src/helpers/stringutil.cpp

@@ -1,4 +1,5 @@
 #include "stringutil.h"
+#include "fast_float/fast_float.h"
 #include <algorithm>
 #include <array>
 #include <cmath>
@@ -333,4 +334,37 @@ namespace bricksim::stringutil {
         }
         return result;
     }
+    std::size_t findClosingQuote(std::string_view str, std::size_t start) {
+        std::size_t end = start;
+        do {
+            end = str.find('"', end);
+        } while (end!=std::string_view::npos && (end==0 || str[end-1]=='\\'));
+        return end;
+    }
+    std::chrono::year_month_day parseYYYY_MM_DD(std::string_view str) {
+        if (str.length() != std::strlen("0000-00-00")) {
+            throw std::invalid_argument("invalid size");
+        }
+        int year;
+        int month;
+        int day;
+        fast_float::from_chars(str.data()+0, str.data()+4, year);
+        fast_float::from_chars(str.data()+5, str.data()+7, month);
+        fast_float::from_chars(str.data()+8, str.data()+10, day);
+        return std::chrono::year_month_day(std::chrono::year(year),
+                                           std::chrono::month(month),
+                                           std::chrono::day(day));
+    }
+    std::chrono::year_month_day parseYYYY_MM(std::string_view str, int day) {
+        if (str.length() != std::strlen("0000-00")) {
+            throw std::invalid_argument("invalid size");
+        }
+        int year;
+        int month;
+        fast_float::from_chars(str.data()+0, str.data()+4, year);
+        fast_float::from_chars(str.data()+5, str.data()+7, month);
+        return std::chrono::year_month_day(std::chrono::year(year),
+                                           std::chrono::month(month),
+                                           std::chrono::day(day));
+    }
 }

src/helpers/stringutil.h

@@ -2,6 +2,7 @@
 #include "../gui/icons.h"
 #include "../lib/IconFontCppHeaders/IconsFontAwesome6.h"
 #include <algorithm>
+#include <chrono>
 #include <glm/glm.hpp>
 #include <spdlog/fmt/fmt.h>
 #include <string>
@@ -77,4 +78,12 @@ namespace bricksim::stringutil {
     }
 
     std::string removeIcons(std::string_view withIcons);
+
+    /**
+     * @return the index of the first occurrence of " that is >= start and not preceded by a backslash
+     */
+    std::size_t findClosingQuote(std::string_view str, std::size_t start = 0);
+
+    std::chrono::year_month_day parseYYYY_MM_DD(std::string_view str);
+    std::chrono::year_month_day parseYYYY_MM(std::string_view str, int day=1);
 }