diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index 6595eec2..b22719f7 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run clang-format style check - uses: jidicula/clang-format-action@v4.14.0 + uses: jidicula/clang-format-action@v4.16.0 with: clang-format-version: '18' check-path: ${{ matrix.path }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 49868ad9..bafd8ce1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,14 +95,14 @@ file( "src/dsf/mdt/*.cpp") include(FetchContent) -# Get rapidcsv +# Get csv-parser FetchContent_Declare( - rapidcsv - GIT_REPOSITORY https://github.com/d99kris/rapidcsv - GIT_TAG v8.89) -FetchContent_GetProperties(rapidcsv) -if(NOT rapidcsv_POPULATED) - FetchContent_MakeAvailable(rapidcsv) + csv-parser + GIT_REPOSITORY https://github.com/vincentlaucsb/csv-parser + GIT_TAG 2.4.1) +FetchContent_GetProperties(csv-parser) +if(NOT csv-parser_POPULATED) + FetchContent_MakeAvailable(csv-parser) endif() # Get spdlog set(SPDLOG_USE_STD_FORMAT @@ -131,7 +131,7 @@ endif() FetchContent_Declare( simdjson GIT_REPOSITORY https://github.com/simdjson/simdjson - GIT_TAG v4.2.3) + GIT_TAG v4.2.4) FetchContent_GetProperties(simdjson) if(NOT simdjson_POPULATED) FetchContent_MakeAvailable(simdjson) @@ -163,9 +163,9 @@ target_include_directories( dsf PUBLIC $ $) -target_include_directories(dsf PRIVATE ${rapidcsv_SOURCE_DIR}/src) +target_include_directories(dsf PRIVATE ${csv-parser_SOURCE_DIR}/single_include) -# Link other libraries - no csv dependency needed now +# Link other libraries target_link_libraries(dsf PUBLIC TBB::tbb SQLiteCpp PRIVATE simdjson::simdjson spdlog::spdlog) # Install dsf library @@ -243,8 +243,7 @@ if(DSF_TESTS) target_include_directories(${testname} PRIVATE ../src/ ../src/dsf/utility/TypeTraits/) target_include_directories( - ${testname} SYSTEM PRIVATE ${doctest_SOURCE_DIR}/doctest - ${rapidcsv_SOURCE_DIR}/src) + ${testname} SYSTEM PRIVATE ${doctest_SOURCE_DIR}/doctest) target_compile_definitions(${testname} PRIVATE SPDLOG_USE_STD_FORMAT) target_link_libraries(${testname} PRIVATE dsf TBB::tbb simdjson::simdjson spdlog::spdlog) # Put test binaries under build/tests so they are easy to find. Use diff --git a/README.md b/README.md index c64a58e1..b2eaeca9 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ [![Standard](https://img.shields.io/badge/C%2B%2B-20/23-blue.svg)](https://en.wikipedia.org/wiki/C%2B%2B#Standardization) [![TBB](https://img.shields.io/badge/TBB-2022.3.0-blue.svg)](https://github.com/oneapi-src/oneTBB) [![SPDLOG](https://img.shields.io/badge/spdlog-1.17.0-blue.svg)](https://github.com/gabime/spdlog) -[![CSV](https://img.shields.io/badge/rapidcsv-8.89-blue.svg)](https://github.com/d99kris/rapidcsv) -[![JSON](https://img.shields.io/badge/simdjson-4.2.1-blue.svg)](https://github.com/simdjson/simdjson) +[![CSV](https://img.shields.io/badge/csv-parser-4.3.1-blue.svg)](https://github.com/vincentlaucsb/csv-parser) +[![JSON](https://img.shields.io/badge/simdjson-4.2.4-blue.svg)](https://github.com/simdjson/simdjson) [![SQLite](https://img.shields.io/badge/SQLiteCpp-3.3.3-blue.svg)](https://github.com/SRombauts/SQLiteCpp) [![codecov](https://codecov.io/gh/physycom/DynamicalSystemFramework/graph/badge.svg?token=JV53J6IUJ3)](https://codecov.io/gh/physycom/DynamicalSystemFramework) @@ -36,7 +36,7 @@ print(dsf.__version__) ## Installation (from source) ### Requirements -The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog`, `rapidcsv` and `SQLiteCpp`. +The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog`, `csv-parser` and `SQLiteCpp`. To install requirements on Ubuntu: ```shell sudo apt install cmake libtbb-dev diff --git a/profiling/CMakeLists.txt b/profiling/CMakeLists.txt index 0475771a..9bae955f 100644 --- a/profiling/CMakeLists.txt +++ b/profiling/CMakeLists.txt @@ -18,14 +18,14 @@ string(APPEND CMAKE_CXX_FLAGS "-Wall -Wextra -Os") set(EXECUTABLE_OUTPUT_PATH ../) include(FetchContent) -# Get rapidcsv +# Get csv-parser FetchContent_Declare( - rapidcsv - GIT_REPOSITORY https://github.com/d99kris/rapidcsv - GIT_TAG v8.89) -FetchContent_GetProperties(rapidcsv) -if(NOT rapidcsv_POPULATED) - FetchContent_MakeAvailable(rapidcsv) + csv-parser + GIT_REPOSITORY https://github.com/vincentlaucsb/csv-parser + GIT_TAG 2.4.1) +FetchContent_GetProperties(csv-parser) +if(NOT csv-parser_POPULATED) + FetchContent_MakeAvailable(csv-parser) endif() file(GLOB SOURCES "../src/dsf/base/*.cpp" "../src/dsf/mobility/*.cpp" @@ -33,13 +33,13 @@ file(GLOB SOURCES "../src/dsf/base/*.cpp" "../src/dsf/mobility/*.cpp" # Define the executable add_executable(prof.out main.cpp ${SOURCES}) -target_include_directories(prof.out PRIVATE ../src/ ${rapidcsv_SOURCE_DIR}/src) +target_include_directories(prof.out PRIVATE ../src/ ${csv-parser_SOURCE_DIR}/single_include) target_link_libraries(prof.out PRIVATE TBB::tbb fmt::fmt spdlog::spdlog simdjson::simdjson) target_compile_options(prof.out PRIVATE -pg) target_link_options(prof.out PRIVATE -pg) add_executable(mem.out main.cpp ${SOURCES}) -target_include_directories(mem.out PRIVATE ../src/ ${rapidcsv_SOURCE_DIR}/src) +target_include_directories(mem.out PRIVATE ../src/ ${csv-parser_SOURCE_DIR}/single_include) target_link_libraries(mem.out PRIVATE TBB::tbb fmt::fmt spdlog::spdlog simdjson::simdjson) add_executable(parse_massif.out parse_massif.cpp) diff --git a/src/dsf/dsf.hpp b/src/dsf/dsf.hpp index f23e8685..6f4931a1 100644 --- a/src/dsf/dsf.hpp +++ b/src/dsf/dsf.hpp @@ -9,7 +9,7 @@ static constexpr uint8_t DSF_VERSION_MAJOR = 5; static constexpr uint8_t DSF_VERSION_MINOR = 0; -static constexpr uint8_t DSF_VERSION_PATCH = 1; +static constexpr uint8_t DSF_VERSION_PATCH = 2; static auto const DSF_VERSION = std::format("{}.{}.{}", DSF_VERSION_MAJOR, DSF_VERSION_MINOR, DSF_VERSION_PATCH); diff --git a/src/dsf/mdt/TrajectoryCollection.cpp b/src/dsf/mdt/TrajectoryCollection.cpp index 14a4b8cc..427c67f5 100644 --- a/src/dsf/mdt/TrajectoryCollection.cpp +++ b/src/dsf/mdt/TrajectoryCollection.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -58,8 +58,9 @@ namespace dsf::mdt { std::unordered_map const& column_mapping, char const sep, std::array const& bbox) { - rapidcsv::Document doc( - fileName, rapidcsv::LabelParams(0, -1), rapidcsv::SeparatorParams(sep)); + csv::CSVFormat csvFormat; + csvFormat.delimiter(sep); + csv::CSVReader reader(fileName, csvFormat); std::unordered_map column_names = { {"uid", "uid"}, {"timestamp", "timestamp"}, {"lat", "lat"}, {"lon", "lon"}}; @@ -71,14 +72,26 @@ namespace dsf::mdt { column_names[key] = value; } + std::vector uids; + std::vector timestamps; + std::vector lats; + std::vector lons; + for (auto& row : reader) { + uids.push_back(static_cast(row[column_names.at("uid")].get())); + timestamps.push_back( + static_cast(row[column_names.at("timestamp")].get())); + lats.push_back(row[column_names.at("lat")].get()); + lons.push_back(row[column_names.at("lon")].get()); + } + std::unordered_map< std::string, std::variant, std::vector, std::vector>> dataframe; - dataframe["uid"] = doc.GetColumn(column_names.at("uid")); - dataframe["timestamp"] = doc.GetColumn(column_names.at("timestamp")); - dataframe["lat"] = doc.GetColumn(column_names.at("lat")); - dataframe["lon"] = doc.GetColumn(column_names.at("lon")); + dataframe["uid"] = std::move(uids); + dataframe["timestamp"] = std::move(timestamps); + dataframe["lat"] = std::move(lats); + dataframe["lon"] = std::move(lons); *this = TrajectoryCollection(std::move(dataframe), bbox); } diff --git a/src/dsf/mobility/RoadNetwork.cpp b/src/dsf/mobility/RoadNetwork.cpp index 14fed803..aa8feaa2 100644 --- a/src/dsf/mobility/RoadNetwork.cpp +++ b/src/dsf/mobility/RoadNetwork.cpp @@ -5,7 +5,7 @@ #include #include -#include +#include #include #include @@ -20,10 +20,13 @@ namespace dsf::mobility { } } - void RoadNetwork::m_csvEdgesImporter(std::ifstream& file, const char separator) { - rapidcsv::Document csvReader( - file, rapidcsv::LabelParams(0, -1), rapidcsv::SeparatorParams(separator)); - auto const& colNames = csvReader.GetColumnNames(); + void RoadNetwork::m_csvEdgesImporter(const std::string& fileName, + const char separator) { + csv::CSVFormat format; + format.delimiter(separator); + csv::CSVReader reader(fileName, format); + + auto const& colNames = reader.get_col_names(); bool const bHasGeometry = (std::find(colNames.begin(), colNames.end(), "geometry") != colNames.end()); if (!bHasGeometry) { @@ -37,29 +40,27 @@ namespace dsf::mobility { (std::find(colNames.begin(), colNames.end(), "coilcode") != colNames.end()); bool const bHasCustomWeight = (std::find(colNames.begin(), colNames.end(), "customWeight") != colNames.end()); - // bool const bHasForbiddenTurns = (std::find(colNames.begin(), colNames.end(), "forbiddenTurns") != colNames.end()); - auto const rowCount = csvReader.GetRowCount(); - for (std::size_t i = 0; i < rowCount; ++i) { - auto const sourceId = csvReader.GetCell("source", i); - auto const targetId = csvReader.GetCell("target", i); + for (auto& row : reader) { + auto const sourceId = static_cast(row["source"].get()); + auto const targetId = static_cast(row["target"].get()); if (sourceId == targetId) { spdlog::warn("Skipping self-loop edge {}->{}", sourceId, targetId); continue; } - auto const streetId = csvReader.GetCell("id", i); - auto const dLength = csvReader.GetCell("length", i); - auto const name = csvReader.GetCell("name", i); - auto strType = csvReader.GetCell("type", i); + auto const streetId = static_cast(row["id"].get()); + auto const dLength = row["length"].get(); + auto const name = row["name"].get(); + auto strType = row["type"].get(); geometry::PolyLine polyline; if (bHasGeometry) { - polyline = geometry::PolyLine(csvReader.GetCell("geometry", i)); + polyline = geometry::PolyLine(row["geometry"].get()); } auto iLanes = 1; if (bHasLanes) { try { - iLanes = csvReader.GetCell("nlanes", i); + iLanes = row["nlanes"].get(); } catch (...) { spdlog::warn("Invalid number of lanes for edge {}->{}. Defaulting to 1 lane.", sourceId, @@ -70,7 +71,7 @@ namespace dsf::mobility { double dMaxSpeed = 30.; // Default to 30 km/h try { - dMaxSpeed = csvReader.GetCell("maxspeed", i); + dMaxSpeed = row["maxspeed"].get(); } catch (...) { spdlog::warn("Invalid maxspeed provided for edge {}->{}. Defaulting to 30 km/h.", sourceId, @@ -105,7 +106,7 @@ namespace dsf::mobility { } if (bHasCoilcode) { - auto strCoilCode = csvReader.GetCell("coilcode", i); + auto strCoilCode = row["coilcode"].get(); // Make this lowercase std::transform(strCoilCode.begin(), strCoilCode.end(), @@ -118,7 +119,7 @@ namespace dsf::mobility { } if (bHasCustomWeight) { try { - edge(streetId)->setWeight(csvReader.GetCell("customWeight", i)); + edge(streetId)->setWeight(row["customWeight"].get()); } catch (...) { spdlog::warn("Invalid custom weight for {}", *edge(streetId)); } @@ -146,19 +147,20 @@ namespace dsf::mobility { // } // } } - void RoadNetwork::m_csvNodePropertiesImporter(std::ifstream& file, + void RoadNetwork::m_csvNodePropertiesImporter(const std::string& fileName, const char separator) { - rapidcsv::Document csvReader( - file, rapidcsv::LabelParams(0, -1), rapidcsv::SeparatorParams(separator)); - auto const rowCount = csvReader.GetRowCount(); - for (std::size_t i = 0; i < rowCount; ++i) { - auto const nodeId = csvReader.GetCell("id", i); + csv::CSVFormat format; + format.delimiter(separator); + csv::CSVReader reader(fileName, format); + + for (auto& row : reader) { + auto const nodeId = static_cast(row["id"].get()); if (m_nodes.find(nodeId) == m_nodes.end()) { spdlog::warn("Node {} not found in the network. Skipping properties import.", nodeId); continue; } - auto strType = csvReader.GetCell("type", i); + auto strType = row["type"].get(); std::transform( strType.begin(), strType.end(), strType.begin(), [](unsigned char c) { return std::tolower(c); @@ -168,7 +170,7 @@ namespace dsf::mobility { } else if (strType.find("roundabout") != std::string::npos) { makeRoundabout(nodeId); } - auto const& strGeometry = csvReader.GetCell("geometry", i); + auto const& strGeometry = row["geometry"].get(); if (!strGeometry.empty()) { auto const point = geometry::Point(strGeometry); auto const& pNode{node(nodeId)}; diff --git a/src/dsf/mobility/RoadNetwork.hpp b/src/dsf/mobility/RoadNetwork.hpp index adbcaca7..94436c70 100644 --- a/src/dsf/mobility/RoadNetwork.hpp +++ b/src/dsf/mobility/RoadNetwork.hpp @@ -55,8 +55,9 @@ namespace dsf::mobility { void m_updateMaxAgentCapacity(); - void m_csvEdgesImporter(std::ifstream& file, const char separator = ';'); - void m_csvNodePropertiesImporter(std::ifstream& file, const char separator = ';'); + void m_csvEdgesImporter(const std::string& fileName, const char separator = ';'); + void m_csvNodePropertiesImporter(const std::string& fileName, + const char separator = ';'); void m_jsonEdgesImporter(std::ifstream& file); @@ -285,7 +286,7 @@ namespace dsf::mobility { switch (fileExtMap.at(fileExt)) { case FileExt::CSV: spdlog::debug("Importing nodes from CSV file: {}", fileName); - this->m_csvEdgesImporter(file, std::forward(args)...); + this->m_csvEdgesImporter(fileName, std::forward(args)...); break; case FileExt::GEOJSON: case FileExt::JSON: @@ -318,7 +319,7 @@ namespace dsf::mobility { switch (fileExtMap.at(fileExt)) { case FileExt::CSV: spdlog::debug("Importing node properties from CSV file: {}", fileName); - this->m_csvNodePropertiesImporter(file, std::forward(args)...); + this->m_csvNodePropertiesImporter(fileName, std::forward(args)...); break; case FileExt::JSON: case FileExt::GEOJSON: