From 348523b1fec4b488324d3101e6e3570d2008f3cb Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Tue, 16 Jun 2026 14:06:17 +0100 Subject: [PATCH 1/6] refactor(IW3): move asset overriding to a single place and add `StringTable` support --- codxe.vcxproj | 6 +- src/game/iw3/mp/components/assets.cpp | 398 ++++++++++++++++ .../mp/components/{scr_parser.h => assets.h} | 8 +- src/game/iw3/mp/components/cmds.cpp | 29 -- src/game/iw3/mp/components/mpsp.cpp | 52 -- src/game/iw3/mp/components/scr_parser.cpp | 148 ------ src/game/iw3/mp/main.cpp | 80 +--- src/third_party/aria_csv/csv_parser.hpp | 446 ++++++++++++++++++ 8 files changed, 854 insertions(+), 313 deletions(-) create mode 100644 src/game/iw3/mp/components/assets.cpp rename src/game/iw3/mp/components/{scr_parser.h => assets.h} (62%) delete mode 100644 src/game/iw3/mp/components/scr_parser.cpp create mode 100644 src/third_party/aria_csv/csv_parser.hpp diff --git a/codxe.vcxproj b/codxe.vcxproj index 59dc1627..ebb30d29 100644 --- a/codxe.vcxproj +++ b/codxe.vcxproj @@ -87,6 +87,7 @@ + @@ -102,7 +103,6 @@ - @@ -200,6 +200,8 @@ + + @@ -213,6 +215,7 @@ + @@ -223,7 +226,6 @@ - diff --git a/src/game/iw3/mp/components/assets.cpp b/src/game/iw3/mp/components/assets.cpp new file mode 100644 index 00000000..1d592f89 --- /dev/null +++ b/src/game/iw3/mp/components/assets.cpp @@ -0,0 +1,398 @@ +#include "pch.h" +#include "assets.h" +#include "third_party/aria_csv/csv_parser.hpp" +#include + +namespace iw3 +{ +namespace mp +{ + +namespace Assets +{ +namespace Utils +{ +std::string BuildAssetPath(const char *asset_name, const char *extension = nullptr) +{ + const std::string base_path = Config::GetModBasePath(); + if (base_path.empty() || asset_name == nullptr || asset_name[0] == '\0') + { + return ""; + } + + std::string relative_path = asset_name; + std::replace(relative_path.begin(), relative_path.end(), '/', '\\'); + + std::string path = base_path + "\\" + relative_path; + if (extension != nullptr) + { + path += extension; + } + + return path; +} + +bool ReadAssetOverride(const std::string &path, std::string &buffer) +{ + buffer.clear(); + + HANDLE file = CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) + { + return false; + } + + const DWORD file_size = GetFileSize(file, nullptr); + if (file_size == INVALID_FILE_SIZE || file_size > 0x7FFFFFFF) + { + CloseHandle(file); + return false; + } + + if (file_size > 0) + { + buffer.resize(file_size); + + DWORD bytes_read = 0; + if (!ReadFile(file, &buffer[0], file_size, &bytes_read, nullptr) || bytes_read != file_size) + { + CloseHandle(file); + buffer.clear(); + return false; + } + } + + CloseHandle(file); + return true; +} + +} // namespace Utils + +namespace MapEnts_ +{ +std::unordered_map> mapents_buffers; + +void _override(MapEnts *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + const std::string path = Utils::BuildAssetPath(asset->name, ".ents"); + if (path.empty()) + { + return; + } + + std::string buffer; + if (!Utils::ReadAssetOverride(path, buffer)) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding map ents '%s' from '%s'\n", asset->name, path.c_str()); + + auto itr = mapents_buffers.find(asset->name); + if (itr != mapents_buffers.end()) + { + mapents_buffers.erase(itr); + } + + mapents_buffers[asset->name] = make_unique(); + itr = mapents_buffers.find(asset->name); + std::string *mapents_buffer = itr->second.get(); + + mapents_buffer->assign(buffer); + + asset->entityString = const_cast(mapents_buffer->c_str()); + asset->numEntityChars = static_cast(mapents_buffer->length()); +} +} // namespace MapEnts_ + +namespace RawFile_ +{ +struct RawFileOverride +{ + std::string name; + std::string buffer; + RawFile asset; +}; + +std::unordered_map> rawfile_overrides; + +std::string BuildRawFilePath(const char *asset_name) +{ + return Utils::BuildAssetPath(asset_name); +} + +RawFile *FindOverride(const char *asset_name) +{ + if (asset_name == nullptr || asset_name[0] == '\0') + { + return nullptr; + } + + auto itr = rawfile_overrides.find(asset_name); + if (itr != rawfile_overrides.end()) + { + return itr->second ? &itr->second->asset : nullptr; + } + + const std::string path = BuildRawFilePath(asset_name); + if (path.empty()) + { + rawfile_overrides[asset_name] = nullptr; + return nullptr; + } + + std::string buffer; + if (!Utils::ReadAssetOverride(path, buffer)) + { + rawfile_overrides[asset_name] = nullptr; + return nullptr; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded rawfile '%s' from '%s'\n", asset_name, path.c_str()); + + rawfile_overrides[asset_name] = make_unique(); + itr = rawfile_overrides.find(asset_name); + RawFileOverride *rawfile_override = itr->second.get(); + + rawfile_override->name = asset_name; + rawfile_override->buffer.assign(buffer); + rawfile_override->asset.name = rawfile_override->name.c_str(); + rawfile_override->asset.len = static_cast(rawfile_override->buffer.length()); + rawfile_override->asset.buffer = rawfile_override->buffer.c_str(); + + return &rawfile_override->asset; +} + +void _override(RawFile *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + RawFile *override_asset = FindOverride(asset->name); + if (override_asset == nullptr) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding rawfile '%s'\n", asset->name); + asset->len = override_asset->len; + asset->buffer = override_asset->buffer; +} +} // namespace RawFile_ + +namespace StringTable_ +{ +struct StringTableOverride +{ + std::string name; + std::vector cells; + std::vector values; + StringTable asset; +}; + +std::unordered_map> stringtable_overrides; + +bool LoadCsv(const std::string &path, std::vector> &rows) +{ + rows.clear(); + + std::ifstream file(path.c_str(), std::ios::binary); + if (!file) + { + return false; + } + + try + { + aria::csv::CsvParser parser(file); + for (aria::csv::CsvParser::iterator itr = parser.begin(); itr != parser.end(); ++itr) + { + rows.push_back(*itr); + } + } + catch (...) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to parse stringtable csv '%s'\n", path.c_str()); + rows.clear(); + return false; + } + + return true; +} + +StringTable *FindOverride(const char *asset_name) +{ + if (asset_name == nullptr || asset_name[0] == '\0') + { + return nullptr; + } + + auto itr = stringtable_overrides.find(asset_name); + if (itr != stringtable_overrides.end()) + { + return itr->second ? &itr->second->asset : nullptr; + } + + const std::string path = Utils::BuildAssetPath(asset_name); + if (path.empty()) + { + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + std::vector> rows; + if (!LoadCsv(path, rows)) + { + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + size_t column_count = 0; + for (size_t row = 0; row < rows.size(); ++row) + { + if (rows[row].size() > column_count) + { + column_count = rows[row].size(); + } + } + + const size_t cell_count = rows.size() * column_count; + if (rows.size() > 0x7FFFFFFF || column_count > 0x7FFFFFFF || cell_count > 0x7FFFFFFF) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Stringtable csv too large '%s'\n", path.c_str()); + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded stringtable '%s' from '%s'\n", asset_name, path.c_str()); + + stringtable_overrides[asset_name] = make_unique(); + itr = stringtable_overrides.find(asset_name); + StringTableOverride *stringtable_override = itr->second.get(); + + stringtable_override->name = asset_name; + stringtable_override->cells.resize(cell_count); + stringtable_override->values.resize(cell_count); + + for (size_t row = 0; row < rows.size(); ++row) + { + for (size_t column = 0; column < column_count; ++column) + { + const size_t cell_index = row * column_count + column; + if (column < rows[row].size()) + { + stringtable_override->cells[cell_index] = rows[row][column]; + } + } + } + + for (size_t i = 0; i < cell_count; ++i) + { + stringtable_override->values[i] = stringtable_override->cells[i].c_str(); + } + + stringtable_override->asset.name = stringtable_override->name.c_str(); + stringtable_override->asset.columnCount = static_cast(column_count); + stringtable_override->asset.rowCount = static_cast(rows.size()); + stringtable_override->asset.values = + stringtable_override->values.empty() ? nullptr : &stringtable_override->values[0]; + + return &stringtable_override->asset; +} + +void _override(StringTable *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + StringTable *override_asset = FindOverride(asset->name); + if (override_asset == nullptr) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding stringtable '%s'\n", asset->name); + asset->columnCount = override_asset->columnCount; + asset->rowCount = override_asset->rowCount; + asset->values = override_asset->values; +} +} // namespace StringTable_ +} // namespace Assets + +Detour DB_LinkXAssetEntry_Detour; +Detour DB_FindXAssetHeader_Detour; + +XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) +{ + if (newEntry != nullptr) + { + switch (newEntry->asset.type) + { + case ASSET_TYPE_MAP_ENTS: + Assets::MapEnts_::_override(newEntry->asset.header.mapEnts); + break; + case ASSET_TYPE_RAWFILE: + Assets::RawFile_::_override(newEntry->asset.header.rawfile); + break; + case ASSET_TYPE_STRINGTABLE: + Assets::StringTable_::_override(newEntry->asset.header.stringTable); + break; + } + } + + return DB_LinkXAssetEntry_Detour.GetOriginal()(newEntry, allowOverride); +} + +XAssetHeader DB_FindXAssetHeader_Hook(const XAssetType type, const char *name) +{ + if (type == ASSET_TYPE_RAWFILE) + { + RawFile *rawfile = Assets::RawFile_::FindOverride(name); + if (rawfile != nullptr) + { + XAssetHeader header; + header.rawfile = rawfile; + return header; + } + } + else if (type == ASSET_TYPE_STRINGTABLE) + { + StringTable *stringtable = Assets::StringTable_::FindOverride(name); + if (stringtable != nullptr) + { + XAssetHeader header; + header.stringTable = stringtable; + return header; + } + } + + return DB_FindXAssetHeader_Detour.GetOriginal()(type, name); +} + +assets::assets() +{ + DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); + DB_LinkXAssetEntry_Detour.Install(); + + DB_FindXAssetHeader_Detour = Detour(DB_FindXAssetHeader, DB_FindXAssetHeader_Hook); + DB_FindXAssetHeader_Detour.Install(); +} + +assets::~assets() +{ + DB_FindXAssetHeader_Detour.Remove(); + + DB_LinkXAssetEntry_Detour.Remove(); +} +} // namespace mp +} // namespace iw3 diff --git a/src/game/iw3/mp/components/scr_parser.h b/src/game/iw3/mp/components/assets.h similarity index 62% rename from src/game/iw3/mp/components/scr_parser.h rename to src/game/iw3/mp/components/assets.h index 460c157d..67564c10 100644 --- a/src/game/iw3/mp/components/scr_parser.h +++ b/src/game/iw3/mp/components/assets.h @@ -6,15 +6,15 @@ namespace iw3 { namespace mp { -class scr_parser : public Module +class assets : public Module { public: - scr_parser(); - ~scr_parser(); + assets(); + ~assets(); const char *get_name() override { - return "scr_parser"; + return "assets"; }; }; } // namespace mp diff --git a/src/game/iw3/mp/components/cmds.cpp b/src/game/iw3/mp/components/cmds.cpp index e633b616..3daea549 100644 --- a/src/game/iw3/mp/components/cmds.cpp +++ b/src/game/iw3/mp/components/cmds.cpp @@ -111,46 +111,17 @@ void ClientCommand_Hook(int clientNum) ClientCommand_Detour.GetOriginal()(clientNum); } -Detour Cmd_ExecFromFastFile_Detour; - -bool Cmd_ExecFromFastFile_Hook(int localClientNum, int controllerIndex, const char *filename) -{ - auto callOriginal = [&]() - { - return Cmd_ExecFromFastFile_Detour.GetOriginal()(localClientNum, - controllerIndex, filename); - }; - - // Check if mod is active - std::string modBasePath = Config::GetModBasePath(); - if (modBasePath.empty()) - return callOriginal(); - - std::string contents = filesystem::read_file_to_string(modBasePath + "\\" + filename); - if (contents.empty()) - return callOriginal(); - - Com_Printf(CON_CHANNEL_SYSTEM, "execing %s from raw:\\\n", filename); - Cbuf_ExecuteBuffer(localClientNum, controllerIndex, contents.c_str()); - return true; -} - cmds::cmds() { ClientCommand_Detour = Detour(ClientCommand, ClientCommand_Hook); ClientCommand_Detour.Install(); - Cmd_ExecFromFastFile_Detour = Detour(Cmd_ExecFromFastFile, Cmd_ExecFromFastFile_Hook); - Cmd_ExecFromFastFile_Detour.Install(); - command::add("dumpraw", Cmd_Dumpraw_f); } cmds::~cmds() { ClientCommand_Detour.Remove(); - - Cmd_ExecFromFastFile_Detour.Remove(); } } // namespace mp } // namespace iw3 diff --git a/src/game/iw3/mp/components/mpsp.cpp b/src/game/iw3/mp/components/mpsp.cpp index 31747abf..90394e95 100644 --- a/src/game/iw3/mp/components/mpsp.cpp +++ b/src/game/iw3/mp/components/mpsp.cpp @@ -438,52 +438,6 @@ const ZoneOverride ZONE_OVERRIDES[] = { int g_zoneOverrideIndex = -1; -Detour DB_LinkXAssetEntry_Detour; -XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) -{ - XAsset xasset; - xasset.type = newEntry->asset.type; - xasset.header = newEntry->asset.header; - - if (mpsp::is_sp_map) - { - switch (newEntry->asset.type) - { - case ASSET_TYPE_MAP_ENTS: - { - Asset::MapEnts_::override_(newEntry->asset.header.mapEnts); - break; - } - case ASSET_TYPE_RAWFILE: - { - Asset::RawFile_::override_(newEntry->asset.header.rawfile); - break; - } - case ASSET_TYPE_GAMEWORLD_SP: - { - newEntry->asset.type = ASSET_TYPE_GAMEWORLD_MP; - break; - } - // Hijack the reference asset ',' mechanism to avoid reaching asset limits. - case ASSET_TYPE_WEAPON: - { - static const std::string weapon_default_reference_name = - std::string(",") + g_defaultAssetName[ASSET_TYPE_WEAPON]; - DB_SetXAssetName(&xasset, weapon_default_reference_name.c_str()); - break; - } - case ASSET_TYPE_FX: - { - static const std::string fx_default_reference_name = std::string(",") + g_defaultAssetName[ASSET_TYPE_FX]; - DB_SetXAssetName(&xasset, fx_default_reference_name.c_str()); - break; - } - } - } - - return DB_LinkXAssetEntry_Detour.GetOriginal()(newEntry, allowOverride); -} - Detour Com_sprintf_Detour; int Com_sprintf_Hook(char *dest, unsigned int size, const char *fmt...) { @@ -778,10 +732,6 @@ mpsp::mpsp() DB_AuthLoad_Inflate_Detour = Detour(DB_AuthLoad_Inflate, DB_AuthLoad_Inflate_Hook); DB_AuthLoad_Inflate_Detour.Install(); - // Rewrite some assets before linking - DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); - DB_LinkXAssetEntry_Detour.Install(); - Load_XAssetArrayCustom_Detour = Detour(Load_XAssetArrayCustom, Load_XAssetArrayCustom_Stub); Load_XAssetArrayCustom_Detour.Install(); @@ -799,8 +749,6 @@ mpsp::~mpsp() DB_AuthLoad_Inflate_Detour.Remove(); - DB_LinkXAssetEntry_Detour.Remove(); - Load_XAssetArrayCustom_Detour.Remove(); SV_AddEntitiesVisibleFromPoint_Detour.Remove(); diff --git a/src/game/iw3/mp/components/scr_parser.cpp b/src/game/iw3/mp/components/scr_parser.cpp deleted file mode 100644 index e7d6dbc0..00000000 --- a/src/game/iw3/mp/components/scr_parser.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include "pch.h" -#include "scr_parser.h" - -namespace iw3 -{ -namespace mp -{ -namespace -{ -const size_t MAX_SCRIPT_PATH = 512; - -void NormalizePath(char *path) -{ - if (path == nullptr) - return; - - for (char *cursor = path; *cursor != '\0'; ++cursor) - { - if (*cursor == '/') - *cursor = '\\'; - } -} - -bool BuildScriptPath(char *path, size_t path_size, const char *base_path, const char *script_path) -{ - if (path == nullptr || path_size == 0 || base_path == nullptr || base_path[0] == '\0' || script_path == nullptr || - script_path[0] == '\0') - { - return false; - } - - const int written = _snprintf_s(path, path_size, _TRUNCATE, "%s\\%s", base_path, script_path); - path[path_size - 1] = '\0'; - - if (written < 0 || static_cast(written) >= path_size) - return false; - - NormalizePath(path); - return true; -} - -char *ReadFileToGameTempBuffer(const char *path) -{ - HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, nullptr); - if (file == INVALID_HANDLE_VALUE) - return nullptr; - - const DWORD file_size = GetFileSize(file, nullptr); - if (file_size == INVALID_FILE_SIZE || file_size == 0) - { - CloseHandle(file); - return nullptr; - } - - char *buffer = static_cast(Hunk_AllocateTempMemoryHighInternal(file_size + 1)); - if (buffer == nullptr) - { - CloseHandle(file); - return nullptr; - } - - DWORD bytes_read = 0; - if (!ReadFile(file, buffer, file_size, &bytes_read, nullptr) || bytes_read != file_size) - { - CloseHandle(file); - return nullptr; - } - - CloseHandle(file); - buffer[file_size] = '\0'; - return buffer; -} - -void WriteScriptDump(const char *script_path, const char *contents) -{ - if (script_path == nullptr || contents == nullptr) - return; - - char dump_path[MAX_SCRIPT_PATH]; - if (!BuildScriptPath(dump_path, sizeof(dump_path), DUMP_DIR, script_path)) - return; - - char dir_path[MAX_SCRIPT_PATH]; - strncpy(dir_path, dump_path, sizeof(dir_path) - 1); - dir_path[sizeof(dir_path) - 1] = '\0'; - - char *last_slash = strrchr(dir_path, '\\'); - if (last_slash != nullptr) - { - *last_slash = '\0'; - filesystem::create_nested_dirs(dir_path); - } - - HANDLE file = CreateFileA(dump_path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); - if (file == INVALID_HANDLE_VALUE) - return; - - DWORD bytes_written = 0; - const DWORD content_size = static_cast(std::strlen(contents)); - WriteFile(file, contents, content_size, &bytes_written, nullptr); - CloseHandle(file); -} -} // namespace - -Detour Scr_AddSourceBuffer_Detour; - -char *Scr_AddSourceBuffer_Hook(const char *filename, const char *extFilename, const char *codePos, bool archive) -{ - auto callOriginal = [&]() - { - return Scr_AddSourceBuffer_Detour.GetOriginal()(filename, extFilename, codePos, - archive); - }; - - if (Config::dump_rawfile) - { - char *contents = callOriginal(); - if (contents != nullptr) - WriteScriptDump(extFilename, contents); - - return contents; - } - - char override_path[MAX_SCRIPT_PATH]; - if (!BuildScriptPath(override_path, sizeof(override_path), Config::GetModBasePathCStr(), extFilename)) - return callOriginal(); - - char *buffer = ReadFileToGameTempBuffer(override_path); - if (buffer == nullptr) - return callOriginal(); - - DbgPrint("GSCLoader: Loaded override script: %s\n", override_path); - return buffer; -} - -scr_parser::scr_parser() -{ - Scr_AddSourceBuffer_Detour = Detour(Scr_AddSourceBuffer, Scr_AddSourceBuffer_Hook); - Scr_AddSourceBuffer_Detour.Install(); -} - -scr_parser::~scr_parser() -{ - Scr_AddSourceBuffer_Detour.Remove(); -} -} // namespace mp -} // namespace iw3 diff --git a/src/game/iw3/mp/main.cpp b/src/game/iw3/mp/main.cpp index 7205c8e3..b1fd4e51 100644 --- a/src/game/iw3/mp/main.cpp +++ b/src/game/iw3/mp/main.cpp @@ -1,4 +1,5 @@ #include "pch.h" +#include "components/assets.h" #include "components/cg.h" #include "components/cj_tas.h" #include "components/clipmap.h" @@ -14,7 +15,6 @@ #include "components/image_loader.h" #include "components/mpsp.h" #include "components/pm.h" -#include "components/scr_parser.h" #include "components/stats.h" #include "components/sv_bots.h" #include "common/config.h" @@ -24,78 +24,6 @@ namespace iw3 { namespace mp { -Detour Load_MapEntsPtr_Detour; - -void Load_MapEntsPtr_Hook() -{ - // TODO: don't write null byte to file - // and add null byte to entityString when reading from file - - DbgPrint("Load_MapEntsPtr_Hook\n"); - - // TODO: write comment what this is *** - // Get pointer to pointer stored at 0x82475914 - MapEnts **varMapEntsPtr = *(MapEnts ***)0x82475914; - - Load_MapEntsPtr_Detour.GetOriginal()(); - - // Validate pointer before dereferencing - if (varMapEntsPtr && *varMapEntsPtr) - { - MapEnts *mapEnts = *varMapEntsPtr; - - // Write stock map ents to disk - std::string file_path = DUMP_DIR; - file_path += std::string("\\") + mapEnts->name; - file_path += ".ents"; // iw4x naming convention - std::replace(file_path.begin(), file_path.end(), '/', '\\'); // Replace forward slashes with backslashes - filesystem::write_file_to_disk(file_path.c_str(), mapEnts->entityString, mapEnts->numEntityChars); - - // Use new ConfigModule API - // Load map ents from file - // Path to check for existing entity file - std::string raw_file_path = Config::GetModBasePath(); - - raw_file_path += std::string("\\") + mapEnts->name; - raw_file_path += ".ents"; // IW4x naming convention - std::replace(raw_file_path.begin(), raw_file_path.end(), '/', '\\'); // Replace forward slashes with backslashes - - // If the file exists, replace entityString - if (filesystem::file_exists(raw_file_path)) - { - DbgPrint("Found entity file: %s\n", raw_file_path.c_str()); - std::string new_entity_string = filesystem::read_file_to_string(raw_file_path); - if (!new_entity_string.empty()) - { - // Allocate new memory and copy the data - size_t new_size = new_entity_string.size() + 1; // Include null terminator - char *new_memory = static_cast(malloc(new_size)); - - if (new_memory) - { - memcpy(new_memory, new_entity_string.c_str(), new_size); // Copy with null terminator - - // Update the entityString pointer to point to the new memory - mapEnts->entityString = new_memory; - - // // Update numEntityChars - // mapEnts->numEntityChars = static_cast(new_entity_string.size()); // unnecessary - - DbgPrint("Replaced entityString from file: %s\n", raw_file_path.c_str()); - } - else - { - DbgPrint("Failed to allocate memory for entityString replacement.\n"); - } - } - } - } - else - { - DbgPrint("Hooked Load_MapEntsPtr: varMapEntsPtr is NULL or invalid.\n"); - } -} - /** * Patch out the signature checks used during fastfile authentication. * Signature data must still be present in @@ -125,6 +53,7 @@ IW3_MP_Plugin::IW3_MP_Plugin() RegisterModule(new command()); RegisterModule(new cg()); + RegisterModule(new assets()); RegisterModule(new cj_tas()); RegisterModule(new clipmap()); RegisterModule(new cmds()); @@ -137,17 +66,12 @@ IW3_MP_Plugin::IW3_MP_Plugin() RegisterModule(new image_loader()); RegisterModule(new pm()); RegisterModule(new mpsp()); - RegisterModule(new scr_parser()); RegisterModule(new stats()); RegisterModule(new sv_bots()); - - Load_MapEntsPtr_Detour = Detour(Load_MapEntsPtr, Load_MapEntsPtr_Hook); - Load_MapEntsPtr_Detour.Install(); } IW3_MP_Plugin::~IW3_MP_Plugin() { - Load_MapEntsPtr_Detour.Remove(); } } // namespace mp diff --git a/src/third_party/aria_csv/csv_parser.hpp b/src/third_party/aria_csv/csv_parser.hpp new file mode 100644 index 00000000..b589d435 --- /dev/null +++ b/src/third_party/aria_csv/csv_parser.hpp @@ -0,0 +1,446 @@ +#ifndef ARIA_CSV_H +#define ARIA_CSV_H + +// Local vendored copy modified for the Xbox 360 compiler used by this project. +// https://github.com/AriaFallah/csv-parser + +#include +#include +#include +#include +#include + +namespace aria +{ +namespace csv +{ + +namespace Term +{ +enum Enum +{ + CRLF = -2 +}; +} // namespace Term + +namespace FieldType +{ +enum Enum +{ + DATA, + ROW_END, + CSV_END +}; +} // namespace FieldType + +typedef std::vector> CSV; + +// Checking for '\n', '\r', and '\r\n' by default +inline bool operator==(const char c, const Term::Enum t) +{ + switch (t) + { + case Term::CRLF: + return c == '\r' || c == '\n'; + default: + return static_cast(t) == c; + } +} + +inline bool operator!=(const char c, const Term::Enum t) +{ + return !(c == t); +} + +// Wraps returned fields so we can also indicate +// that we hit row endings or the end of the csv itself +struct Field +{ + explicit Field(FieldType::Enum t) : type(t), data(nullptr) + { + } + explicit Field(const std::string &str) : type(FieldType::DATA), data(&str) + { + } + + FieldType::Enum type; + const std::string *data; +}; + +// Reads and parses lines from a csv file +class CsvParser +{ + private: + // CSV state for state machine + struct State + { + enum Enum + { + START_OF_FIELD, + IN_FIELD, + IN_QUOTED_FIELD, + IN_ESCAPED_QUOTE, + END_OF_ROW, + EMPTY + }; + }; + State::Enum m_state; + + // Configurable attributes + char m_quote; + char m_delimiter; + Term::Enum m_terminator; + std::istream &m_input; + + // Buffer capacities + static const int FIELDBUF_CAP = 1024; + static const int INPUTBUF_CAP = 1024 * 128; + + // Buffers + std::string m_fieldbuf; + std::unique_ptr m_inputbuf; + + // Misc + bool m_eof; + size_t m_cursor; + size_t m_inputbuf_size; + std::streamoff m_scanposition; + + public: + // Creates the CSV parser which by default, splits on commas, + // uses quotes to escape, and handles CSV files that end in either + // '\r', '\n', or '\r\n'. + explicit CsvParser(std::istream &input) + : m_state(State::START_OF_FIELD), m_quote('"'), m_delimiter(','), m_terminator(Term::CRLF), m_input(input), + m_inputbuf(new char[INPUTBUF_CAP]()), m_eof(false), m_cursor(INPUTBUF_CAP), m_inputbuf_size(INPUTBUF_CAP), + m_scanposition(-INPUTBUF_CAP) + { + // Reserve space upfront to improve performance + m_fieldbuf.reserve(FIELDBUF_CAP); + if (!m_input.good()) + { + throw std::runtime_error("Something is wrong with input stream"); + } + } + + // Change the quote character + CsvParser &"e(char c) + { + m_quote = c; + return std::move(*this); + } + + // Change the delimiter character + CsvParser &&delimiter(char c) + { + m_delimiter = c; + return std::move(*this); + } + + // Change the terminator character + CsvParser &&terminator(char c) + { + m_terminator = static_cast(c); + return std::move(*this); + } + + // The parser is in the empty state when there are + // no more tokens left to read from the input buffer + bool empty() + { + return m_state == State::EMPTY; + } + + // Not the actual position in the stream (its buffered) just the + // position up to last availiable token + std::streamoff position() const + { + return m_scanposition + static_cast(m_cursor); + } + + // Reads a single field from the CSV + Field next_field() + { + if (empty()) + { + return Field(FieldType::CSV_END); + } + m_fieldbuf.clear(); + + // This loop runs until either the parser has + // read a full field or until there's no tokens left to read + for (;;) + { + char *maybe_token = top_token(); + + // If we're out of tokens to read return whatever's left in the + // field and row buffers. If there's nothing left, return null. + if (!maybe_token) + { + m_state = State::EMPTY; + return !m_fieldbuf.empty() ? Field(m_fieldbuf) : Field(FieldType::CSV_END); + } + + // Parsing the CSV is done using a finite state machine + char c = *maybe_token; + switch (m_state) + { + case State::START_OF_FIELD: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_quote) + { + m_state = State::IN_QUOTED_FIELD; + } + else if (c == m_delimiter) + { + return Field(m_fieldbuf); + } + else + { + m_state = State::IN_FIELD; + m_fieldbuf += c; + } + + break; + + case State::IN_FIELD: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_delimiter) + { + m_state = State::START_OF_FIELD; + return Field(m_fieldbuf); + } + else + { + m_fieldbuf += c; + } + + break; + + case State::IN_QUOTED_FIELD: + m_cursor++; + if (c == m_quote) + { + m_state = State::IN_ESCAPED_QUOTE; + } + else + { + m_fieldbuf += c; + } + + break; + + case State::IN_ESCAPED_QUOTE: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_quote) + { + m_state = State::IN_QUOTED_FIELD; + m_fieldbuf += c; + } + else if (c == m_delimiter) + { + m_state = State::START_OF_FIELD; + return Field(m_fieldbuf); + } + else + { + m_state = State::IN_FIELD; + m_fieldbuf += c; + } + + break; + + case State::END_OF_ROW: + m_state = State::START_OF_FIELD; + return Field(FieldType::ROW_END); + + case State::EMPTY: + throw std::logic_error("You goofed"); + } + } + } + + private: + // When the parser hits the end of a line it needs + // to check the special case of '\r\n' as a terminator. + // If it finds that the previous token was a '\r', and + // the next token will be a '\n', it skips the '\n'. + void handle_crlf(const char c) + { + if (m_terminator != Term::CRLF || c != '\r') + { + return; + } + + char *token = top_token(); + if (token && *token == '\n') + { + m_cursor++; + } + } + + // Pulls the next token from the input buffer, but does not move + // the cursor forward. If the stream is empty and the input buffer + // is also empty return a nullptr. + char *top_token() + { + // Return null if there's nothing left to read + if (m_eof && m_cursor == m_inputbuf_size) + { + return nullptr; + } + + // Refill the input buffer if it's been fully read + if (m_cursor == m_inputbuf_size) + { + m_scanposition += static_cast(m_cursor); + m_cursor = 0; + m_input.read(m_inputbuf.get(), INPUTBUF_CAP); + + // Indicate we hit end of file, and resize + // input buffer to show that it's not at full capacity + if (m_input.eof()) + { + m_eof = true; + m_inputbuf_size = static_cast(m_input.gcount()); + + // Return null if there's nothing left to read + if (m_inputbuf_size == 0) + { + return nullptr; + } + } + } + + return &m_inputbuf[m_cursor]; + } + + public: + // Iterator implementation for the CSV parser, which reads + // from the CSV row by row in the form of a vector of strings + class iterator + { + public: + typedef std::ptrdiff_t difference_type; + typedef std::vector value_type; + typedef const std::vector *pointer; + typedef const std::vector &reference; + typedef std::input_iterator_tag iterator_category; + + explicit iterator(CsvParser *p, bool end = false) : m_parser(p), m_current_row(end ? -1 : 0) + { + if (!end) + { + m_row.reserve(50); + next(); + } + } + + iterator &operator++() + { + next(); + return *this; + } + + iterator operator++(int) + { + iterator i = (*this); + ++(*this); + return i; + } + + bool operator==(const iterator &other) const + { + return m_current_row == other.m_current_row && m_row.size() == other.m_row.size(); + } + + bool operator!=(const iterator &other) const + { + return !(*this == other); + } + + reference operator*() const + { + return m_row; + } + + pointer operator->() const + { + return &m_row; + } + + private: + value_type m_row; + CsvParser *m_parser; + int m_current_row; + + void next() + { + value_type::size_type num_fields = 0; + for (;;) + { + auto field = m_parser->next_field(); + switch (field.type) + { + case FieldType::CSV_END: + if (num_fields < m_row.size()) + { + m_row.resize(num_fields); + } + m_current_row = -1; + return; + case FieldType::ROW_END: + if (num_fields < m_row.size()) + { + m_row.resize(num_fields); + } + m_current_row++; + return; + case FieldType::DATA: + if (num_fields < m_row.size()) + { + m_row[num_fields] = std::move(*field.data); + } + else + { + m_row.push_back(std::move(*field.data)); + } + num_fields++; + } + } + } + }; + + iterator begin() + { + return iterator(this); + }; + iterator end() + { + return iterator(this, true); + }; +}; +} // namespace csv +} // namespace aria +#endif From 5ead4f7bca57df6b7a655b90e9d6c36056f7d508 Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Tue, 16 Jun 2026 14:06:17 +0100 Subject: [PATCH 2/6] refactor(IW3): move asset overriding to a single place and add `StringTable` support --- codxe.vcxproj | 6 +- src/game/iw3/mp/components/assets.cpp | 398 ++++++++++++++++ .../mp/components/{scr_parser.h => assets.h} | 8 +- src/game/iw3/mp/components/cmds.cpp | 29 -- src/game/iw3/mp/components/mpsp.cpp | 52 -- src/game/iw3/mp/components/scr_parser.cpp | 148 ------ src/game/iw3/mp/main.cpp | 80 +--- src/third_party/aria_csv/csv_parser.hpp | 446 ++++++++++++++++++ 8 files changed, 854 insertions(+), 313 deletions(-) create mode 100644 src/game/iw3/mp/components/assets.cpp rename src/game/iw3/mp/components/{scr_parser.h => assets.h} (62%) delete mode 100644 src/game/iw3/mp/components/scr_parser.cpp create mode 100644 src/third_party/aria_csv/csv_parser.hpp diff --git a/codxe.vcxproj b/codxe.vcxproj index 0c208ef1..f2437208 100644 --- a/codxe.vcxproj +++ b/codxe.vcxproj @@ -87,6 +87,7 @@ + @@ -102,7 +103,6 @@ - @@ -201,6 +201,8 @@ + + @@ -214,6 +216,7 @@ + @@ -224,7 +227,6 @@ - diff --git a/src/game/iw3/mp/components/assets.cpp b/src/game/iw3/mp/components/assets.cpp new file mode 100644 index 00000000..1d592f89 --- /dev/null +++ b/src/game/iw3/mp/components/assets.cpp @@ -0,0 +1,398 @@ +#include "pch.h" +#include "assets.h" +#include "third_party/aria_csv/csv_parser.hpp" +#include + +namespace iw3 +{ +namespace mp +{ + +namespace Assets +{ +namespace Utils +{ +std::string BuildAssetPath(const char *asset_name, const char *extension = nullptr) +{ + const std::string base_path = Config::GetModBasePath(); + if (base_path.empty() || asset_name == nullptr || asset_name[0] == '\0') + { + return ""; + } + + std::string relative_path = asset_name; + std::replace(relative_path.begin(), relative_path.end(), '/', '\\'); + + std::string path = base_path + "\\" + relative_path; + if (extension != nullptr) + { + path += extension; + } + + return path; +} + +bool ReadAssetOverride(const std::string &path, std::string &buffer) +{ + buffer.clear(); + + HANDLE file = CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) + { + return false; + } + + const DWORD file_size = GetFileSize(file, nullptr); + if (file_size == INVALID_FILE_SIZE || file_size > 0x7FFFFFFF) + { + CloseHandle(file); + return false; + } + + if (file_size > 0) + { + buffer.resize(file_size); + + DWORD bytes_read = 0; + if (!ReadFile(file, &buffer[0], file_size, &bytes_read, nullptr) || bytes_read != file_size) + { + CloseHandle(file); + buffer.clear(); + return false; + } + } + + CloseHandle(file); + return true; +} + +} // namespace Utils + +namespace MapEnts_ +{ +std::unordered_map> mapents_buffers; + +void _override(MapEnts *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + const std::string path = Utils::BuildAssetPath(asset->name, ".ents"); + if (path.empty()) + { + return; + } + + std::string buffer; + if (!Utils::ReadAssetOverride(path, buffer)) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding map ents '%s' from '%s'\n", asset->name, path.c_str()); + + auto itr = mapents_buffers.find(asset->name); + if (itr != mapents_buffers.end()) + { + mapents_buffers.erase(itr); + } + + mapents_buffers[asset->name] = make_unique(); + itr = mapents_buffers.find(asset->name); + std::string *mapents_buffer = itr->second.get(); + + mapents_buffer->assign(buffer); + + asset->entityString = const_cast(mapents_buffer->c_str()); + asset->numEntityChars = static_cast(mapents_buffer->length()); +} +} // namespace MapEnts_ + +namespace RawFile_ +{ +struct RawFileOverride +{ + std::string name; + std::string buffer; + RawFile asset; +}; + +std::unordered_map> rawfile_overrides; + +std::string BuildRawFilePath(const char *asset_name) +{ + return Utils::BuildAssetPath(asset_name); +} + +RawFile *FindOverride(const char *asset_name) +{ + if (asset_name == nullptr || asset_name[0] == '\0') + { + return nullptr; + } + + auto itr = rawfile_overrides.find(asset_name); + if (itr != rawfile_overrides.end()) + { + return itr->second ? &itr->second->asset : nullptr; + } + + const std::string path = BuildRawFilePath(asset_name); + if (path.empty()) + { + rawfile_overrides[asset_name] = nullptr; + return nullptr; + } + + std::string buffer; + if (!Utils::ReadAssetOverride(path, buffer)) + { + rawfile_overrides[asset_name] = nullptr; + return nullptr; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded rawfile '%s' from '%s'\n", asset_name, path.c_str()); + + rawfile_overrides[asset_name] = make_unique(); + itr = rawfile_overrides.find(asset_name); + RawFileOverride *rawfile_override = itr->second.get(); + + rawfile_override->name = asset_name; + rawfile_override->buffer.assign(buffer); + rawfile_override->asset.name = rawfile_override->name.c_str(); + rawfile_override->asset.len = static_cast(rawfile_override->buffer.length()); + rawfile_override->asset.buffer = rawfile_override->buffer.c_str(); + + return &rawfile_override->asset; +} + +void _override(RawFile *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + RawFile *override_asset = FindOverride(asset->name); + if (override_asset == nullptr) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding rawfile '%s'\n", asset->name); + asset->len = override_asset->len; + asset->buffer = override_asset->buffer; +} +} // namespace RawFile_ + +namespace StringTable_ +{ +struct StringTableOverride +{ + std::string name; + std::vector cells; + std::vector values; + StringTable asset; +}; + +std::unordered_map> stringtable_overrides; + +bool LoadCsv(const std::string &path, std::vector> &rows) +{ + rows.clear(); + + std::ifstream file(path.c_str(), std::ios::binary); + if (!file) + { + return false; + } + + try + { + aria::csv::CsvParser parser(file); + for (aria::csv::CsvParser::iterator itr = parser.begin(); itr != parser.end(); ++itr) + { + rows.push_back(*itr); + } + } + catch (...) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to parse stringtable csv '%s'\n", path.c_str()); + rows.clear(); + return false; + } + + return true; +} + +StringTable *FindOverride(const char *asset_name) +{ + if (asset_name == nullptr || asset_name[0] == '\0') + { + return nullptr; + } + + auto itr = stringtable_overrides.find(asset_name); + if (itr != stringtable_overrides.end()) + { + return itr->second ? &itr->second->asset : nullptr; + } + + const std::string path = Utils::BuildAssetPath(asset_name); + if (path.empty()) + { + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + std::vector> rows; + if (!LoadCsv(path, rows)) + { + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + size_t column_count = 0; + for (size_t row = 0; row < rows.size(); ++row) + { + if (rows[row].size() > column_count) + { + column_count = rows[row].size(); + } + } + + const size_t cell_count = rows.size() * column_count; + if (rows.size() > 0x7FFFFFFF || column_count > 0x7FFFFFFF || cell_count > 0x7FFFFFFF) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Stringtable csv too large '%s'\n", path.c_str()); + stringtable_overrides[asset_name] = nullptr; + return nullptr; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded stringtable '%s' from '%s'\n", asset_name, path.c_str()); + + stringtable_overrides[asset_name] = make_unique(); + itr = stringtable_overrides.find(asset_name); + StringTableOverride *stringtable_override = itr->second.get(); + + stringtable_override->name = asset_name; + stringtable_override->cells.resize(cell_count); + stringtable_override->values.resize(cell_count); + + for (size_t row = 0; row < rows.size(); ++row) + { + for (size_t column = 0; column < column_count; ++column) + { + const size_t cell_index = row * column_count + column; + if (column < rows[row].size()) + { + stringtable_override->cells[cell_index] = rows[row][column]; + } + } + } + + for (size_t i = 0; i < cell_count; ++i) + { + stringtable_override->values[i] = stringtable_override->cells[i].c_str(); + } + + stringtable_override->asset.name = stringtable_override->name.c_str(); + stringtable_override->asset.columnCount = static_cast(column_count); + stringtable_override->asset.rowCount = static_cast(rows.size()); + stringtable_override->asset.values = + stringtable_override->values.empty() ? nullptr : &stringtable_override->values[0]; + + return &stringtable_override->asset; +} + +void _override(StringTable *asset) +{ + if (!asset || !asset->name || asset->name[0] == '\0') + { + return; + } + + StringTable *override_asset = FindOverride(asset->name); + if (override_asset == nullptr) + { + return; + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding stringtable '%s'\n", asset->name); + asset->columnCount = override_asset->columnCount; + asset->rowCount = override_asset->rowCount; + asset->values = override_asset->values; +} +} // namespace StringTable_ +} // namespace Assets + +Detour DB_LinkXAssetEntry_Detour; +Detour DB_FindXAssetHeader_Detour; + +XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) +{ + if (newEntry != nullptr) + { + switch (newEntry->asset.type) + { + case ASSET_TYPE_MAP_ENTS: + Assets::MapEnts_::_override(newEntry->asset.header.mapEnts); + break; + case ASSET_TYPE_RAWFILE: + Assets::RawFile_::_override(newEntry->asset.header.rawfile); + break; + case ASSET_TYPE_STRINGTABLE: + Assets::StringTable_::_override(newEntry->asset.header.stringTable); + break; + } + } + + return DB_LinkXAssetEntry_Detour.GetOriginal()(newEntry, allowOverride); +} + +XAssetHeader DB_FindXAssetHeader_Hook(const XAssetType type, const char *name) +{ + if (type == ASSET_TYPE_RAWFILE) + { + RawFile *rawfile = Assets::RawFile_::FindOverride(name); + if (rawfile != nullptr) + { + XAssetHeader header; + header.rawfile = rawfile; + return header; + } + } + else if (type == ASSET_TYPE_STRINGTABLE) + { + StringTable *stringtable = Assets::StringTable_::FindOverride(name); + if (stringtable != nullptr) + { + XAssetHeader header; + header.stringTable = stringtable; + return header; + } + } + + return DB_FindXAssetHeader_Detour.GetOriginal()(type, name); +} + +assets::assets() +{ + DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); + DB_LinkXAssetEntry_Detour.Install(); + + DB_FindXAssetHeader_Detour = Detour(DB_FindXAssetHeader, DB_FindXAssetHeader_Hook); + DB_FindXAssetHeader_Detour.Install(); +} + +assets::~assets() +{ + DB_FindXAssetHeader_Detour.Remove(); + + DB_LinkXAssetEntry_Detour.Remove(); +} +} // namespace mp +} // namespace iw3 diff --git a/src/game/iw3/mp/components/scr_parser.h b/src/game/iw3/mp/components/assets.h similarity index 62% rename from src/game/iw3/mp/components/scr_parser.h rename to src/game/iw3/mp/components/assets.h index 460c157d..67564c10 100644 --- a/src/game/iw3/mp/components/scr_parser.h +++ b/src/game/iw3/mp/components/assets.h @@ -6,15 +6,15 @@ namespace iw3 { namespace mp { -class scr_parser : public Module +class assets : public Module { public: - scr_parser(); - ~scr_parser(); + assets(); + ~assets(); const char *get_name() override { - return "scr_parser"; + return "assets"; }; }; } // namespace mp diff --git a/src/game/iw3/mp/components/cmds.cpp b/src/game/iw3/mp/components/cmds.cpp index e633b616..3daea549 100644 --- a/src/game/iw3/mp/components/cmds.cpp +++ b/src/game/iw3/mp/components/cmds.cpp @@ -111,46 +111,17 @@ void ClientCommand_Hook(int clientNum) ClientCommand_Detour.GetOriginal()(clientNum); } -Detour Cmd_ExecFromFastFile_Detour; - -bool Cmd_ExecFromFastFile_Hook(int localClientNum, int controllerIndex, const char *filename) -{ - auto callOriginal = [&]() - { - return Cmd_ExecFromFastFile_Detour.GetOriginal()(localClientNum, - controllerIndex, filename); - }; - - // Check if mod is active - std::string modBasePath = Config::GetModBasePath(); - if (modBasePath.empty()) - return callOriginal(); - - std::string contents = filesystem::read_file_to_string(modBasePath + "\\" + filename); - if (contents.empty()) - return callOriginal(); - - Com_Printf(CON_CHANNEL_SYSTEM, "execing %s from raw:\\\n", filename); - Cbuf_ExecuteBuffer(localClientNum, controllerIndex, contents.c_str()); - return true; -} - cmds::cmds() { ClientCommand_Detour = Detour(ClientCommand, ClientCommand_Hook); ClientCommand_Detour.Install(); - Cmd_ExecFromFastFile_Detour = Detour(Cmd_ExecFromFastFile, Cmd_ExecFromFastFile_Hook); - Cmd_ExecFromFastFile_Detour.Install(); - command::add("dumpraw", Cmd_Dumpraw_f); } cmds::~cmds() { ClientCommand_Detour.Remove(); - - Cmd_ExecFromFastFile_Detour.Remove(); } } // namespace mp } // namespace iw3 diff --git a/src/game/iw3/mp/components/mpsp.cpp b/src/game/iw3/mp/components/mpsp.cpp index 31747abf..90394e95 100644 --- a/src/game/iw3/mp/components/mpsp.cpp +++ b/src/game/iw3/mp/components/mpsp.cpp @@ -438,52 +438,6 @@ const ZoneOverride ZONE_OVERRIDES[] = { int g_zoneOverrideIndex = -1; -Detour DB_LinkXAssetEntry_Detour; -XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) -{ - XAsset xasset; - xasset.type = newEntry->asset.type; - xasset.header = newEntry->asset.header; - - if (mpsp::is_sp_map) - { - switch (newEntry->asset.type) - { - case ASSET_TYPE_MAP_ENTS: - { - Asset::MapEnts_::override_(newEntry->asset.header.mapEnts); - break; - } - case ASSET_TYPE_RAWFILE: - { - Asset::RawFile_::override_(newEntry->asset.header.rawfile); - break; - } - case ASSET_TYPE_GAMEWORLD_SP: - { - newEntry->asset.type = ASSET_TYPE_GAMEWORLD_MP; - break; - } - // Hijack the reference asset ',' mechanism to avoid reaching asset limits. - case ASSET_TYPE_WEAPON: - { - static const std::string weapon_default_reference_name = - std::string(",") + g_defaultAssetName[ASSET_TYPE_WEAPON]; - DB_SetXAssetName(&xasset, weapon_default_reference_name.c_str()); - break; - } - case ASSET_TYPE_FX: - { - static const std::string fx_default_reference_name = std::string(",") + g_defaultAssetName[ASSET_TYPE_FX]; - DB_SetXAssetName(&xasset, fx_default_reference_name.c_str()); - break; - } - } - } - - return DB_LinkXAssetEntry_Detour.GetOriginal()(newEntry, allowOverride); -} - Detour Com_sprintf_Detour; int Com_sprintf_Hook(char *dest, unsigned int size, const char *fmt...) { @@ -778,10 +732,6 @@ mpsp::mpsp() DB_AuthLoad_Inflate_Detour = Detour(DB_AuthLoad_Inflate, DB_AuthLoad_Inflate_Hook); DB_AuthLoad_Inflate_Detour.Install(); - // Rewrite some assets before linking - DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); - DB_LinkXAssetEntry_Detour.Install(); - Load_XAssetArrayCustom_Detour = Detour(Load_XAssetArrayCustom, Load_XAssetArrayCustom_Stub); Load_XAssetArrayCustom_Detour.Install(); @@ -799,8 +749,6 @@ mpsp::~mpsp() DB_AuthLoad_Inflate_Detour.Remove(); - DB_LinkXAssetEntry_Detour.Remove(); - Load_XAssetArrayCustom_Detour.Remove(); SV_AddEntitiesVisibleFromPoint_Detour.Remove(); diff --git a/src/game/iw3/mp/components/scr_parser.cpp b/src/game/iw3/mp/components/scr_parser.cpp deleted file mode 100644 index e7d6dbc0..00000000 --- a/src/game/iw3/mp/components/scr_parser.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include "pch.h" -#include "scr_parser.h" - -namespace iw3 -{ -namespace mp -{ -namespace -{ -const size_t MAX_SCRIPT_PATH = 512; - -void NormalizePath(char *path) -{ - if (path == nullptr) - return; - - for (char *cursor = path; *cursor != '\0'; ++cursor) - { - if (*cursor == '/') - *cursor = '\\'; - } -} - -bool BuildScriptPath(char *path, size_t path_size, const char *base_path, const char *script_path) -{ - if (path == nullptr || path_size == 0 || base_path == nullptr || base_path[0] == '\0' || script_path == nullptr || - script_path[0] == '\0') - { - return false; - } - - const int written = _snprintf_s(path, path_size, _TRUNCATE, "%s\\%s", base_path, script_path); - path[path_size - 1] = '\0'; - - if (written < 0 || static_cast(written) >= path_size) - return false; - - NormalizePath(path); - return true; -} - -char *ReadFileToGameTempBuffer(const char *path) -{ - HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, nullptr); - if (file == INVALID_HANDLE_VALUE) - return nullptr; - - const DWORD file_size = GetFileSize(file, nullptr); - if (file_size == INVALID_FILE_SIZE || file_size == 0) - { - CloseHandle(file); - return nullptr; - } - - char *buffer = static_cast(Hunk_AllocateTempMemoryHighInternal(file_size + 1)); - if (buffer == nullptr) - { - CloseHandle(file); - return nullptr; - } - - DWORD bytes_read = 0; - if (!ReadFile(file, buffer, file_size, &bytes_read, nullptr) || bytes_read != file_size) - { - CloseHandle(file); - return nullptr; - } - - CloseHandle(file); - buffer[file_size] = '\0'; - return buffer; -} - -void WriteScriptDump(const char *script_path, const char *contents) -{ - if (script_path == nullptr || contents == nullptr) - return; - - char dump_path[MAX_SCRIPT_PATH]; - if (!BuildScriptPath(dump_path, sizeof(dump_path), DUMP_DIR, script_path)) - return; - - char dir_path[MAX_SCRIPT_PATH]; - strncpy(dir_path, dump_path, sizeof(dir_path) - 1); - dir_path[sizeof(dir_path) - 1] = '\0'; - - char *last_slash = strrchr(dir_path, '\\'); - if (last_slash != nullptr) - { - *last_slash = '\0'; - filesystem::create_nested_dirs(dir_path); - } - - HANDLE file = CreateFileA(dump_path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); - if (file == INVALID_HANDLE_VALUE) - return; - - DWORD bytes_written = 0; - const DWORD content_size = static_cast(std::strlen(contents)); - WriteFile(file, contents, content_size, &bytes_written, nullptr); - CloseHandle(file); -} -} // namespace - -Detour Scr_AddSourceBuffer_Detour; - -char *Scr_AddSourceBuffer_Hook(const char *filename, const char *extFilename, const char *codePos, bool archive) -{ - auto callOriginal = [&]() - { - return Scr_AddSourceBuffer_Detour.GetOriginal()(filename, extFilename, codePos, - archive); - }; - - if (Config::dump_rawfile) - { - char *contents = callOriginal(); - if (contents != nullptr) - WriteScriptDump(extFilename, contents); - - return contents; - } - - char override_path[MAX_SCRIPT_PATH]; - if (!BuildScriptPath(override_path, sizeof(override_path), Config::GetModBasePathCStr(), extFilename)) - return callOriginal(); - - char *buffer = ReadFileToGameTempBuffer(override_path); - if (buffer == nullptr) - return callOriginal(); - - DbgPrint("GSCLoader: Loaded override script: %s\n", override_path); - return buffer; -} - -scr_parser::scr_parser() -{ - Scr_AddSourceBuffer_Detour = Detour(Scr_AddSourceBuffer, Scr_AddSourceBuffer_Hook); - Scr_AddSourceBuffer_Detour.Install(); -} - -scr_parser::~scr_parser() -{ - Scr_AddSourceBuffer_Detour.Remove(); -} -} // namespace mp -} // namespace iw3 diff --git a/src/game/iw3/mp/main.cpp b/src/game/iw3/mp/main.cpp index 7205c8e3..b1fd4e51 100644 --- a/src/game/iw3/mp/main.cpp +++ b/src/game/iw3/mp/main.cpp @@ -1,4 +1,5 @@ #include "pch.h" +#include "components/assets.h" #include "components/cg.h" #include "components/cj_tas.h" #include "components/clipmap.h" @@ -14,7 +15,6 @@ #include "components/image_loader.h" #include "components/mpsp.h" #include "components/pm.h" -#include "components/scr_parser.h" #include "components/stats.h" #include "components/sv_bots.h" #include "common/config.h" @@ -24,78 +24,6 @@ namespace iw3 { namespace mp { -Detour Load_MapEntsPtr_Detour; - -void Load_MapEntsPtr_Hook() -{ - // TODO: don't write null byte to file - // and add null byte to entityString when reading from file - - DbgPrint("Load_MapEntsPtr_Hook\n"); - - // TODO: write comment what this is *** - // Get pointer to pointer stored at 0x82475914 - MapEnts **varMapEntsPtr = *(MapEnts ***)0x82475914; - - Load_MapEntsPtr_Detour.GetOriginal()(); - - // Validate pointer before dereferencing - if (varMapEntsPtr && *varMapEntsPtr) - { - MapEnts *mapEnts = *varMapEntsPtr; - - // Write stock map ents to disk - std::string file_path = DUMP_DIR; - file_path += std::string("\\") + mapEnts->name; - file_path += ".ents"; // iw4x naming convention - std::replace(file_path.begin(), file_path.end(), '/', '\\'); // Replace forward slashes with backslashes - filesystem::write_file_to_disk(file_path.c_str(), mapEnts->entityString, mapEnts->numEntityChars); - - // Use new ConfigModule API - // Load map ents from file - // Path to check for existing entity file - std::string raw_file_path = Config::GetModBasePath(); - - raw_file_path += std::string("\\") + mapEnts->name; - raw_file_path += ".ents"; // IW4x naming convention - std::replace(raw_file_path.begin(), raw_file_path.end(), '/', '\\'); // Replace forward slashes with backslashes - - // If the file exists, replace entityString - if (filesystem::file_exists(raw_file_path)) - { - DbgPrint("Found entity file: %s\n", raw_file_path.c_str()); - std::string new_entity_string = filesystem::read_file_to_string(raw_file_path); - if (!new_entity_string.empty()) - { - // Allocate new memory and copy the data - size_t new_size = new_entity_string.size() + 1; // Include null terminator - char *new_memory = static_cast(malloc(new_size)); - - if (new_memory) - { - memcpy(new_memory, new_entity_string.c_str(), new_size); // Copy with null terminator - - // Update the entityString pointer to point to the new memory - mapEnts->entityString = new_memory; - - // // Update numEntityChars - // mapEnts->numEntityChars = static_cast(new_entity_string.size()); // unnecessary - - DbgPrint("Replaced entityString from file: %s\n", raw_file_path.c_str()); - } - else - { - DbgPrint("Failed to allocate memory for entityString replacement.\n"); - } - } - } - } - else - { - DbgPrint("Hooked Load_MapEntsPtr: varMapEntsPtr is NULL or invalid.\n"); - } -} - /** * Patch out the signature checks used during fastfile authentication. * Signature data must still be present in @@ -125,6 +53,7 @@ IW3_MP_Plugin::IW3_MP_Plugin() RegisterModule(new command()); RegisterModule(new cg()); + RegisterModule(new assets()); RegisterModule(new cj_tas()); RegisterModule(new clipmap()); RegisterModule(new cmds()); @@ -137,17 +66,12 @@ IW3_MP_Plugin::IW3_MP_Plugin() RegisterModule(new image_loader()); RegisterModule(new pm()); RegisterModule(new mpsp()); - RegisterModule(new scr_parser()); RegisterModule(new stats()); RegisterModule(new sv_bots()); - - Load_MapEntsPtr_Detour = Detour(Load_MapEntsPtr, Load_MapEntsPtr_Hook); - Load_MapEntsPtr_Detour.Install(); } IW3_MP_Plugin::~IW3_MP_Plugin() { - Load_MapEntsPtr_Detour.Remove(); } } // namespace mp diff --git a/src/third_party/aria_csv/csv_parser.hpp b/src/third_party/aria_csv/csv_parser.hpp new file mode 100644 index 00000000..b589d435 --- /dev/null +++ b/src/third_party/aria_csv/csv_parser.hpp @@ -0,0 +1,446 @@ +#ifndef ARIA_CSV_H +#define ARIA_CSV_H + +// Local vendored copy modified for the Xbox 360 compiler used by this project. +// https://github.com/AriaFallah/csv-parser + +#include +#include +#include +#include +#include + +namespace aria +{ +namespace csv +{ + +namespace Term +{ +enum Enum +{ + CRLF = -2 +}; +} // namespace Term + +namespace FieldType +{ +enum Enum +{ + DATA, + ROW_END, + CSV_END +}; +} // namespace FieldType + +typedef std::vector> CSV; + +// Checking for '\n', '\r', and '\r\n' by default +inline bool operator==(const char c, const Term::Enum t) +{ + switch (t) + { + case Term::CRLF: + return c == '\r' || c == '\n'; + default: + return static_cast(t) == c; + } +} + +inline bool operator!=(const char c, const Term::Enum t) +{ + return !(c == t); +} + +// Wraps returned fields so we can also indicate +// that we hit row endings or the end of the csv itself +struct Field +{ + explicit Field(FieldType::Enum t) : type(t), data(nullptr) + { + } + explicit Field(const std::string &str) : type(FieldType::DATA), data(&str) + { + } + + FieldType::Enum type; + const std::string *data; +}; + +// Reads and parses lines from a csv file +class CsvParser +{ + private: + // CSV state for state machine + struct State + { + enum Enum + { + START_OF_FIELD, + IN_FIELD, + IN_QUOTED_FIELD, + IN_ESCAPED_QUOTE, + END_OF_ROW, + EMPTY + }; + }; + State::Enum m_state; + + // Configurable attributes + char m_quote; + char m_delimiter; + Term::Enum m_terminator; + std::istream &m_input; + + // Buffer capacities + static const int FIELDBUF_CAP = 1024; + static const int INPUTBUF_CAP = 1024 * 128; + + // Buffers + std::string m_fieldbuf; + std::unique_ptr m_inputbuf; + + // Misc + bool m_eof; + size_t m_cursor; + size_t m_inputbuf_size; + std::streamoff m_scanposition; + + public: + // Creates the CSV parser which by default, splits on commas, + // uses quotes to escape, and handles CSV files that end in either + // '\r', '\n', or '\r\n'. + explicit CsvParser(std::istream &input) + : m_state(State::START_OF_FIELD), m_quote('"'), m_delimiter(','), m_terminator(Term::CRLF), m_input(input), + m_inputbuf(new char[INPUTBUF_CAP]()), m_eof(false), m_cursor(INPUTBUF_CAP), m_inputbuf_size(INPUTBUF_CAP), + m_scanposition(-INPUTBUF_CAP) + { + // Reserve space upfront to improve performance + m_fieldbuf.reserve(FIELDBUF_CAP); + if (!m_input.good()) + { + throw std::runtime_error("Something is wrong with input stream"); + } + } + + // Change the quote character + CsvParser &"e(char c) + { + m_quote = c; + return std::move(*this); + } + + // Change the delimiter character + CsvParser &&delimiter(char c) + { + m_delimiter = c; + return std::move(*this); + } + + // Change the terminator character + CsvParser &&terminator(char c) + { + m_terminator = static_cast(c); + return std::move(*this); + } + + // The parser is in the empty state when there are + // no more tokens left to read from the input buffer + bool empty() + { + return m_state == State::EMPTY; + } + + // Not the actual position in the stream (its buffered) just the + // position up to last availiable token + std::streamoff position() const + { + return m_scanposition + static_cast(m_cursor); + } + + // Reads a single field from the CSV + Field next_field() + { + if (empty()) + { + return Field(FieldType::CSV_END); + } + m_fieldbuf.clear(); + + // This loop runs until either the parser has + // read a full field or until there's no tokens left to read + for (;;) + { + char *maybe_token = top_token(); + + // If we're out of tokens to read return whatever's left in the + // field and row buffers. If there's nothing left, return null. + if (!maybe_token) + { + m_state = State::EMPTY; + return !m_fieldbuf.empty() ? Field(m_fieldbuf) : Field(FieldType::CSV_END); + } + + // Parsing the CSV is done using a finite state machine + char c = *maybe_token; + switch (m_state) + { + case State::START_OF_FIELD: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_quote) + { + m_state = State::IN_QUOTED_FIELD; + } + else if (c == m_delimiter) + { + return Field(m_fieldbuf); + } + else + { + m_state = State::IN_FIELD; + m_fieldbuf += c; + } + + break; + + case State::IN_FIELD: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_delimiter) + { + m_state = State::START_OF_FIELD; + return Field(m_fieldbuf); + } + else + { + m_fieldbuf += c; + } + + break; + + case State::IN_QUOTED_FIELD: + m_cursor++; + if (c == m_quote) + { + m_state = State::IN_ESCAPED_QUOTE; + } + else + { + m_fieldbuf += c; + } + + break; + + case State::IN_ESCAPED_QUOTE: + m_cursor++; + if (c == m_terminator) + { + handle_crlf(c); + m_state = State::END_OF_ROW; + return Field(m_fieldbuf); + } + + if (c == m_quote) + { + m_state = State::IN_QUOTED_FIELD; + m_fieldbuf += c; + } + else if (c == m_delimiter) + { + m_state = State::START_OF_FIELD; + return Field(m_fieldbuf); + } + else + { + m_state = State::IN_FIELD; + m_fieldbuf += c; + } + + break; + + case State::END_OF_ROW: + m_state = State::START_OF_FIELD; + return Field(FieldType::ROW_END); + + case State::EMPTY: + throw std::logic_error("You goofed"); + } + } + } + + private: + // When the parser hits the end of a line it needs + // to check the special case of '\r\n' as a terminator. + // If it finds that the previous token was a '\r', and + // the next token will be a '\n', it skips the '\n'. + void handle_crlf(const char c) + { + if (m_terminator != Term::CRLF || c != '\r') + { + return; + } + + char *token = top_token(); + if (token && *token == '\n') + { + m_cursor++; + } + } + + // Pulls the next token from the input buffer, but does not move + // the cursor forward. If the stream is empty and the input buffer + // is also empty return a nullptr. + char *top_token() + { + // Return null if there's nothing left to read + if (m_eof && m_cursor == m_inputbuf_size) + { + return nullptr; + } + + // Refill the input buffer if it's been fully read + if (m_cursor == m_inputbuf_size) + { + m_scanposition += static_cast(m_cursor); + m_cursor = 0; + m_input.read(m_inputbuf.get(), INPUTBUF_CAP); + + // Indicate we hit end of file, and resize + // input buffer to show that it's not at full capacity + if (m_input.eof()) + { + m_eof = true; + m_inputbuf_size = static_cast(m_input.gcount()); + + // Return null if there's nothing left to read + if (m_inputbuf_size == 0) + { + return nullptr; + } + } + } + + return &m_inputbuf[m_cursor]; + } + + public: + // Iterator implementation for the CSV parser, which reads + // from the CSV row by row in the form of a vector of strings + class iterator + { + public: + typedef std::ptrdiff_t difference_type; + typedef std::vector value_type; + typedef const std::vector *pointer; + typedef const std::vector &reference; + typedef std::input_iterator_tag iterator_category; + + explicit iterator(CsvParser *p, bool end = false) : m_parser(p), m_current_row(end ? -1 : 0) + { + if (!end) + { + m_row.reserve(50); + next(); + } + } + + iterator &operator++() + { + next(); + return *this; + } + + iterator operator++(int) + { + iterator i = (*this); + ++(*this); + return i; + } + + bool operator==(const iterator &other) const + { + return m_current_row == other.m_current_row && m_row.size() == other.m_row.size(); + } + + bool operator!=(const iterator &other) const + { + return !(*this == other); + } + + reference operator*() const + { + return m_row; + } + + pointer operator->() const + { + return &m_row; + } + + private: + value_type m_row; + CsvParser *m_parser; + int m_current_row; + + void next() + { + value_type::size_type num_fields = 0; + for (;;) + { + auto field = m_parser->next_field(); + switch (field.type) + { + case FieldType::CSV_END: + if (num_fields < m_row.size()) + { + m_row.resize(num_fields); + } + m_current_row = -1; + return; + case FieldType::ROW_END: + if (num_fields < m_row.size()) + { + m_row.resize(num_fields); + } + m_current_row++; + return; + case FieldType::DATA: + if (num_fields < m_row.size()) + { + m_row[num_fields] = std::move(*field.data); + } + else + { + m_row.push_back(std::move(*field.data)); + } + num_fields++; + } + } + } + }; + + iterator begin() + { + return iterator(this); + }; + iterator end() + { + return iterator(this, true); + }; +}; +} // namespace csv +} // namespace aria +#endif From fce972b0248fc9ab64e51b3ace2021a38128d525 Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Wed, 24 Jun 2026 09:53:18 +0100 Subject: [PATCH 3/6] wip: manual csv parser avoids heap allocations --- src/game/iw3/mp/components/assets.cpp | 1069 ++++++++++++++++++++----- 1 file changed, 866 insertions(+), 203 deletions(-) diff --git a/src/game/iw3/mp/components/assets.cpp b/src/game/iw3/mp/components/assets.cpp index 1d592f89..61e28f53 100644 --- a/src/game/iw3/mp/components/assets.cpp +++ b/src/game/iw3/mp/components/assets.cpp @@ -1,336 +1,1023 @@ #include "pch.h" #include "assets.h" -#include "third_party/aria_csv/csv_parser.hpp" -#include + +#ifndef INVALID_FILE_ATTRIBUTES +#define INVALID_FILE_ATTRIBUTES ((DWORD) - 1) +#endif + +#ifndef INVALID_FILE_SIZE +#define INVALID_FILE_SIZE ((DWORD) - 1) +#endif namespace iw3 { namespace mp { -namespace Assets +namespace +{ +Detour DB_LinkXAssetEntry_Detour; + +const size_t MAX_OVERRIDE_CACHE_ENTRIES = 512; +const size_t STRINGTABLE_VALUE_ALIGNMENT = sizeof(const char *); + +enum OverrideCacheState +{ + CACHE_EMPTY, + CACHE_LOADED +}; + +struct OverrideCacheEntry { -namespace Utils + OverrideCacheState state; + XAssetType type; + char name[MAX_PATH]; + void *storage; + DWORD storage_size; + DWORD payload_size; + StringTable stringtable; +}; + +OverrideCacheEntry *s_override_cache = nullptr; +size_t s_override_cache_capacity = 0; + +const char *ProcessTypeName(DWORD process_type) { -std::string BuildAssetPath(const char *asset_name, const char *extension = nullptr) + return process_type == PROC_TYPE_SYSTEM ? "system" : "title"; +} + +size_t AlignSize(size_t value, size_t alignment) { - const std::string base_path = Config::GetModBasePath(); - if (base_path.empty() || asset_name == nullptr || asset_name[0] == '\0') + return (value + alignment - 1) & ~(alignment - 1); +} + +void *AllocVirtualBlock(size_t size, const char *reason) +{ + const DWORD process_type = KeGetCurrentProcessType(); + DbgPrint("[codxe][assets] VirtualAlloc begin. reason='%s' size=%u processType=%u(%s)\n", + reason ? reason : "", static_cast(size), process_type, ProcessTypeName(process_type)); + + void *storage = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + if (storage == nullptr) + { + const DWORD error = GetLastError(); + DbgPrint("[codxe][assets] VirtualAlloc failed. reason='%s' size=%u processType=%u(%s) error=0x%08X\n", + reason ? reason : "", static_cast(size), process_type, + ProcessTypeName(process_type), error); + } + else + { + DbgPrint("[codxe][assets] VirtualAlloc end. reason='%s' ptr=%p size=%u processType=%u(%s)\n", + reason ? reason : "", storage, static_cast(size), process_type, + ProcessTypeName(process_type)); + } + + return storage; +} + +bool FreeVirtualBlock(void *storage, DWORD size, const char *reason) +{ + if (storage == nullptr) + { + return true; + } + + const DWORD process_type = KeGetCurrentProcessType(); + DbgPrint("[codxe][assets] VirtualFree begin. reason='%s' ptr=%p size=%u processType=%u(%s)\n", + reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type)); + + const BOOL result = VirtualFree(storage, 0, MEM_RELEASE); + if (!result) { - return ""; + const DWORD error = GetLastError(); + DbgPrint("[codxe][assets] VirtualFree failed. reason='%s' ptr=%p size=%u processType=%u(%s) error=0x%08X\n", + reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type), error); + return false; } - std::string relative_path = asset_name; - std::replace(relative_path.begin(), relative_path.end(), '/', '\\'); + DbgPrint("[codxe][assets] VirtualFree end. reason='%s' ptr=%p size=%u processType=%u(%s)\n", + reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type)); + return true; +} - std::string path = base_path + "\\" + relative_path; - if (extension != nullptr) +bool ResetCacheEntry(OverrideCacheEntry &entry, DWORD &storage_bytes) +{ + bool freed = false; + if (entry.storage != nullptr) { - path += extension; + if (FreeVirtualBlock(entry.storage, entry.storage_size, entry.name)) + { + storage_bytes += entry.storage_size; + freed = true; + } } - return path; + memset(&entry, 0, sizeof(entry)); + return freed; } -bool ReadAssetOverride(const std::string &path, std::string &buffer) +void ClearOverrideCache() { - buffer.clear(); + DbgPrint("[codxe][assets] ClearOverrideCache begin. cache=%p capacity=%u\n", s_override_cache, + static_cast(s_override_cache_capacity)); - HANDLE file = CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, nullptr); - if (file == INVALID_HANDLE_VALUE) + if (s_override_cache == nullptr) { - return false; + DbgPrint("[codxe][assets] ClearOverrideCache end. no cache allocated.\n"); + return; } - const DWORD file_size = GetFileSize(file, nullptr); - if (file_size == INVALID_FILE_SIZE || file_size > 0x7FFFFFFF) + unsigned int loaded_entries = 0; + unsigned int free_attempts = 0; + unsigned int free_successes = 0; + unsigned int free_failures = 0; + DWORD freed_storage_bytes = 0; + for (size_t i = 0; i < s_override_cache_capacity; ++i) { - CloseHandle(file); + if (s_override_cache[i].state == CACHE_LOADED) + { + ++loaded_entries; + } + + if (s_override_cache[i].storage != nullptr) + { + ++free_attempts; + } + + if (ResetCacheEntry(s_override_cache[i], freed_storage_bytes)) + { + ++free_successes; + } + else if (free_attempts != free_successes + free_failures) + { + ++free_failures; + } + } + + DbgPrint("[codxe][assets] ClearOverrideCache end. loaded=%u freeAttempts=%u freeSuccesses=%u freeFailures=%u " + "storageBytes=%u\n", + loaded_entries, free_attempts, free_successes, free_failures, freed_storage_bytes); +} + +bool InitializeOverrideCache() +{ + DbgPrint("[codxe][assets] InitializeOverrideCache begin. existing=%p capacity=%u\n", s_override_cache, + static_cast(s_override_cache_capacity)); + + if (s_override_cache != nullptr) + { + ClearOverrideCache(); + DbgPrint("[codxe][assets] InitializeOverrideCache end. reused existing cache.\n"); + return true; + } + + const size_t cache_size = sizeof(OverrideCacheEntry) * MAX_OVERRIDE_CACHE_ENTRIES; + s_override_cache = static_cast(AllocVirtualBlock(cache_size, "override cache metadata")); + if (s_override_cache == nullptr) + { + Com_Printf(CON_CHANNEL_FILES, + "[codxe][assets] Failed to allocate override cache. entries=%u bytes=%u error=0x%08X\n", + static_cast(MAX_OVERRIDE_CACHE_ENTRIES), static_cast(cache_size), + GetLastError()); + DbgPrint("[codxe][assets] InitializeOverrideCache end. failed bytes=%u error=0x%08X\n", + static_cast(cache_size), GetLastError()); + s_override_cache_capacity = 0; return false; } - if (file_size > 0) + s_override_cache_capacity = MAX_OVERRIDE_CACHE_ENTRIES; + memset(s_override_cache, 0, cache_size); + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override cache allocated. entries=%u bytes=%u storage=%p\n", + static_cast(s_override_cache_capacity), static_cast(cache_size), + s_override_cache); + DbgPrint("[codxe][assets] InitializeOverrideCache end. cache=%p bytes=%u\n", s_override_cache, + static_cast(cache_size)); + return true; +} + +void ShutdownOverrideCache() +{ + DbgPrint("[codxe][assets] ShutdownOverrideCache begin. cache=%p capacity=%u\n", s_override_cache, + static_cast(s_override_cache_capacity)); + + const DWORD cache_bytes = static_cast(sizeof(OverrideCacheEntry) * s_override_cache_capacity); + ClearOverrideCache(); + + if (s_override_cache != nullptr) { - buffer.resize(file_size); + if (FreeVirtualBlock(s_override_cache, cache_bytes, "override cache metadata")) + { + DbgPrint("[codxe][assets] ShutdownOverrideCache freed cache metadata. bytes=%u\n", cache_bytes); + } + else + { + DbgPrint("[codxe][assets] ShutdownOverrideCache failed to free cache metadata. bytes=%u\n", cache_bytes); + } + } - DWORD bytes_read = 0; - if (!ReadFile(file, &buffer[0], file_size, &bytes_read, nullptr) || bytes_read != file_size) + s_override_cache = nullptr; + s_override_cache_capacity = 0; + DbgPrint("[codxe][assets] ShutdownOverrideCache end.\n"); +} + +bool CopyAssetName(char *dest, size_t dest_size, const char *name) +{ + if (dest == nullptr || dest_size == 0 || name == nullptr || name[0] == '\0') + { + return false; + } + + size_t i = 0; + for (; name[i] != '\0'; ++i) + { + if (i + 1 >= dest_size) { - CloseHandle(file); - buffer.clear(); + dest[0] = '\0'; return false; } + + dest[i] = name[i]; } - CloseHandle(file); + dest[i] = '\0'; return true; } -} // namespace Utils - -namespace MapEnts_ +bool AppendPathPart(char *path, size_t path_size, const char *text, bool normalize_slashes) { -std::unordered_map> mapents_buffers; + if (path == nullptr || path_size == 0 || text == nullptr) + { + return false; + } -void _override(MapEnts *asset) + size_t length = strlen(path); + for (size_t i = 0; text[i] != '\0'; ++i) + { + if (length + 1 >= path_size) + { + return false; + } + + char c = text[i]; + if (normalize_slashes && c == '/') + { + c = '\\'; + } + + path[length++] = c; + } + + path[length] = '\0'; + return true; +} + +bool BuildAssetPath(char *path, size_t path_size, const char *asset_name, const char *extension = nullptr) { - if (!asset || !asset->name || asset->name[0] == '\0') + if (path == nullptr || path_size == 0) { - return; + return false; } - const std::string path = Utils::BuildAssetPath(asset->name, ".ents"); - if (path.empty()) + path[0] = '\0'; + + const char *base_path = Config::GetModBasePathCStr(); + if (base_path == nullptr || base_path[0] == '\0' || asset_name == nullptr || asset_name[0] == '\0') { - return; + return false; } - std::string buffer; - if (!Utils::ReadAssetOverride(path, buffer)) + if (!AppendPathPart(path, path_size, base_path, false)) { - return; + return false; + } + + if (!AppendPathPart(path, path_size, "\\", false)) + { + return false; } - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding map ents '%s' from '%s'\n", asset->name, path.c_str()); + if (!AppendPathPart(path, path_size, asset_name, true)) + { + return false; + } - auto itr = mapents_buffers.find(asset->name); - if (itr != mapents_buffers.end()) + if (extension != nullptr && !AppendPathPart(path, path_size, extension, false)) { - mapents_buffers.erase(itr); + return false; } - mapents_buffers[asset->name] = make_unique(); - itr = mapents_buffers.find(asset->name); - std::string *mapents_buffer = itr->second.get(); + return true; +} - mapents_buffer->assign(buffer); +OverrideCacheEntry *FindCacheEntry(XAssetType type, const char *name) +{ + if (name == nullptr || name[0] == '\0') + { + return nullptr; + } + + if (s_override_cache == nullptr) + { + return nullptr; + } + + for (size_t i = 0; i < s_override_cache_capacity; ++i) + { + OverrideCacheEntry &entry = s_override_cache[i]; + if (entry.state == CACHE_LOADED && entry.type == type && strcmp(entry.name, name) == 0) + { + return &entry; + } + } - asset->entityString = const_cast(mapents_buffer->c_str()); - asset->numEntityChars = static_cast(mapents_buffer->length()); + return nullptr; } -} // namespace MapEnts_ -namespace RawFile_ +OverrideCacheEntry *AllocateCacheEntry(XAssetType type, const char *name) { -struct RawFileOverride + if (s_override_cache == nullptr) + { + return nullptr; + } + + for (size_t i = 0; i < s_override_cache_capacity; ++i) + { + if (s_override_cache[i].state == CACHE_EMPTY) + { + OverrideCacheEntry *entry = &s_override_cache[i]; + memset(entry, 0, sizeof(*entry)); + entry->type = type; + if (!CopyAssetName(entry->name, sizeof(entry->name), name)) + { + memset(entry, 0, sizeof(*entry)); + return nullptr; + } + + return entry; + } + } + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override cache full. type=%d name='%s'\n", type, name); + return nullptr; +} + +void *AllocStorage(size_t size, const char *reason) { - std::string name; - std::string buffer; - RawFile asset; -}; + if (size == 0 || size > 0x7FFFFFFF) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] VirtualAlloc rejected for '%s'. size=%u\n", + reason ? reason : "", static_cast(size)); + return nullptr; + } -std::unordered_map> rawfile_overrides; + void *storage = AllocVirtualBlock(size, reason); + if (storage == nullptr) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] VirtualAlloc failed for '%s'. size=%u error=0x%08X\n", + reason ? reason : "", static_cast(size), GetLastError()); + } -std::string BuildRawFilePath(const char *asset_name) + return storage; +} + +bool FileExistsFast(const char *path) { - return Utils::BuildAssetPath(asset_name); + const DWORD attrs = GetFileAttributesA(path); + return attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY) == 0; } -RawFile *FindOverride(const char *asset_name) +bool LoadFileBuffer(const char *path, bool null_terminate, void *&storage, DWORD &storage_size, DWORD &payload_size) { - if (asset_name == nullptr || asset_name[0] == '\0') + storage = nullptr; + storage_size = 0; + payload_size = 0; + + HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) { - return nullptr; + return false; } - auto itr = rawfile_overrides.find(asset_name); - if (itr != rawfile_overrides.end()) + const DWORD file_size = GetFileSize(file, nullptr); + if (file_size == INVALID_FILE_SIZE || file_size > 0x7FFFFFFF) { - return itr->second ? &itr->second->asset : nullptr; + CloseHandle(file); + return false; } - const std::string path = BuildRawFilePath(asset_name); - if (path.empty()) + const DWORD extra_byte = null_terminate ? 1 : 0; + const size_t alloc_size = static_cast(file_size) + extra_byte; + storage = AllocStorage(alloc_size == 0 ? 1 : alloc_size, path); + if (storage == nullptr) { - rawfile_overrides[asset_name] = nullptr; - return nullptr; + CloseHandle(file); + return false; } - std::string buffer; - if (!Utils::ReadAssetOverride(path, buffer)) + if (file_size > 0) { - rawfile_overrides[asset_name] = nullptr; - return nullptr; + DWORD bytes_read = 0; + if (!ReadFile(file, storage, file_size, &bytes_read, nullptr) || bytes_read != file_size) + { + CloseHandle(file); + FreeVirtualBlock(storage, static_cast(alloc_size == 0 ? 1 : alloc_size), path); + storage = nullptr; + return false; + } } - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded rawfile '%s' from '%s'\n", asset_name, path.c_str()); + CloseHandle(file); - rawfile_overrides[asset_name] = make_unique(); - itr = rawfile_overrides.find(asset_name); - RawFileOverride *rawfile_override = itr->second.get(); + if (null_terminate) + { + static_cast(storage)[file_size] = '\0'; + } - rawfile_override->name = asset_name; - rawfile_override->buffer.assign(buffer); - rawfile_override->asset.name = rawfile_override->name.c_str(); - rawfile_override->asset.len = static_cast(rawfile_override->buffer.length()); - rawfile_override->asset.buffer = rawfile_override->buffer.c_str(); + storage_size = static_cast(alloc_size == 0 ? 1 : alloc_size); + payload_size = file_size; + return true; +} - return &rawfile_override->asset; +enum CsvReaderState +{ + CSV_START_OF_FIELD, + CSV_IN_FIELD, + CSV_IN_QUOTED_FIELD, + CSV_IN_ESCAPED_QUOTE, + CSV_END_OF_ROW, + CSV_EMPTY +}; + +struct CsvReader +{ + const char *data; + size_t size; + size_t cursor; + CsvReaderState state; +}; + +struct CsvStats +{ + size_t row_count; + size_t column_count; + size_t field_count; + size_t string_bytes; +}; + +bool IsCsvTerminator(char c) +{ + return c == '\r' || c == '\n'; } -void _override(RawFile *asset) +void SkipCsvLfAfterCr(CsvReader &reader, char c) { - if (!asset || !asset->name || asset->name[0] == '\0') + if (c == '\r' && reader.cursor < reader.size && reader.data[reader.cursor] == '\n') { - return; + ++reader.cursor; } +} - RawFile *override_asset = FindOverride(asset->name); - if (override_asset == nullptr) +bool AppendCsvChar(char *output, char *output_end, size_t &field_length, char c) +{ + if (field_length >= 0x7FFFFFFF) { - return; + return false; + } + + if (output != nullptr) + { + if (output + field_length >= output_end) + { + return false; + } + + output[field_length] = c; } - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding rawfile '%s'\n", asset->name); - asset->len = override_asset->len; - asset->buffer = override_asset->buffer; + ++field_length; + return true; } -} // namespace RawFile_ -namespace StringTable_ +bool TerminateCsvField(char *output, char *output_end, size_t field_length) { -struct StringTableOverride + if (output == nullptr) + { + return true; + } + + if (output + field_length >= output_end) + { + return false; + } + + output[field_length] = '\0'; + return true; +} + +bool ReadCsvField(CsvReader &reader, char *output, char *output_end, size_t &field_length, bool &row_end, + bool &csv_end) { - std::string name; - std::vector cells; - std::vector values; - StringTable asset; -}; + field_length = 0; + row_end = false; + csv_end = false; + + if (reader.state == CSV_EMPTY) + { + csv_end = true; + return true; + } + + if (reader.state == CSV_END_OF_ROW) + { + reader.state = CSV_START_OF_FIELD; + row_end = true; + return true; + } + + bool field_started = false; + for (;;) + { + if (reader.cursor >= reader.size) + { + reader.state = CSV_EMPTY; + if (!field_started && field_length == 0) + { + csv_end = true; + return true; + } + + return TerminateCsvField(output, output_end, field_length); + } + + const char c = reader.data[reader.cursor++]; + field_started = true; + + switch (reader.state) + { + case CSV_START_OF_FIELD: + if (IsCsvTerminator(c)) + { + SkipCsvLfAfterCr(reader, c); + reader.state = CSV_END_OF_ROW; + return TerminateCsvField(output, output_end, field_length); + } + + if (c == '"') + { + reader.state = CSV_IN_QUOTED_FIELD; + } + else if (c == ',') + { + return TerminateCsvField(output, output_end, field_length); + } + else + { + reader.state = CSV_IN_FIELD; + if (!AppendCsvChar(output, output_end, field_length, c)) + { + return false; + } + } + break; -std::unordered_map> stringtable_overrides; + case CSV_IN_FIELD: + if (IsCsvTerminator(c)) + { + SkipCsvLfAfterCr(reader, c); + reader.state = CSV_END_OF_ROW; + return TerminateCsvField(output, output_end, field_length); + } + + if (c == ',') + { + reader.state = CSV_START_OF_FIELD; + return TerminateCsvField(output, output_end, field_length); + } + + if (!AppendCsvChar(output, output_end, field_length, c)) + { + return false; + } + break; -bool LoadCsv(const std::string &path, std::vector> &rows) + case CSV_IN_QUOTED_FIELD: + if (c == '"') + { + reader.state = CSV_IN_ESCAPED_QUOTE; + } + else if (!AppendCsvChar(output, output_end, field_length, c)) + { + return false; + } + break; + + case CSV_IN_ESCAPED_QUOTE: + if (IsCsvTerminator(c)) + { + SkipCsvLfAfterCr(reader, c); + reader.state = CSV_END_OF_ROW; + return TerminateCsvField(output, output_end, field_length); + } + + if (c == '"') + { + reader.state = CSV_IN_QUOTED_FIELD; + if (!AppendCsvChar(output, output_end, field_length, c)) + { + return false; + } + } + else if (c == ',') + { + reader.state = CSV_START_OF_FIELD; + return TerminateCsvField(output, output_end, field_length); + } + else + { + reader.state = CSV_IN_FIELD; + if (!AppendCsvChar(output, output_end, field_length, c)) + { + return false; + } + } + break; + + default: + return false; + } + } +} + +bool AddCsvFieldToStats(CsvStats &stats, size_t ¤t_columns, size_t field_length) { - rows.clear(); + if (stats.field_count >= 0x7FFFFFFF || field_length > 0x7FFFFFFF || + stats.string_bytes > 0x7FFFFFFF - (field_length + 1)) + { + return false; + } - std::ifstream file(path.c_str(), std::ios::binary); - if (!file) + ++current_columns; + ++stats.field_count; + stats.string_bytes += field_length + 1; + return true; +} + +bool FinishCsvStatsRow(CsvStats &stats, size_t ¤t_columns) +{ + if (stats.row_count >= 0x7FFFFFFF || current_columns > 0x7FFFFFFF) { return false; } - try + ++stats.row_count; + if (current_columns > stats.column_count) { - aria::csv::CsvParser parser(file); - for (aria::csv::CsvParser::iterator itr = parser.begin(); itr != parser.end(); ++itr) + stats.column_count = current_columns; + } + + current_columns = 0; + return true; +} + +bool ScanStringTableCsvBuffer(const char *data, size_t data_size, CsvStats &stats) +{ + memset(&stats, 0, sizeof(stats)); + + CsvReader reader = {data, data_size, 0, CSV_START_OF_FIELD}; + size_t current_columns = 0; + for (;;) + { + size_t field_length = 0; + bool row_end = false; + bool csv_end = false; + if (!ReadCsvField(reader, nullptr, nullptr, field_length, row_end, csv_end)) { - rows.push_back(*itr); + return false; + } + + if (csv_end) + { + if (current_columns != 0 && !FinishCsvStatsRow(stats, current_columns)) + { + return false; + } + + break; } + + if (row_end) + { + if (!FinishCsvStatsRow(stats, current_columns)) + { + return false; + } + + continue; + } + + if (!AddCsvFieldToStats(stats, current_columns, field_length)) + { + return false; + } + } + + const size_t cell_count = stats.row_count * stats.column_count; + if (stats.column_count != 0 && cell_count / stats.column_count != stats.row_count) + { + return false; + } + + if (stats.field_count > cell_count) + { + return false; + } + + const size_t padded_empty_fields = cell_count - stats.field_count; + if (stats.string_bytes > 0x7FFFFFFF - padded_empty_fields) + { + return false; + } + + stats.string_bytes += padded_empty_fields; + return true; +} + +bool WriteEmptyStringCell(const char **values, size_t cell_index, char *&string_cursor, char *string_end) +{ + if (string_cursor >= string_end) + { + return false; } - catch (...) + + values[cell_index] = string_cursor; + *string_cursor++ = '\0'; + return true; +} + +bool FinishStringTableRow(const char **values, size_t row_count, size_t column_count, size_t &row_index, + size_t ¤t_column, char *&string_cursor, char *string_end) +{ + if (row_index >= row_count) { - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to parse stringtable csv '%s'\n", path.c_str()); - rows.clear(); return false; } + while (current_column < column_count) + { + if (!WriteEmptyStringCell(values, row_index * column_count + current_column, string_cursor, string_end)) + { + return false; + } + + ++current_column; + } + + ++row_index; + current_column = 0; return true; } -StringTable *FindOverride(const char *asset_name) +bool FillStringTableCsvBuffer(const char *data, size_t data_size, void *storage, size_t row_count, size_t column_count, + size_t values_bytes_aligned, size_t string_bytes) +{ + const size_t cell_count = row_count * column_count; + if (cell_count == 0) + { + return row_count == 0 && column_count == 0; + } + + const char **values = reinterpret_cast(storage); + char *string_cursor = static_cast(storage) + values_bytes_aligned; + char *string_end = string_cursor + string_bytes; + CsvReader reader = {data, data_size, 0, CSV_START_OF_FIELD}; + size_t row_index = 0; + size_t current_column = 0; + + for (;;) + { + char *field_start = string_cursor; + size_t field_length = 0; + bool row_end = false; + bool csv_end = false; + if (!ReadCsvField(reader, field_start, string_end, field_length, row_end, csv_end)) + { + return false; + } + + if (csv_end) + { + if (current_column != 0 && + !FinishStringTableRow(values, row_count, column_count, row_index, current_column, string_cursor, + string_end)) + { + return false; + } + + return row_index == row_count; + } + + if (row_end) + { + if (!FinishStringTableRow(values, row_count, column_count, row_index, current_column, string_cursor, + string_end)) + { + return false; + } + + continue; + } + + if (row_index >= row_count || current_column >= column_count) + { + return false; + } + + values[row_index * column_count + current_column] = field_start; + string_cursor = field_start + field_length + 1; + ++current_column; + } +} + +StringTable *LoadStringTableOverride(const char *asset_name) { - if (asset_name == nullptr || asset_name[0] == '\0') + OverrideCacheEntry *entry = FindCacheEntry(ASSET_TYPE_STRINGTABLE, asset_name); + if (entry != nullptr) + { + return &entry->stringtable; + } + + char path[MAX_PATH]; + if (!BuildAssetPath(path, sizeof(path), asset_name)) { return nullptr; } - auto itr = stringtable_overrides.find(asset_name); - if (itr != stringtable_overrides.end()) + if (!FileExistsFast(path)) { - return itr->second ? &itr->second->asset : nullptr; + return nullptr; + } + + void *file_storage = nullptr; + DWORD file_storage_size = 0; + DWORD file_size = 0; + if (!LoadFileBuffer(path, true, file_storage, file_storage_size, file_size)) + { + return nullptr; } - const std::string path = Utils::BuildAssetPath(asset_name); - if (path.empty()) + CsvStats stats; + if (!ScanStringTableCsvBuffer(static_cast(file_storage), file_size, stats)) { - stringtable_overrides[asset_name] = nullptr; + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to parse stringtable csv '%s'\n", path); + FreeVirtualBlock(file_storage, file_storage_size, path); return nullptr; } - std::vector> rows; - if (!LoadCsv(path, rows)) + const size_t cell_count = stats.row_count * stats.column_count; + const size_t values_bytes = cell_count * sizeof(const char *); + const size_t values_bytes_aligned = AlignSize(values_bytes, STRINGTABLE_VALUE_ALIGNMENT); + if (cell_count > 0x7FFFFFFF || values_bytes_aligned > 0x7FFFFFFF || + stats.string_bytes > 0x7FFFFFFF - values_bytes_aligned) { - stringtable_overrides[asset_name] = nullptr; + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Stringtable csv too large '%s'\n", path); + FreeVirtualBlock(file_storage, file_storage_size, path); return nullptr; } - size_t column_count = 0; - for (size_t row = 0; row < rows.size(); ++row) + void *storage = nullptr; + const size_t storage_size = values_bytes_aligned + stats.string_bytes; + if (storage_size > 0) { - if (rows[row].size() > column_count) + storage = AllocStorage(storage_size, path); + if (storage == nullptr) { - column_count = rows[row].size(); + FreeVirtualBlock(file_storage, file_storage_size, path); + return nullptr; } + + memset(storage, 0, storage_size); } - const size_t cell_count = rows.size() * column_count; - if (rows.size() > 0x7FFFFFFF || column_count > 0x7FFFFFFF || cell_count > 0x7FFFFFFF) + if (storage_size > 0 && + !FillStringTableCsvBuffer(static_cast(file_storage), file_size, storage, stats.row_count, + stats.column_count, values_bytes_aligned, stats.string_bytes)) { - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Stringtable csv too large '%s'\n", path.c_str()); - stringtable_overrides[asset_name] = nullptr; + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to build stringtable csv '%s'\n", path); + FreeVirtualBlock(file_storage, file_storage_size, path); + FreeVirtualBlock(storage, static_cast(storage_size), path); return nullptr; } - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded stringtable '%s' from '%s'\n", asset_name, path.c_str()); + FreeVirtualBlock(file_storage, file_storage_size, path); - stringtable_overrides[asset_name] = make_unique(); - itr = stringtable_overrides.find(asset_name); - StringTableOverride *stringtable_override = itr->second.get(); - - stringtable_override->name = asset_name; - stringtable_override->cells.resize(cell_count); - stringtable_override->values.resize(cell_count); - - for (size_t row = 0; row < rows.size(); ++row) + entry = AllocateCacheEntry(ASSET_TYPE_STRINGTABLE, asset_name); + if (entry == nullptr) { - for (size_t column = 0; column < column_count; ++column) + if (storage != nullptr) { - const size_t cell_index = row * column_count + column; - if (column < rows[row].size()) - { - stringtable_override->cells[cell_index] = rows[row][column]; - } + FreeVirtualBlock(storage, static_cast(storage_size), path); } + + return nullptr; } - for (size_t i = 0; i < cell_count; ++i) + entry->storage = storage; + entry->storage_size = static_cast(storage_size); + entry->payload_size = static_cast(storage_size); + entry->stringtable.name = entry->name; + entry->stringtable.columnCount = static_cast(stats.column_count); + entry->stringtable.rowCount = static_cast(stats.row_count); + entry->stringtable.values = cell_count == 0 ? nullptr : reinterpret_cast(storage); + entry->state = CACHE_LOADED; + + Com_Printf(CON_CHANNEL_FILES, + "[codxe][assets] Loaded stringtable override '%s' from '%s'. rows=%u columns=%u storage=%p size=%u\n", + asset_name, path, static_cast(stats.row_count), + static_cast(stats.column_count), entry->storage, entry->storage_size); + + return &entry->stringtable; +} + +OverrideCacheEntry *LoadMapEntsOverride(const char *asset_name) +{ + OverrideCacheEntry *entry = FindCacheEntry(ASSET_TYPE_MAP_ENTS, asset_name); + if (entry != nullptr) { - stringtable_override->values[i] = stringtable_override->cells[i].c_str(); + return entry; } - stringtable_override->asset.name = stringtable_override->name.c_str(); - stringtable_override->asset.columnCount = static_cast(column_count); - stringtable_override->asset.rowCount = static_cast(rows.size()); - stringtable_override->asset.values = - stringtable_override->values.empty() ? nullptr : &stringtable_override->values[0]; + char path[MAX_PATH]; + if (!BuildAssetPath(path, sizeof(path), asset_name, ".ents")) + { + return nullptr; + } - return &stringtable_override->asset; + void *storage = nullptr; + DWORD storage_size = 0; + DWORD payload_size = 0; + if (!LoadFileBuffer(path, true, storage, storage_size, payload_size)) + { + return nullptr; + } + + entry = AllocateCacheEntry(ASSET_TYPE_MAP_ENTS, asset_name); + if (entry == nullptr) + { + FreeVirtualBlock(storage, storage_size, path); + return nullptr; + } + + entry->storage = storage; + entry->storage_size = storage_size; + entry->payload_size = payload_size; + entry->state = CACHE_LOADED; + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded map ents override '%s' from '%s'. storage=%p size=%u\n", + asset_name, path, entry->storage, entry->payload_size); + + return entry; } -void _override(StringTable *asset) +void OverrideMapEnts(MapEnts *asset) { - if (!asset || !asset->name || asset->name[0] == '\0') + if (asset == nullptr || asset->name == nullptr || asset->name[0] == '\0') + { + return; + } + + OverrideCacheEntry *entry = LoadMapEntsOverride(asset->name); + if (entry == nullptr || entry->state != CACHE_LOADED) { return; } - StringTable *override_asset = FindOverride(asset->name); + asset->entityString = static_cast(entry->storage); + asset->numEntityChars = static_cast(entry->payload_size); + + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override applied. type=mapents name='%s'\n", asset->name); +} + +void OverrideStringTable(StringTable *asset) +{ + if (asset == nullptr || asset->name == nullptr || asset->name[0] == '\0') + { + return; + } + + StringTable *override_asset = LoadStringTableOverride(asset->name); if (override_asset == nullptr) { return; } - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Overriding stringtable '%s'\n", asset->name); asset->columnCount = override_asset->columnCount; asset->rowCount = override_asset->rowCount; asset->values = override_asset->values; -} -} // namespace StringTable_ -} // namespace Assets -Detour DB_LinkXAssetEntry_Detour; -Detour DB_FindXAssetHeader_Detour; + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override applied. type=stringtable name='%s'\n", asset->name); +} XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) { @@ -339,13 +1026,10 @@ XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) switch (newEntry->asset.type) { case ASSET_TYPE_MAP_ENTS: - Assets::MapEnts_::_override(newEntry->asset.header.mapEnts); - break; - case ASSET_TYPE_RAWFILE: - Assets::RawFile_::_override(newEntry->asset.header.rawfile); + OverrideMapEnts(newEntry->asset.header.mapEnts); break; case ASSET_TYPE_STRINGTABLE: - Assets::StringTable_::_override(newEntry->asset.header.stringTable); + OverrideStringTable(newEntry->asset.header.stringTable); break; } } @@ -353,46 +1037,25 @@ XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) return DB_LinkXAssetEntry_Detour.GetOriginal()(newEntry, allowOverride); } -XAssetHeader DB_FindXAssetHeader_Hook(const XAssetType type, const char *name) -{ - if (type == ASSET_TYPE_RAWFILE) - { - RawFile *rawfile = Assets::RawFile_::FindOverride(name); - if (rawfile != nullptr) - { - XAssetHeader header; - header.rawfile = rawfile; - return header; - } - } - else if (type == ASSET_TYPE_STRINGTABLE) - { - StringTable *stringtable = Assets::StringTable_::FindOverride(name); - if (stringtable != nullptr) - { - XAssetHeader header; - header.stringTable = stringtable; - return header; - } - } - - return DB_FindXAssetHeader_Detour.GetOriginal()(type, name); -} +} // namespace assets::assets() { + DbgPrint("[codxe][assets] ctor begin.\n"); + InitializeOverrideCache(); + DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); DB_LinkXAssetEntry_Detour.Install(); - - DB_FindXAssetHeader_Detour = Detour(DB_FindXAssetHeader, DB_FindXAssetHeader_Hook); - DB_FindXAssetHeader_Detour.Install(); + DbgPrint("[codxe][assets] ctor end.\n"); } assets::~assets() { - DB_FindXAssetHeader_Detour.Remove(); - + DbgPrint("[codxe][assets] dtor begin.\n"); DB_LinkXAssetEntry_Detour.Remove(); + + ShutdownOverrideCache(); + DbgPrint("[codxe][assets] dtor end.\n"); } } // namespace mp } // namespace iw3 From e0c4228e4a2d39eb07cac8bb33f5afd5a7fc9a16 Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Wed, 24 Jun 2026 10:31:43 +0100 Subject: [PATCH 4/6] modify csv parser to keep a static buffer --- src/third_party/aria_csv/csv_parser.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/third_party/aria_csv/csv_parser.hpp b/src/third_party/aria_csv/csv_parser.hpp index b589d435..ba117a2d 100644 --- a/src/third_party/aria_csv/csv_parser.hpp +++ b/src/third_party/aria_csv/csv_parser.hpp @@ -93,12 +93,11 @@ class CsvParser std::istream &m_input; // Buffer capacities - static const int FIELDBUF_CAP = 1024; - static const int INPUTBUF_CAP = 1024 * 128; + static const int INPUTBUF_CAP = 4096; // Buffers std::string m_fieldbuf; - std::unique_ptr m_inputbuf; + char m_inputbuf[INPUTBUF_CAP]; // Misc bool m_eof; @@ -106,17 +105,17 @@ class CsvParser size_t m_inputbuf_size; std::streamoff m_scanposition; + CsvParser(const CsvParser &); + CsvParser &operator=(const CsvParser &); + public: // Creates the CSV parser which by default, splits on commas, // uses quotes to escape, and handles CSV files that end in either // '\r', '\n', or '\r\n'. explicit CsvParser(std::istream &input) : m_state(State::START_OF_FIELD), m_quote('"'), m_delimiter(','), m_terminator(Term::CRLF), m_input(input), - m_inputbuf(new char[INPUTBUF_CAP]()), m_eof(false), m_cursor(INPUTBUF_CAP), m_inputbuf_size(INPUTBUF_CAP), - m_scanposition(-INPUTBUF_CAP) + m_eof(false), m_cursor(INPUTBUF_CAP), m_inputbuf_size(INPUTBUF_CAP), m_scanposition(-INPUTBUF_CAP) { - // Reserve space upfront to improve performance - m_fieldbuf.reserve(FIELDBUF_CAP); if (!m_input.good()) { throw std::runtime_error("Something is wrong with input stream"); @@ -316,7 +315,7 @@ class CsvParser { m_scanposition += static_cast(m_cursor); m_cursor = 0; - m_input.read(m_inputbuf.get(), INPUTBUF_CAP); + m_input.read(m_inputbuf, INPUTBUF_CAP); // Indicate we hit end of file, and resize // input buffer to show that it's not at full capacity From 8862605c2b08e16b29af9e78d177027531b0d288 Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Wed, 24 Jun 2026 10:32:01 +0100 Subject: [PATCH 5/6] rework csv override to avoid heap allocations --- src/game/iw3/mp/components/assets.cpp | 603 +++++++++----------------- 1 file changed, 210 insertions(+), 393 deletions(-) diff --git a/src/game/iw3/mp/components/assets.cpp b/src/game/iw3/mp/components/assets.cpp index 61e28f53..9d7593c4 100644 --- a/src/game/iw3/mp/components/assets.cpp +++ b/src/game/iw3/mp/components/assets.cpp @@ -1,13 +1,9 @@ +#include + #include "pch.h" #include "assets.h" +#include "third_party/aria_csv/csv_parser.hpp" -#ifndef INVALID_FILE_ATTRIBUTES -#define INVALID_FILE_ATTRIBUTES ((DWORD) - 1) -#endif - -#ifndef INVALID_FILE_SIZE -#define INVALID_FILE_SIZE ((DWORD) - 1) -#endif namespace iw3 { @@ -34,6 +30,7 @@ struct OverrideCacheEntry char name[MAX_PATH]; void *storage; DWORD storage_size; + DWORD storage_process_type; DWORD payload_size; StringTable stringtable; }; @@ -41,11 +38,6 @@ struct OverrideCacheEntry OverrideCacheEntry *s_override_cache = nullptr; size_t s_override_cache_capacity = 0; -const char *ProcessTypeName(DWORD process_type) -{ - return process_type == PROC_TYPE_SYSTEM ? "system" : "title"; -} - size_t AlignSize(size_t value, size_t alignment) { return (value + alignment - 1) & ~(alignment - 1); @@ -53,25 +45,8 @@ size_t AlignSize(size_t value, size_t alignment) void *AllocVirtualBlock(size_t size, const char *reason) { - const DWORD process_type = KeGetCurrentProcessType(); - DbgPrint("[codxe][assets] VirtualAlloc begin. reason='%s' size=%u processType=%u(%s)\n", - reason ? reason : "", static_cast(size), process_type, ProcessTypeName(process_type)); - void *storage = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (storage == nullptr) - { - const DWORD error = GetLastError(); - DbgPrint("[codxe][assets] VirtualAlloc failed. reason='%s' size=%u processType=%u(%s) error=0x%08X\n", - reason ? reason : "", static_cast(size), process_type, - ProcessTypeName(process_type), error); - } - else - { - DbgPrint("[codxe][assets] VirtualAlloc end. reason='%s' ptr=%p size=%u processType=%u(%s)\n", - reason ? reason : "", storage, static_cast(size), process_type, - ProcessTypeName(process_type)); - } - + UNREFERENCED_PARAMETER(reason); return storage; } @@ -82,92 +57,44 @@ bool FreeVirtualBlock(void *storage, DWORD size, const char *reason) return true; } - const DWORD process_type = KeGetCurrentProcessType(); - DbgPrint("[codxe][assets] VirtualFree begin. reason='%s' ptr=%p size=%u processType=%u(%s)\n", - reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type)); - const BOOL result = VirtualFree(storage, 0, MEM_RELEASE); - if (!result) - { - const DWORD error = GetLastError(); - DbgPrint("[codxe][assets] VirtualFree failed. reason='%s' ptr=%p size=%u processType=%u(%s) error=0x%08X\n", - reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type), error); - return false; - } - - DbgPrint("[codxe][assets] VirtualFree end. reason='%s' ptr=%p size=%u processType=%u(%s)\n", - reason ? reason : "", storage, size, process_type, ProcessTypeName(process_type)); - return true; + UNREFERENCED_PARAMETER(size); + UNREFERENCED_PARAMETER(reason); + return result != FALSE; } -bool ResetCacheEntry(OverrideCacheEntry &entry, DWORD &storage_bytes) +void ResetCacheEntry(OverrideCacheEntry &entry) { - bool freed = false; if (entry.storage != nullptr) { - if (FreeVirtualBlock(entry.storage, entry.storage_size, entry.name)) + const DWORD current_process_type = KeGetCurrentProcessType(); + if (entry.storage_process_type == 0 || entry.storage_process_type == current_process_type) { - storage_bytes += entry.storage_size; - freed = true; + FreeVirtualBlock(entry.storage, entry.storage_size, entry.name); } } memset(&entry, 0, sizeof(entry)); - return freed; } void ClearOverrideCache() { - DbgPrint("[codxe][assets] ClearOverrideCache begin. cache=%p capacity=%u\n", s_override_cache, - static_cast(s_override_cache_capacity)); - if (s_override_cache == nullptr) { - DbgPrint("[codxe][assets] ClearOverrideCache end. no cache allocated.\n"); return; } - unsigned int loaded_entries = 0; - unsigned int free_attempts = 0; - unsigned int free_successes = 0; - unsigned int free_failures = 0; - DWORD freed_storage_bytes = 0; for (size_t i = 0; i < s_override_cache_capacity; ++i) { - if (s_override_cache[i].state == CACHE_LOADED) - { - ++loaded_entries; - } - - if (s_override_cache[i].storage != nullptr) - { - ++free_attempts; - } - - if (ResetCacheEntry(s_override_cache[i], freed_storage_bytes)) - { - ++free_successes; - } - else if (free_attempts != free_successes + free_failures) - { - ++free_failures; - } + ResetCacheEntry(s_override_cache[i]); } - - DbgPrint("[codxe][assets] ClearOverrideCache end. loaded=%u freeAttempts=%u freeSuccesses=%u freeFailures=%u " - "storageBytes=%u\n", - loaded_entries, free_attempts, free_successes, free_failures, freed_storage_bytes); } bool InitializeOverrideCache() { - DbgPrint("[codxe][assets] InitializeOverrideCache begin. existing=%p capacity=%u\n", s_override_cache, - static_cast(s_override_cache_capacity)); - if (s_override_cache != nullptr) { ClearOverrideCache(); - DbgPrint("[codxe][assets] InitializeOverrideCache end. reused existing cache.\n"); return true; } @@ -179,46 +106,31 @@ bool InitializeOverrideCache() "[codxe][assets] Failed to allocate override cache. entries=%u bytes=%u error=0x%08X\n", static_cast(MAX_OVERRIDE_CACHE_ENTRIES), static_cast(cache_size), GetLastError()); - DbgPrint("[codxe][assets] InitializeOverrideCache end. failed bytes=%u error=0x%08X\n", - static_cast(cache_size), GetLastError()); s_override_cache_capacity = 0; return false; } s_override_cache_capacity = MAX_OVERRIDE_CACHE_ENTRIES; memset(s_override_cache, 0, cache_size); - - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override cache allocated. entries=%u bytes=%u storage=%p\n", - static_cast(s_override_cache_capacity), static_cast(cache_size), - s_override_cache); - DbgPrint("[codxe][assets] InitializeOverrideCache end. cache=%p bytes=%u\n", s_override_cache, - static_cast(cache_size)); return true; } void ShutdownOverrideCache() { - DbgPrint("[codxe][assets] ShutdownOverrideCache begin. cache=%p capacity=%u\n", s_override_cache, - static_cast(s_override_cache_capacity)); - const DWORD cache_bytes = static_cast(sizeof(OverrideCacheEntry) * s_override_cache_capacity); ClearOverrideCache(); if (s_override_cache != nullptr) { - if (FreeVirtualBlock(s_override_cache, cache_bytes, "override cache metadata")) + if (!FreeVirtualBlock(s_override_cache, cache_bytes, "override cache metadata")) { - DbgPrint("[codxe][assets] ShutdownOverrideCache freed cache metadata. bytes=%u\n", cache_bytes); - } - else - { - DbgPrint("[codxe][assets] ShutdownOverrideCache failed to free cache metadata. bytes=%u\n", cache_bytes); + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Failed to free override cache metadata. bytes=%u\n", + cache_bytes); } } s_override_cache = nullptr; s_override_cache_capacity = 0; - DbgPrint("[codxe][assets] ShutdownOverrideCache end.\n"); } bool CopyAssetName(char *dest, size_t dest_size, const char *name) @@ -310,6 +222,60 @@ bool BuildAssetPath(char *path, size_t path_size, const char *asset_name, const return true; } +bool BuildSourceDisplayPath(char *path, size_t path_size, const char *asset_name, const char *extension = nullptr) +{ + if (path == nullptr || path_size == 0) + { + return false; + } + + path[0] = '\0'; + if (Config::active_mod.empty() || asset_name == nullptr || asset_name[0] == '\0') + { + return false; + } + + if (!AppendPathPart(path, path_size, "mods\\", false)) + { + return false; + } + + if (!AppendPathPart(path, path_size, Config::active_mod.c_str(), false)) + { + return false; + } + + if (!AppendPathPart(path, path_size, "\\", false)) + { + return false; + } + + if (!AppendPathPart(path, path_size, asset_name, true)) + { + return false; + } + + if (extension != nullptr && !AppendPathPart(path, path_size, extension, false)) + { + return false; + } + + return true; +} + +void PrintOverrideApplied(const char *type, const char *asset_name, const char *source) +{ + char display_name[MAX_PATH]; + display_name[0] = '\0'; + const char *name = asset_name; + if (AppendPathPart(display_name, sizeof(display_name), asset_name, true)) + { + name = display_name; + } + + Com_Printf(CON_CHANNEL_FILES, "^2codxe^7: %s \"%s\" -> \"%s\"\n", type, name, source); +} + OverrideCacheEntry *FindCacheEntry(XAssetType type, const char *name) { if (name == nullptr || name[0] == '\0') @@ -384,7 +350,7 @@ void *AllocStorage(size_t size, const char *reason) bool FileExistsFast(const char *path) { const DWORD attrs = GetFileAttributesA(path); - return attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY) == 0; + return attrs != INVALID_FILE_SIZE && (attrs & FILE_ATTRIBUTE_DIRECTORY) == 0; } bool LoadFileBuffer(const char *path, bool null_terminate, void *&storage, DWORD &storage_size, DWORD &payload_size) @@ -440,24 +406,6 @@ bool LoadFileBuffer(const char *path, bool null_terminate, void *&storage, DWORD return true; } -enum CsvReaderState -{ - CSV_START_OF_FIELD, - CSV_IN_FIELD, - CSV_IN_QUOTED_FIELD, - CSV_IN_ESCAPED_QUOTE, - CSV_END_OF_ROW, - CSV_EMPTY -}; - -struct CsvReader -{ - const char *data; - size_t size; - size_t cursor; - CsvReaderState state; -}; - struct CsvStats { size_t row_count; @@ -466,192 +414,19 @@ struct CsvStats size_t string_bytes; }; -bool IsCsvTerminator(char c) -{ - return c == '\r' || c == '\n'; -} - -void SkipCsvLfAfterCr(CsvReader &reader, char c) -{ - if (c == '\r' && reader.cursor < reader.size && reader.data[reader.cursor] == '\n') - { - ++reader.cursor; - } -} - -bool AppendCsvChar(char *output, char *output_end, size_t &field_length, char c) -{ - if (field_length >= 0x7FFFFFFF) - { - return false; - } - - if (output != nullptr) - { - if (output + field_length >= output_end) - { - return false; - } - - output[field_length] = c; - } - - ++field_length; - return true; -} - -bool TerminateCsvField(char *output, char *output_end, size_t field_length) -{ - if (output == nullptr) - { - return true; - } - - if (output + field_length >= output_end) - { - return false; - } - - output[field_length] = '\0'; - return true; -} - -bool ReadCsvField(CsvReader &reader, char *output, char *output_end, size_t &field_length, bool &row_end, - bool &csv_end) +class MemoryCsvStreamBuf : public std::streambuf { - field_length = 0; - row_end = false; - csv_end = false; - - if (reader.state == CSV_EMPTY) + public: + MemoryCsvStreamBuf(const char *data, size_t size) { - csv_end = true; - return true; - } - - if (reader.state == CSV_END_OF_ROW) - { - reader.state = CSV_START_OF_FIELD; - row_end = true; - return true; - } - - bool field_started = false; - for (;;) - { - if (reader.cursor >= reader.size) - { - reader.state = CSV_EMPTY; - if (!field_started && field_length == 0) - { - csv_end = true; - return true; - } - - return TerminateCsvField(output, output_end, field_length); - } - - const char c = reader.data[reader.cursor++]; - field_started = true; - - switch (reader.state) - { - case CSV_START_OF_FIELD: - if (IsCsvTerminator(c)) - { - SkipCsvLfAfterCr(reader, c); - reader.state = CSV_END_OF_ROW; - return TerminateCsvField(output, output_end, field_length); - } - - if (c == '"') - { - reader.state = CSV_IN_QUOTED_FIELD; - } - else if (c == ',') - { - return TerminateCsvField(output, output_end, field_length); - } - else - { - reader.state = CSV_IN_FIELD; - if (!AppendCsvChar(output, output_end, field_length, c)) - { - return false; - } - } - break; - - case CSV_IN_FIELD: - if (IsCsvTerminator(c)) - { - SkipCsvLfAfterCr(reader, c); - reader.state = CSV_END_OF_ROW; - return TerminateCsvField(output, output_end, field_length); - } - - if (c == ',') - { - reader.state = CSV_START_OF_FIELD; - return TerminateCsvField(output, output_end, field_length); - } - - if (!AppendCsvChar(output, output_end, field_length, c)) - { - return false; - } - break; - - case CSV_IN_QUOTED_FIELD: - if (c == '"') - { - reader.state = CSV_IN_ESCAPED_QUOTE; - } - else if (!AppendCsvChar(output, output_end, field_length, c)) - { - return false; - } - break; - - case CSV_IN_ESCAPED_QUOTE: - if (IsCsvTerminator(c)) - { - SkipCsvLfAfterCr(reader, c); - reader.state = CSV_END_OF_ROW; - return TerminateCsvField(output, output_end, field_length); - } - - if (c == '"') - { - reader.state = CSV_IN_QUOTED_FIELD; - if (!AppendCsvChar(output, output_end, field_length, c)) - { - return false; - } - } - else if (c == ',') - { - reader.state = CSV_START_OF_FIELD; - return TerminateCsvField(output, output_end, field_length); - } - else - { - reader.state = CSV_IN_FIELD; - if (!AppendCsvChar(output, output_end, field_length, c)) - { - return false; - } - } - break; - - default: - return false; - } + char *begin = const_cast(data); + setg(begin, begin, begin + size); } -} +}; -bool AddCsvFieldToStats(CsvStats &stats, size_t ¤t_columns, size_t field_length) +bool AddCsvFieldToStats(CsvStats &stats, size_t ¤t_columns, const std::string &field) { + const size_t field_length = field.size(); if (stats.field_count >= 0x7FFFFFFF || field_length > 0x7FFFFFFF || stats.string_bytes > 0x7FFFFFFF - (field_length + 1)) { @@ -681,48 +456,8 @@ bool FinishCsvStatsRow(CsvStats &stats, size_t ¤t_columns) return true; } -bool ScanStringTableCsvBuffer(const char *data, size_t data_size, CsvStats &stats) +bool FinalizeCsvStats(CsvStats &stats) { - memset(&stats, 0, sizeof(stats)); - - CsvReader reader = {data, data_size, 0, CSV_START_OF_FIELD}; - size_t current_columns = 0; - for (;;) - { - size_t field_length = 0; - bool row_end = false; - bool csv_end = false; - if (!ReadCsvField(reader, nullptr, nullptr, field_length, row_end, csv_end)) - { - return false; - } - - if (csv_end) - { - if (current_columns != 0 && !FinishCsvStatsRow(stats, current_columns)) - { - return false; - } - - break; - } - - if (row_end) - { - if (!FinishCsvStatsRow(stats, current_columns)) - { - return false; - } - - continue; - } - - if (!AddCsvFieldToStats(stats, current_columns, field_length)) - { - return false; - } - } - const size_t cell_count = stats.row_count * stats.column_count; if (stats.column_count != 0 && cell_count / stats.column_count != stats.row_count) { @@ -744,6 +479,56 @@ bool ScanStringTableCsvBuffer(const char *data, size_t data_size, CsvStats &stat return true; } +bool ScanStringTableCsvBuffer(const char *data, size_t data_size, CsvStats &stats) +{ + memset(&stats, 0, sizeof(stats)); + + MemoryCsvStreamBuf streambuf(data, data_size); + std::istream input(&streambuf); + + try + { + aria::csv::CsvParser parser(input); + size_t current_columns = 0; + + for (;;) + { + const aria::csv::Field field = parser.next_field(); + switch (field.type) + { + case aria::csv::FieldType::CSV_END: + if (current_columns != 0 && !FinishCsvStatsRow(stats, current_columns)) + { + return false; + } + + return FinalizeCsvStats(stats); + + case aria::csv::FieldType::ROW_END: + if (!FinishCsvStatsRow(stats, current_columns)) + { + return false; + } + + break; + + case aria::csv::FieldType::DATA: + if (field.data == nullptr || !AddCsvFieldToStats(stats, current_columns, *field.data)) + { + return false; + } + + break; + } + } + } + catch (const std::exception &e) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] aria csv scan exception: %s\n", e.what()); + return false; + } +} + bool WriteEmptyStringCell(const char **values, size_t cell_index, char *&string_cursor, char *string_end) { if (string_cursor >= string_end) @@ -779,6 +564,38 @@ bool FinishStringTableRow(const char **values, size_t row_count, size_t column_c return true; } +bool WriteStringTableField(const std::string &field, const char **values, size_t row_count, size_t column_count, + size_t row_index, size_t ¤t_column, char *&string_cursor, char *string_end) +{ + if (row_index >= row_count || current_column >= column_count) + { + return false; + } + + const size_t field_length = field.size(); + if (field_length > 0x7FFFFFFF) + { + return false; + } + + const size_t remaining = static_cast(string_end - string_cursor); + if (remaining < field_length + 1) + { + return false; + } + + values[row_index * column_count + current_column] = string_cursor; + if (field_length > 0) + { + memcpy(string_cursor, field.c_str(), field_length); + } + + string_cursor[field_length] = '\0'; + string_cursor += field_length + 1; + ++current_column; + return true; +} + bool FillStringTableCsvBuffer(const char *data, size_t data_size, void *storage, size_t row_count, size_t column_count, size_t values_bytes_aligned, size_t string_bytes) { @@ -791,52 +608,54 @@ bool FillStringTableCsvBuffer(const char *data, size_t data_size, void *storage, const char **values = reinterpret_cast(storage); char *string_cursor = static_cast(storage) + values_bytes_aligned; char *string_end = string_cursor + string_bytes; - CsvReader reader = {data, data_size, 0, CSV_START_OF_FIELD}; + MemoryCsvStreamBuf streambuf(data, data_size); + std::istream input(&streambuf); size_t row_index = 0; size_t current_column = 0; - for (;;) + try { - char *field_start = string_cursor; - size_t field_length = 0; - bool row_end = false; - bool csv_end = false; - if (!ReadCsvField(reader, field_start, string_end, field_length, row_end, csv_end)) - { - return false; - } + aria::csv::CsvParser parser(input); - if (csv_end) + for (;;) { - if (current_column != 0 && - !FinishStringTableRow(values, row_count, column_count, row_index, current_column, string_cursor, - string_end)) + const aria::csv::Field field = parser.next_field(); + switch (field.type) { - return false; - } + case aria::csv::FieldType::CSV_END: + if (current_column != 0 && !FinishStringTableRow(values, row_count, column_count, row_index, + current_column, string_cursor, string_end)) + { + return false; + } - return row_index == row_count; - } + return row_index == row_count; - if (row_end) - { - if (!FinishStringTableRow(values, row_count, column_count, row_index, current_column, string_cursor, - string_end)) - { - return false; - } + case aria::csv::FieldType::ROW_END: + if (!FinishStringTableRow(values, row_count, column_count, row_index, current_column, string_cursor, + string_end)) + { + return false; + } - continue; - } + break; - if (row_index >= row_count || current_column >= column_count) - { - return false; - } + case aria::csv::FieldType::DATA: + if (field.data == nullptr || + !WriteStringTableField(*field.data, values, row_count, column_count, row_index, current_column, + string_cursor, string_end)) + { + return false; + } - values[row_index * column_count + current_column] = field_start; - string_cursor = field_start + field_length + 1; - ++current_column; + break; + } + } + } + catch (const std::exception &e) + { + Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] aria csv fill exception: %s\n", e.what()); + return false; } } @@ -925,6 +744,7 @@ StringTable *LoadStringTableOverride(const char *asset_name) entry->storage = storage; entry->storage_size = static_cast(storage_size); + entry->storage_process_type = KeGetCurrentProcessType(); entry->payload_size = static_cast(storage_size); entry->stringtable.name = entry->name; entry->stringtable.columnCount = static_cast(stats.column_count); @@ -932,11 +752,6 @@ StringTable *LoadStringTableOverride(const char *asset_name) entry->stringtable.values = cell_count == 0 ? nullptr : reinterpret_cast(storage); entry->state = CACHE_LOADED; - Com_Printf(CON_CHANNEL_FILES, - "[codxe][assets] Loaded stringtable override '%s' from '%s'. rows=%u columns=%u storage=%p size=%u\n", - asset_name, path, static_cast(stats.row_count), - static_cast(stats.column_count), entry->storage, entry->storage_size); - return &entry->stringtable; } @@ -971,12 +786,10 @@ OverrideCacheEntry *LoadMapEntsOverride(const char *asset_name) entry->storage = storage; entry->storage_size = storage_size; + entry->storage_process_type = KeGetCurrentProcessType(); entry->payload_size = payload_size; entry->state = CACHE_LOADED; - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Loaded map ents override '%s' from '%s'. storage=%p size=%u\n", - asset_name, path, entry->storage, entry->payload_size); - return entry; } @@ -996,7 +809,11 @@ void OverrideMapEnts(MapEnts *asset) asset->entityString = static_cast(entry->storage); asset->numEntityChars = static_cast(entry->payload_size); - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override applied. type=mapents name='%s'\n", asset->name); + char source[MAX_PATH]; + if (BuildSourceDisplayPath(source, sizeof(source), asset->name, ".ents")) + { + PrintOverrideApplied("mapents", asset->name, source); + } } void OverrideStringTable(StringTable *asset) @@ -1016,7 +833,11 @@ void OverrideStringTable(StringTable *asset) asset->rowCount = override_asset->rowCount; asset->values = override_asset->values; - Com_Printf(CON_CHANNEL_FILES, "[codxe][assets] Override applied. type=stringtable name='%s'\n", asset->name); + char source[MAX_PATH]; + if (BuildSourceDisplayPath(source, sizeof(source), asset->name)) + { + PrintOverrideApplied("stringtable", asset->name, source); + } } XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) @@ -1041,21 +862,17 @@ XAssetEntry *DB_LinkXAssetEntry_Hook(XAssetEntry *newEntry, int allowOverride) assets::assets() { - DbgPrint("[codxe][assets] ctor begin.\n"); InitializeOverrideCache(); DB_LinkXAssetEntry_Detour = Detour(DB_LinkXAssetEntry, DB_LinkXAssetEntry_Hook); DB_LinkXAssetEntry_Detour.Install(); - DbgPrint("[codxe][assets] ctor end.\n"); } assets::~assets() { - DbgPrint("[codxe][assets] dtor begin.\n"); DB_LinkXAssetEntry_Detour.Remove(); ShutdownOverrideCache(); - DbgPrint("[codxe][assets] dtor end.\n"); } } // namespace mp } // namespace iw3 From 6f456235f26825288e5ff286debf015427b784cf Mon Sep 17 00:00:00 2001 From: Michael Oliver Date: Wed, 24 Jun 2026 10:46:39 +0100 Subject: [PATCH 6/6] format --- src/game/iw3/mp/components/assets.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/game/iw3/mp/components/assets.cpp b/src/game/iw3/mp/components/assets.cpp index 9d7593c4..5077f43d 100644 --- a/src/game/iw3/mp/components/assets.cpp +++ b/src/game/iw3/mp/components/assets.cpp @@ -4,7 +4,6 @@ #include "assets.h" #include "third_party/aria_csv/csv_parser.hpp" - namespace iw3 { namespace mp