diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..e3c90c46 --- /dev/null +++ b/.clang-format @@ -0,0 +1,16 @@ +--- +BasedOnStyle: LLVM +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ColumnLimit: 100 +AlignAfterOpenBracket: Align +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +BreakBeforeBraces: Attach +NamespaceIndentation: None +PointerAlignment: Left +SpacesBeforeTrailingComments: 2 +IncludeBlocks: Preserve +SortIncludes: Never diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000..c7630d0a --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,19 @@ +--- +Checks: > + -*, + bugprone-*, + -bugprone-easily-swappable-parameters, + -bugprone-macro-parentheses, + -bugprone-narrowing-conversions, + -bugprone-branch-clone, + -bugprone-signed-char-misuse, + cppcoreguidelines-pro-type-static-cast-downcast, + cppcoreguidelines-slicing, + misc-const-correctness, + misc-misplaced-const, + performance-*, + -performance-no-int-to-ptr, + readability-misleading-indentation, + readability-redundant-smartptr-get +WarningsAsErrors: '' +HeaderFilterRegex: '^.*/(src|include)/(?!pcre|s7|networking-ts-impl|ffi|shims|linenoise).*' diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index af8ad811..f8d850f0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -72,4 +72,4 @@ jobs: - name: Test timeout-minutes: 30 - run: ctest --test-dir build --build-config Release --label-regex "libs-core|libs-external|compiler-unit|examples-core|examples-audio" --output-on-failure + run: ctest --test-dir build --build-config Release --label-regex "libs-core|libs-external|compiler-unit|cpp-unit|examples-core|examples-audio" --output-on-failure diff --git a/.gitignore b/.gitignore index e488d596..11d28e9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# extmpore binary executable +# extempore binary executable extempore extempore.exe extempore.lib @@ -7,6 +7,12 @@ extempore.dll extempore.dylib libextempore.so +# local dev state (per-session assistants, sanitizer builds) +.remember/ +/build-asan/ +/build-ubsan/ +/build-tsan/ + # boost /boost diff --git a/BUILDING.md b/BUILDING.md index e7766339..a6d09cc8 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -6,112 +6,100 @@ in more depth, here's some more information. ## Build-time deps -- a C++ compiler (`clang`, `gcc` >= 4.9, `msvc` >= VS2015) +- a C++17 compiler (recent `clang` or `gcc`; MSVC from Visual Studio 2022) - Git -- CMake >= 3.19 -- Python >= 2.7 (for LLVM) +- CMake >= 3.28 +- Ninja (recommended on Linux/macOS) +- Python >= 3.8 (for LLVM) -For platform-specific deps, see "Platform-specific notes" below. +## Build options -## Options +See the top of `CMakeLists.txt` for all the available options. The ones most +users will care about: -See the top of `CMakeLists.txt` for all the available build options. +- `ASSETS` (default `OFF`) --- download the multimedia assets (audio samples, + impulse responses) needed to run many of the examples. ~250MB download. If + you don't set `-DASSETS=ON` at configure time, CMake still creates an + `assets` target you can build afterwards. +- `BUILD_TESTS` (default `ON`) --- build the test targets (including examples + registered as ctest tests). +- `EXTERNAL_SHLIBS_AUDIO` (default `ON`) --- build the audio dependencies + (portaudio, portmidi, sndfile, kissfft). +- `EXTERNAL_SHLIBS_GRAPHICS` (default `OFF`) --- build the WebGPU graphics + stack (glfw, wgpu-native, stb_image). Required for the WebGPU examples. +- `JACK` (default `OFF`) --- use the Jack PortAudio backend on Linux instead + of ALSA. -The most relevant option for new Extempore users is the `ASSETS` option. It's -off by default, but if set to `ON` the Extempore build process will download a -bunch of assets (e.g. sound files, 3D model files) which are necessary to run -many of the Extempore examples. These asset files live in a [separate -repo](https://github.com/extemporelang/extempore-assets). +## LLVM -This option is off by default because it's a pretty big (~300MB) download. If -you don't set `-DASSETS=ON` at build time that's ok---CMake will still create an -`assets` target which you can "build" afterwards to downoad the assets and move -them into place. +Extempore links against a specific version of LLVM (currently 22.1.1, pinned in +`CMakeLists.txt`). LLVM is fetched and built in-tree via CMake's `FetchContent` +--- no system LLVM required. + +The first build takes ~10-30 min because LLVM is compiled from source. +Subsequent builds reuse the cached artifacts under `build/_deps/llvm-*`. Only +the components Extempore needs are built (OrcJIT, target codegen, AsmParser, +Passes, MCDisassembler, IRPrinter). + +CI caches `build/_deps/` across runs --- see `.github/workflows/build-and-test.yml`. ## Targets -The default target will build Extempore, all the dependencies, and AOT-compile -the standard library (for faster startup). However, in other situations the -following targets might come in handy: +The default target builds Extempore, all the dependencies, and AOT-compiles the +standard library (for faster startup). Other targets worth knowing about: + +- `aot_core` --- AOT-compile just the core standard library (pure-xtlang + libraries with no external C library dependencies). +- `aot_external_audio` --- AOT-compile the external audio libraries (portmidi, + sndfile, fft, etc). This is the default AOT target. +- `clean_aot` --- remove all AOT-compiled files. +- `assets` --- download and unpack the assets tarball. -- on macOS and Linux, the `install` target will move the extempore executable to - `/usr/local/bin` (or similar) and the rest of the Extempore share directory to - `/usr/local/share/extempore` (does nothing on Windows) +## Running Extempore -- the `aot` target will ahead-of-time compile just the core standard library, - i.e. the pure-xtlang libraries with no external C library dependencies +The `extempore` binary locates its share directory (`runtime/`, `libs/`, +`examples/`) relative to the source tree at build time. Run it from the build +directory: -- the `clean_aot` target will remove all AOT-compiled files + ./extempore # audio + Scheme interpreter, listens on port 7099 + ./extempore --noaudio # same, without audio + ./extempore --repl # interactive linenoise REPL (Linux/macOS only) + ./extempore --batch "(begin (println 'hello) (quit 0))" ## Platform-specific notes ### macOS -#### macOS 10.15 Catalina +Extempore's macOS builds target arm64 (Apple Silicon) with a minimum deployment +target of macOS 11.0 Big Sur. -Since macOS 10.15 Apple requires all binaries to be signed & notarized, and the -Extempore core team (Andy & Ben) haven't yet got an Apple Developer account set -up to do that (it's on the to-do list). So if you have problems with the macOS -Gatekeeper saying that it doesn't trust the `extempore` binary then reach out on -the [mailing list](mailto:extemporelang@googlegroups.com)---there's a workaround -which isn't ideal (disabling the "is this binary legit?" check) but we can keep -you up-to-date on the best way to deal with the issue. +Apple requires distributed binaries to be signed & notarised, and the Extempore +core team haven't got an Apple Developer account set up for that. If Gatekeeper +refuses to run the `extempore` binary, reach out on the +[mailing list](mailto:extemporelang@googlegroups.com). ### Linux -Extempore is built & tested on Ubuntu, but is also known to work with Debian, -Fedora, Arch, NixOS, and inside a docker container. +Extempore is built & tested on Ubuntu 24.04 (x86_64 and aarch64) in CI. -On Linux, you'll need to specify an [ALSA](http://www.alsa-project.org/) backend -for portaudio. To use the `asound` portaudio backend (the default) you'll need -the `libasound` package. Extempore also includes a legacy -[Jack](http://www.jackaudio.org/) portaudio backend, but it has bitrotted in -recent years. PortAudio should pick it up if it's there, but you might have to -do some spelunking. +You'll need the ALSA dev headers for PortAudio. On Ubuntu: -#### Ubuntu + sudo apt-get install libasound2-dev -On Ubuntu 18.04-20.04 you can get the required deps with: +For the WebGPU graphics build (`-DEXTERNAL_SHLIBS_GRAPHICS=ON`) you'll also need: - sudo apt-get install libasound2-dev xorg-dev libglu1-mesa-dev - -#### Arch - -There's an [AUR package](https://aur.archlinux.org/packages/extempore-git/) + sudo apt-get install xorg-dev libglu1-mesa-dev ### Windows -Extempore is built & tested on Windows 10 with Visual Studio 2017 and Visual -Studio 2019. If you don't already have VS installed, you can download the free -[Visual Studio -Community](https://www.visualstudio.com/en-us/products/visual-studio-community-vs.aspx)---that's -perfectly fine for building Extempore (although the paid versions of VS will -work as well). - -In the CMake build process, you'll need to specify which version you're -using---confusingly the Visual Studio version numbers and years don't _quite_ line up, so -the respective generators are: +Extempore is built & tested on Windows Server 2022 with Visual Studio 2022. If +you don't already have VS installed, download the free +[Visual Studio Community](https://visualstudio.microsoft.com/vs/community/). -- VS2017: "-G "Visual Studio 15 2017" -A x64" -- VS2019: "-G "Visual Studio 16 2019" -A x64" +The CMake generator for VS2022 is `-G "Visual Studio 17 2022" -A x64`. #### Missing `VCRUNTIME140_1.dll` -If you ever see the error message _VCRUNTIME140_1.dll was not found_, then -you'll need to download the x64 `vc_redist.x64.exe`. Make sure you get it from -the official [Windows -website](https://support.microsoft.com/en-au/help/2977003/the-latest-supported-visual-c-downloads), -because there are lots of sketchy places on the web which will try and get you -to download theirs instead. - -#### ASIO - -If you want to use the **ASIO** audio backend on Windows (which _might_ give you -lower-latency audio, or it might not) you need to download the [ASIO -SDK](http://www.steinberg.net/nc/en/company/developer/sdk_download_portal.html) -from Steinberg. You have to create a [third party developer -account](http://www.steinberg.net/nc/en/company/developer/sdk_download_portal/create_3rd_party_developer_account.html), -then you can log in and download the ASIO SDK (make sure you get the right SDK). -You also need to download and install [ASIO4ALL](http://www.asio4all.com/) with -the 'offline setup panel' option enabled. After that, copy the ASIO files into -the `src/portaudio/src/hostapi/asio`. +If you see _VCRUNTIME140_1.dll was not found_, install the x64 +`vc_redist.x64.exe` from the +[official Microsoft page](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). diff --git a/CMakeLists.txt b/CMakeLists.txt index b9abca80..44b7ab53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) option(ASSETS "download multimedia assets (approx 500MB)" OFF) option(BUILD_TESTS "build test targets (including examples)" ON) +set(EXTEMPORE_SANITIZE "" CACHE STRING "Build with sanitizer: asan, ubsan, tsan, asan+ubsan") option(EXTERNAL_SHLIBS_AUDIO "download & build audio-related external library dependencies" ON) option(EXTERNAL_SHLIBS_GRAPHICS "WebGPU graphics (GLFW, wgpu-native, stb_image)" OFF) option(EXTERNAL_SHLIBS_GRAPHICS_OPENGL "legacy OpenGL graphics (nanovg, assimp)" OFF) @@ -13,6 +14,7 @@ option(EXT_DYLIB "build extempore as a dynamic library" OFF) option(JACK "use the Jack Portaudio backend (see BUILDING.md)" OFF) set_property(GLOBAL PROPERTY USE_FOLDERS ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) ######### # setup # @@ -180,6 +182,16 @@ FetchContent_Declare(LLVM FetchContent_MakeAvailable(LLVM) +# GoogleTest for C++ unit tests. +if(BUILD_TESTS) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) +endif() + set(EXTEMPORE_LLVM_COMPONENTS OrcJIT ${LLVM_TARGET_ARCH} @@ -245,7 +257,9 @@ add_dependencies(extempore s7_lib portaudio_static) target_include_directories(extempore PRIVATE include - src/pcre + src/pcre) + +target_include_directories(extempore SYSTEM PRIVATE ${CMAKE_BINARY_DIR}/portaudio/include ${LLVM_INCLUDE_DIRS}) @@ -272,17 +286,44 @@ if(UNIX) -fvisibility-inlines-hidden -fno-rtti -fno-common + -Wall + -Wextra -Woverloaded-virtual - -Wno-unused-result) + -Wshadow + -Wno-unused-result + -Wno-unused-parameter + -Wno-unused-variable + -Wno-unused-function + -Wno-missing-field-initializers + -Wno-sign-compare + -Wno-deprecated-declarations + -Werror=return-type + -Werror=uninitialized) + + if(EXTEMPORE_SANITIZE) + set(SAN_FLAGS "") + if(EXTEMPORE_SANITIZE MATCHES "asan") + list(APPEND SAN_FLAGS -fsanitize=address -fno-omit-frame-pointer) + endif() + if(EXTEMPORE_SANITIZE MATCHES "ubsan") + list(APPEND SAN_FLAGS -fsanitize=undefined -fno-sanitize-recover=undefined) + endif() + if(EXTEMPORE_SANITIZE MATCHES "tsan") + list(APPEND SAN_FLAGS -fsanitize=thread) + endif() + target_compile_options(extempore PRIVATE ${SAN_FLAGS}) + target_link_options(extempore PRIVATE ${SAN_FLAGS}) + message(STATUS "Extempore sanitizers enabled: ${EXTEMPORE_SANITIZE}") + endif() target_link_libraries(extempore PRIVATE pthread linenoise) endif() if(WIN32) - target_include_directories(extempore PRIVATE src/networking-ts-impl/include) + target_include_directories(extempore SYSTEM PRIVATE src/networking-ts-impl/include) target_compile_definitions(extempore PRIVATE -DPCRE_STATIC -D_CRT_SECURE_NO_WARNINGS -DNOMINMAX) - target_compile_options(extempore PRIVATE /bigobj) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/extempore.def") + target_compile_options(extempore PRIVATE /bigobj /W3 /wd4244 /wd4267 /wd4018 /wd4996 /we4715) + target_link_options(extempore PRIVATE "/DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/extempore.def") set_target_properties(extempore PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_SOURCE_DIR} RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_SOURCE_DIR} @@ -309,6 +350,26 @@ else() target_link_options(extempore PRIVATE -rdynamic) endif() +############## +# clang-tidy # +############## + +# Requires CMAKE_EXPORT_COMPILE_COMMANDS=ON (Ninja sets this automatically). +# Filters to extempore source files only, not vendored third-party code. +set(TIDY_SOURCES "") +foreach(_src ${EXTEMPORE_SOURCES}) + if(NOT _src MATCHES "(s7\\.c|pcre|networking-ts-impl|ffi/|shims/|linenoise)") + list(APPEND TIDY_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/${_src}") + endif() +endforeach() + +add_custom_target(tidy + COMMAND clang-tidy -p ${CMAKE_BINARY_DIR} ${TIDY_SOURCES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Running clang-tidy on Extempore sources" + USES_TERMINAL + VERBATIM) + ########## # assets # ########## diff --git a/README.md b/README.md index cde13414..e4430b55 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,27 @@ -# Extempore ![Build & test](https://github.com/digego/extempore/workflows/Build%20&%20run%20tests/badge.svg?branch=master) ![Release](https://github.com/digego/extempore/workflows/Release/badge.svg) +# Extempore ![Build & test](https://github.com/digego/extempore/workflows/Build%20&%20test/badge.svg?branch=master) ![Release](https://github.com/digego/extempore/workflows/Release/badge.svg) -A programming environment for cyberphysical programming (Linux/macOS/Windows, including Apple Silicon and Linux aarch64). +A programming environment for cyberphysical programming. Runs on Linux +(x86_64 and aarch64), macOS (Apple Silicon) and Windows (x86_64). -## Getting started - -### The easy way +## What's new in v0.9.0 -Download [VSCode](https://code.visualstudio.com/), install the Extempore -extension and then use the _Extempore: Download binary_ command to do the rest. +v0.9.0-beta is a large release. The highlights: -**Note**: Extempore's binary releases are -[built automatically](https://github.com/digego/extempore/actions?query=workflow%3ARelease) -for Windows, macOS and Linux (Linux release are built on Ubuntu, on other -distros YMMV). +- **LLVM 22** --- bumped from LLVM 17, fetched and built in-tree via CMake's + `FetchContent` (no separate LLVM install required). +- **ORC JIT** --- replaces the legacy MCJIT, including DSP hot-swap and module + tracking. +- **s7 Scheme** --- replaces TinyScheme as the Scheme interpreter. +- **WebGPU graphics** --- the OpenGL stack has been replaced with WebGPU via + `wgpu-native`. Enable with `-DEXTERNAL_SHLIBS_GRAPHICS=ON`. +- **Linux aarch64** --- first-class support, alongside macOS arm64, Linux x64 + and Windows x64. All four platforms are tested in CI. +- **Interactive REPL** --- pass `--repl` for a linenoise-based REPL (Linux and + macOS only). -For more details, head to the -[Quickstart page](https://extemporelang.github.io/docs/overview/quickstart/) in -Extempore's online docs. +## Getting started -### The _slightly_ harder way (for those who don't want to use VSCode) +### The easy way Download the latest [binary release](https://github.com/digego/extempore/releases) for your @@ -29,29 +32,41 @@ Then, [set up your text editor of choice](https://extemporelang.github.io/docs/guides/editor-support/) and away you go. +_Note_: the VSCode extension used to offer an _Extempore: Download binary_ +command for one-click setup. It hasn't been updated for v0.9.0 and may not work +--- downloading the release manually is the safest option for now. + ### Build from source **For more information**, check out [BUILDING.md](./BUILDING.md). -Extempore's CMake build process downloads and build all the dependencies you -need (including LLVM). So, if you've got a C++ compiler, git and CMake, here are -some one-liner build commands: +Extempore's CMake build process downloads and builds all the dependencies you +need (including LLVM). So, if you've got a C++ compiler, git, CMake >= 3.28 and +Ninja, here are some one-liner build commands. On **Linux/macOS**: - git clone https://github.com/digego/extempore && mkdir extempore/build && cd extempore/build && cmake -DASSETS=ON .. && make && sudo make install + git clone https://github.com/digego/extempore && cmake -S extempore -B extempore/build -G Ninja -DASSETS=ON && cmake --build extempore/build -j$(nproc) -On **Windows** (if you're using VS2019---adjust as necessary for your VS -version): +On **Windows** (adjust the generator for your VS version): - git clone https://github.com/digego/extempore && mkdir extempore/build && cd extempore/build && cmake -G "Visual Studio 16 2019" -A x64 -DASSETS=ON .. && cmake --build . --target INSTALL --config Release + git clone https://github.com/digego/extempore && cmake -S extempore -B extempore/build -G "Visual Studio 17 2022" -A x64 -DASSETS=ON && cmake --build extempore/build --config Release -_Note:_ in the above one-liners the `ASSETS` build-time option (boolean, default -`OFF`) is set to `ON`. This will download the Extempore binary assets---required -for many of the examples, but adds a ~300MB download to build process. If you'd +_Note on build time_: the first build takes ~10-30 minutes because LLVM is +compiled from source. Subsequent builds reuse the cached LLVM artifacts under +`build/_deps/`. + +_Note on ASSETS_: the `ASSETS` build-time option (boolean, default `OFF`) is set +to `ON` above. This will download the Extempore binary assets --- required for +many of the examples, but adds a ~250MB download to the build process. If you'd rather not do that, and are happy with some of the examples not working, then set `-DASSETS=OFF` instead. +_Note on running_: the `extempore` binary locates its runtime files +(`runtime/`, `libs/`, `examples/`) relative to the source tree at build time. +Run it from the build directory (`./extempore`) rather than installing it to a +system location. + ## See Extempore in action Check out these videos: @@ -90,7 +105,7 @@ You can also join the Extempore community: ## Licence -Copyright (c) 2011-2025, Andrew Sorensen +Copyright (c) 2011-2026, Andrew Sorensen All rights reserved. diff --git a/backlog/tasks/task-038 - Improve-new-user-onboarding-experience.md b/backlog/tasks/task-038 - Improve-new-user-onboarding-experience.md new file mode 100644 index 00000000..993eb14a --- /dev/null +++ b/backlog/tasks/task-038 - Improve-new-user-onboarding-experience.md @@ -0,0 +1,57 @@ +--- +id: TASK-038 +title: Improve new-user onboarding experience +status: To Do +assignee: [] +created_date: '2026-04-19 01:07' +labels: + - docs + - onboarding + - ux +dependencies: [] +priority: medium +--- + +## Description + + +Address newcomer friction found in the 2026-04-19 onboarding audit. A determined self-starter can reach 'hello sine' in about an hour, but casual users bounce at dead links, draft pages that were accidentally published, and a stale contributing guide. This task bundles the quick wins and deeper gaps into one track so they can be prioritised and split out when picked up. + +**Audit summary (2026-04-19)** + +Top 5 quick wins: +1. README tagline: replace 'A programming environment for cyberphysical programming' with a plain-English version (CLAUDE.md has a good one). Add a 3-line paste-and-hear-sine snippet under Getting Started. +2. docs/reference/index.md currently shows a literal org-mode outline and the string 'Use Cian's stuff here, e.g.'. Rewrite as a real landing page linking the existing types.md/memory-management.md/concurrency.md. +3. Resolve the VSCode 'Download binary' disagreement: README says the command is unmaintained for v0.9.0; docs/overview/quickstart.md:51 still recommends it. +4. Rewrite docs/overview/contributing.md: it still documents the Jekyll flow with 'bundle install'; repo moved to VitePress. Also delete wishlist items already shipped in v0.9.0 (LLVM+ORC upgrade, aarch64). +5. Add examples/hello_world.xtm (println + gentle 2s 440Hz sine, block-by-block comments). Link from README and quickstart. + +Deeper gaps: +- A real xtlang tutorial (docs/reference/new/ is a draft with typos like 'beging' and undefined syntax like `($ ...)`) +- A JIT/type error-message glossary +- A one-line 'listening on :7099' startup pointer in src/Extempore.cpp +- Publish the xtmdoc-generated API reference on the docs site +- Pick a canonical GitHub org (digego/extempore vs extemporelang/extempore) and normalise all links +- Modernise the community surface (GitHub Discussions, curated talks list) + +Small inconsistencies worth fixing in passing: +- ASSETS size disagreement: CMakeLists.txt:7 says ~500MB, README.md:61 and BUILDING.md:20 say ~250MB +- BUILDING.md:77 mentions Gatekeeper but not the actual fix (xattr -dr com.apple.quarantine) +- --sharedir is warned about in README.md:65 but never shown in use +- Startup banner year in src/Extempore.cpp is 2010-2025; README licence says 2011-2026 +- examples/ has no explanation of core/ vs external/ vs sharedsystem/ or which require optional CMake flags + + +## Acceptance Criteria + +- [ ] #1 README tagline replaced and 'paste this to hear sine' block added under Getting Started +- [ ] #2 docs/reference/index.md is a real landing page with links, no org-mode draft content visible +- [ ] #3 VSCode binary download instructions are consistent between README and quickstart +- [ ] #4 docs/overview/contributing.md documents the current VitePress workflow and omits shipped wishlist items +- [ ] #5 examples/hello_world.xtm exists and is linked from README and quickstart +- [ ] #6 xtlang tutorial covering bind-func, types, closures, and memory zones is drafted or scoped as its own follow-up task +- [ ] #7 JIT/type error-message glossary is drafted or scoped as its own follow-up task +- [ ] #8 Extempore prints a 'listening on :7099' pointer on startup +- [ ] #9 Canonical GitHub org is chosen and all README/docs links normalised +- [ ] #10 ASSETS size, macOS quarantine fix, --sharedir, and banner year inconsistencies are reconciled + diff --git a/backlog/tasks/task-039 - RAII-conversion-audio-scheme-thread-ownership.md b/backlog/tasks/task-039 - RAII-conversion-audio-scheme-thread-ownership.md new file mode 100644 index 00000000..06e3ad46 --- /dev/null +++ b/backlog/tasks/task-039 - RAII-conversion-audio-scheme-thread-ownership.md @@ -0,0 +1,26 @@ +--- +id: TASK-039 +title: 'RAII conversion: audio/scheme thread ownership' +status: To Do +assignee: [] +created_date: '2026-04-19 02:04' +labels: + - cpp + - refactor +dependencies: [] +priority: medium +--- + +## Description + + +Convert remaining owning raw pointers to unique_ptr/value types. Follow-up +from the 2026-04-19 C++ modernisation PR (#419): + +- AudioDevice::m_threads[MAX_RT_AUDIO_THREADS] → std::array, N> +- SchemeProcess/SchemeREPL singletons → Meyers singletons +- SchemeTask::m_ptr → std::variant +- Per-thread thread_local PRNGs (EXTLLVM.cpp:392,397) → values not pointers + +See docs/superpowers/plans/2026-04-19-cpp-modernisation.md §5 for context. + diff --git a/backlog/tasks/task-040 - Audit-LLVM_SCHEME_FF_MAP-and-FFI-table-thread-safety.md b/backlog/tasks/task-040 - Audit-LLVM_SCHEME_FF_MAP-and-FFI-table-thread-safety.md new file mode 100644 index 00000000..299f7003 --- /dev/null +++ b/backlog/tasks/task-040 - Audit-LLVM_SCHEME_FF_MAP-and-FFI-table-thread-safety.md @@ -0,0 +1,23 @@ +--- +id: TASK-040 +title: Audit LLVM_SCHEME_FF_MAP and FFI table thread safety +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - concurrency +dependencies: [] +priority: high +--- + +## Description + + +src/EXTLLVM.cpp:137 (LLVM_SCHEME_FF_MAP) and src/SchemeS7.cpp:25-57 +(s_ffiCount/s_ffiTable) are acknowledged in comments as not thread safe. +Audit real access patterns and guard with std::shared_mutex or atomic +counters where appropriate. + +Follow-up from #419. + diff --git a/backlog/tasks/task-041 - Zone-allocator-mutex-on-audio-thread.md b/backlog/tasks/task-041 - Zone-allocator-mutex-on-audio-thread.md new file mode 100644 index 00000000..009e8c5d --- /dev/null +++ b/backlog/tasks/task-041 - Zone-allocator-mutex-on-audio-thread.md @@ -0,0 +1,23 @@ +--- +id: TASK-041 +title: Zone allocator mutex on audio thread +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - rt-audio +dependencies: [] +priority: high +--- + +## Description + + +src/EXTZones.cpp:69-73 takes a global std::recursive_mutex on every +allocation in the audio-callback path. This is an RT-audio anti-pattern +(priority inversion). Investigate lock-free zone design or per-thread +zones. + +Follow-up from #419. + diff --git a/backlog/tasks/task-042 - Audio-thread-dispatcher-race-conditions.md b/backlog/tasks/task-042 - Audio-thread-dispatcher-race-conditions.md new file mode 100644 index 00000000..0ee2b60a --- /dev/null +++ b/backlog/tasks/task-042 - Audio-thread-dispatcher-race-conditions.md @@ -0,0 +1,24 @@ +--- +id: TASK-042 +title: Audio thread dispatcher race conditions +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - rt-audio + - concurrency +dependencies: [] +priority: high +--- + +## Description + + +src/AudioDevice.cpp:169-170, 391 — multi-thread dispatch uses +spin-sleep on atomics and an unsynchronised m_numThreads read +(source comment at line 391 reads 'this is a race :('). Replace with +std::counting_semaphore or condition variable. TSan run required. + +Follow-up from #419. + diff --git a/backlog/tasks/task-043 - Consolidate-OSC.cpp-UDP-TCP-Windows-POSIX-duplication.md b/backlog/tasks/task-043 - Consolidate-OSC.cpp-UDP-TCP-Windows-POSIX-duplication.md new file mode 100644 index 00000000..ee849dd8 --- /dev/null +++ b/backlog/tasks/task-043 - Consolidate-OSC.cpp-UDP-TCP-Windows-POSIX-duplication.md @@ -0,0 +1,24 @@ +--- +id: TASK-043 +title: Consolidate OSC.cpp UDP/TCP + Windows/POSIX duplication +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - refactor +dependencies: [] +priority: medium +--- + +## Description + + +src/OSC.cpp — 1700+ lines with UDP and TCP paths each duplicated +across Windows (networking-TS experimental) and POSIX. Byte-swap helpers +(lines 93-222) reimplement __builtin_bswap*. Std::byteswap (C++23) will +eventually replace them. + +Large standalone project — its own PR. +Follow-up from #419. + diff --git a/backlog/tasks/task-044 - Migrate-EXTThread-to-std-jthread-where-RT-not-needed.md b/backlog/tasks/task-044 - Migrate-EXTThread-to-std-jthread-where-RT-not-needed.md new file mode 100644 index 00000000..8b8277e8 --- /dev/null +++ b/backlog/tasks/task-044 - Migrate-EXTThread-to-std-jthread-where-RT-not-needed.md @@ -0,0 +1,27 @@ +--- +id: TASK-044 +title: 'Migrate EXTThread to std::jthread where RT not needed' +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - refactor +dependencies: [] +priority: medium +--- + +## Description + + +EXTThread was kept in #419 because it provides platform-specific +realtime scheduling (SCHED_RR on Linux, THREAD_TIME_CONSTRAINT_POLICY on +macOS) and a subsume-current-thread mode that std::thread doesn't +natively offer. + +Most call sites don't need the RT features, so they can use std::jthread +directly. Audit and migrate where safe; keep a small rt_thread helper +just for the audio callback threads. + +Follow-up from #419. + diff --git a/backlog/tasks/task-045 - Redesign-rreplace-rsplit-API-with-explicit-capacity.md b/backlog/tasks/task-045 - Redesign-rreplace-rsplit-API-with-explicit-capacity.md new file mode 100644 index 00000000..61fb3fff --- /dev/null +++ b/backlog/tasks/task-045 - Redesign-rreplace-rsplit-API-with-explicit-capacity.md @@ -0,0 +1,27 @@ +--- +id: TASK-045 +title: Redesign rreplace/rsplit API with explicit capacity +status: To Do +assignee: [] +created_date: '2026-04-19 02:05' +labels: + - cpp + - refactor + - xtlang +dependencies: [] +priority: low +--- + +## Description + + +rreplace and rsplit currently pass fixed-size char* buffers from xtlang +(libs/core/adt.xtm uses salloc 2048/4096). #419 added a ceiling check +that returns false if the output would overflow, but the proper fix is +an API that passes capacity, or returns std::string. + +Requires coordinated changes in runtime/bitcode.ll type signatures and +adt.xtm callers. Scope it as its own PR. + +Follow-up from #419. + diff --git a/docs/superpowers/plans/2026-04-19-cpp-modernisation.md b/docs/superpowers/plans/2026-04-19-cpp-modernisation.md new file mode 100644 index 00000000..0aba98ba --- /dev/null +++ b/docs/superpowers/plans/2026-04-19-cpp-modernisation.md @@ -0,0 +1,1123 @@ +# C++ modernisation implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Apply findings from the 2026-04-19 C++ audit to Extempore's runtime, in CI-gated phases that keep Linux x64 / Linux arm64 / macOS arm64 / Windows x64 green at every commit. + +**Architecture:** Work in a `cpp-modernisation` branch, ship each phase as a small commit series, push after every phase, wait for `build-and-test.yml` to go green on all four platforms before starting the next. Phases are ordered so the highest-leverage work (compiler warnings) runs first — every subsequent phase benefits from it. A minimal C++ unit-test target (`tests/cpp-unit/`) is added in Phase 2 and grows throughout. + +**Tech Stack:** C++17, CMake + Ninja, ctest, LLVM 22 ORC, portaudio, s7 Scheme, GoogleTest via FetchContent (added in Phase 2). + +--- + +## Phase summary + +| Phase | Focus | Outcome | +|-------|----------------------------------|----------------------------------------------------------------------| +| 1 | Build hygiene | `-Wall -Wextra`, `.clang-format`, `.clang-tidy`, sanitizer build opt | +| 2 | Critical UNIV.cpp correctness | Fix UB + memory bugs in `UNIV.cpp`; add cpp-unit test harness | +| 3 | Kill hand-rolled sync primitives | Delete `EXTMutex`/`EXTMonitor`/`EXTCondition`/`EXTThread` | +| 4 | Networking safety | `gethostbyname` → `getaddrinfo`; FD sentinel; signal-handler safety | +| 5 | Spin off deferred backlog items | Convert larger findings to backlog tasks | + +CI gate between every phase: push, wait, confirm the 4-platform matrix is green. + +--- + +## Phase 0: Branch setup + +### Task 0.1: Create branch + +**Files:** none + +- [ ] **Step 1: Create and check out a clean branch** + +```bash +git checkout -b cpp-modernisation +``` + +- [ ] **Step 2: Push to establish the remote and confirm CI triggers** + +```bash +git push -u origin cpp-modernisation +gh run list --branch cpp-modernisation --limit 3 +``` + +Expected: `build-and-test.yml` queued for all 4 matrix entries (builds pass; the branch hasn't changed anything yet). + +--- + +## Phase 1: Build hygiene + +**Files:** +- Modify: `CMakeLists.txt:271-276` (UNIX compile options) and `CMakeLists.txt:283` (Windows compile options) +- Create: `.clang-format` +- Create: `.clang-tidy` + +### Task 1.1: Turn on the warning flags that find bugs + +Rationale: `-Wall -Wextra` as warnings (not errors) shows us the lay of the land without blocking the build. `-Werror=` on a small set of *always bugs* gives us a regression gate. + +- [ ] **Step 1: Edit `CMakeLists.txt` UNIX block** + +In the `if(UNIX)` branch around line 271, replace the existing `target_compile_options(extempore PRIVATE ...)` block with: + +```cmake +target_compile_options(extempore PRIVATE + -fvisibility-inlines-hidden + -fno-rtti + -fno-common + -Wall + -Wextra + -Woverloaded-virtual + -Wshadow + -Wno-unused-result + -Wno-unused-parameter + -Wno-missing-field-initializers + -Wno-sign-compare + -Werror=return-type + -Werror=uninitialized + -Werror=sometimes-uninitialized + -Werror=return-stack-address) +``` + +The `-Wno-*` suppressions are the load-bearing ones: we're turning on `-Wall -Wextra` but silencing the categories that produce thousands of hits on vendored s7/pcre/LLVM code without pointing at real bugs. We can remove suppressions one at a time in follow-up tasks. + +`-Werror=` on `return-type`, `uninitialized`, `sometimes-uninitialized`, `return-stack-address` turns always-bugs into build failures. + +- [ ] **Step 2: Edit `CMakeLists.txt` Windows block** + +In the `if(WIN32)` block around line 283-284, append warning flags: + +```cmake +target_compile_options(extempore PRIVATE /bigobj /W3 /wd4244 /wd4267 /wd4018 /we4715) +``` + +`/we4715` promotes "not all paths return a value" to an error (matches the UNIX `-Werror=return-type`). + +- [ ] **Step 3: Configure + build locally to see the warning volume** + +```bash +cmake --build build --parallel 2 2>&1 | tee /tmp/warn.log +grep -cE "warning:|error:" /tmp/warn.log +``` + +Expected: build succeeds (errors should be zero, barring a real bug caught by `-Werror=return-type`). Warnings likely 50-500. + +- [ ] **Step 4: If a `-Werror=` category fires, investigate** + +If the build fails with one of the promoted-to-error warnings, read the error, fix the root cause, do *not* disable the error. Commit the fix alongside the flag change. + +- [ ] **Step 5: Commit** + +```bash +git add CMakeLists.txt +git commit -m "build: enable -Wall -Wextra with bug-finding errors + +Turns on -Wall -Wextra as warnings and promotes four categories +(return-type, uninitialized, sometimes-uninitialized, return-stack-address) +to errors. Silences -Wno-unused-parameter, -Wno-missing-field-initializers, +and -Wno-sign-compare for now; these produce mostly noise from vendored +code and will be revisited in follow-up passes." +``` + +### Task 1.2: Sanitizer build option + +- [ ] **Step 1: Add the option near the top of `CMakeLists.txt`** + +Find the existing `option(BUILD_TESTS ...)` line (~line 8) and insert after it: + +```cmake +set(EXTEMPORE_SANITIZE "" CACHE STRING "Build with sanitizer: asan, ubsan, tsan, asan+ubsan") +``` + +- [ ] **Step 2: Add sanitizer flag wiring to the UNIX block** + +After the `target_compile_options` block in the `if(UNIX)` branch, append: + +```cmake +if(EXTEMPORE_SANITIZE) + set(SAN_FLAGS "") + if(EXTEMPORE_SANITIZE MATCHES "asan") + list(APPEND SAN_FLAGS -fsanitize=address -fno-omit-frame-pointer) + endif() + if(EXTEMPORE_SANITIZE MATCHES "ubsan") + list(APPEND SAN_FLAGS -fsanitize=undefined -fno-sanitize-recover=undefined) + endif() + if(EXTEMPORE_SANITIZE MATCHES "tsan") + list(APPEND SAN_FLAGS -fsanitize=thread) + endif() + target_compile_options(extempore PRIVATE ${SAN_FLAGS}) + target_link_options(extempore PRIVATE ${SAN_FLAGS}) + message(STATUS "Extempore sanitizers enabled: ${EXTEMPORE_SANITIZE}") +endif() +``` + +- [ ] **Step 3: Test locally** + +```bash +rm -rf build-asan && cmake -B build-asan -G Ninja -DCMAKE_BUILD_TYPE=Debug -DEXTEMPORE_SANITIZE=asan +cmake --build build-asan --target extempore --parallel 2 +./build-asan/extempore --batch "(begin (println 'hello) (quit 0))" +``` + +Expected: build succeeds; batch run either completes cleanly or reports real ASan findings. Record findings. + +- [ ] **Step 4: Commit** + +```bash +git add CMakeLists.txt +git commit -m "build: add EXTEMPORE_SANITIZE option for ASan/UBSan/TSan" +``` + +### Task 1.3: `.clang-format` + +- [ ] **Step 1: Create `.clang-format` at repo root** + +Mirror LLVM's own style since the codebase is LLVM-adjacent and roughly matches: + +```yaml +--- +BasedOnStyle: LLVM +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ColumnLimit: 100 +AlignAfterOpenBracket: Align +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +BreakBeforeBraces: Attach +NamespaceIndentation: None +PointerAlignment: Left +SpacesBeforeTrailingComments: 2 +IncludeBlocks: Preserve +SortIncludes: Never +``` + +`SortIncludes: Never` is deliberate — header order in this codebase matters (Windows headers before POSIX headers in several TUs). + +- [ ] **Step 2: Do NOT mass-reformat the codebase yet** + +A reformat-everything commit pollutes `git blame`. Keep `.clang-format` available for incremental use (editor-on-save, `clang-format-diff` on new changes). A full reformat is TASK-023 in the backlog — leave it there. + +- [ ] **Step 3: Commit** + +```bash +git add .clang-format +git commit -m "style: add .clang-format (LLVM-based, no mass reformat)" +``` + +### Task 1.4: `.clang-tidy` + +- [ ] **Step 1: Create `.clang-tidy` at repo root** + +Start conservative — only the bug-finding checks, not style or modernize yet: + +```yaml +--- +Checks: > + -*, + bugprone-*, + -bugprone-easily-swappable-parameters, + -bugprone-macro-parentheses, + -bugprone-narrowing-conversions, + cppcoreguidelines-pro-type-static-cast-downcast, + cppcoreguidelines-slicing, + misc-const-correctness, + misc-misplaced-const, + performance-*, + -performance-no-int-to-ptr, + readability-misleading-indentation, + readability-redundant-smartptr-get +WarningsAsErrors: '' +HeaderFilterRegex: '^(src|include)/(?!pcre|s7|networking-ts-impl|ffi|shims|linenoise).*' +``` + +- [ ] **Step 2: Add a CMake target for running clang-tidy** + +Near the bottom of `CMakeLists.txt` (before the `assets` target), add: + +```cmake +add_custom_target(tidy + COMMAND clang-tidy -p ${CMAKE_BINARY_DIR} ${EXTEMPORE_SOURCES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMENT "Running clang-tidy on Extempore sources") +``` + +This depends on CMake generating `compile_commands.json`, which Ninja does automatically. + +- [ ] **Step 3: Run it locally to see the output volume** + +```bash +cmake --build build --target tidy 2>&1 | tee /tmp/tidy.log +grep -c "warning:" /tmp/tidy.log +``` + +Record the count. Do NOT fix anything yet — this is a baseline. + +- [ ] **Step 4: Commit** + +```bash +git add .clang-tidy CMakeLists.txt +git commit -m "tooling: add .clang-tidy config and 'tidy' make target" +``` + +### Task 1.5: CI gate + +- [ ] **Step 1: Push the phase** + +```bash +git push +``` + +- [ ] **Step 2: Watch CI** + +```bash +gh run watch --branch cpp-modernisation --exit-status +``` + +Expected: all 4 matrix platforms green. If Linux or macOS fails on a new warning, that's a bug — fix it and push again. If Windows fails on `/we4715`, investigate the missing-return path and fix. + +--- + +## Phase 2: Critical UNIV.cpp correctness + cpp-unit test harness + +**Files:** +- Modify: `src/UNIV.cpp:159-210` (`cname_decode`), `src/UNIV.cpp:212-265` (`base64_decode`), `src/UNIV.cpp:326-343` (`rreplace`/`rsplit`), `src/UNIV.cpp:349-372` (`sys_slurp_file`) +- Create: `tests/cpp-unit/CMakeLists.txt`, `tests/cpp-unit/univ_test.cpp` +- Modify: `CMakeLists.txt` (add GoogleTest via FetchContent and `add_subdirectory(tests/cpp-unit)`) +- Modify: `.github/workflows/build-and-test.yml:75` (add `compiler-unit|cpp-unit` to the ctest label regex — it's `compiler-unit` already; add `|cpp-unit`) + +### Task 2.1: Add GoogleTest and the cpp-unit subdirectory + +- [ ] **Step 1: Add GoogleTest to `CMakeLists.txt` via FetchContent** + +Near the top where other `FetchContent_Declare`s live (grep for `FetchContent_Declare(llvm`), add: + +```cmake +if(BUILD_TESTS) + include(FetchContent) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz + URL_HASH SHA256=7b42b4d6ed48810c5362c265a17faebe90dc2373c885e5216439d37927f02926) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) +endif() +``` + +The hash is the published one for v1.15.2; update if CMake complains. + +- [ ] **Step 2: Add subdirectory** + +Near the bottom of `CMakeLists.txt`, before the `assets` target: + +```cmake +if(BUILD_TESTS) + enable_testing() + add_subdirectory(tests/cpp-unit) +endif() +``` + +Check that `enable_testing()` isn't already there — if it is, leave the existing call. + +- [ ] **Step 3: Create `tests/cpp-unit/CMakeLists.txt`** + +```cmake +add_executable(extempore_cpp_unit + univ_test.cpp) + +target_link_libraries(extempore_cpp_unit PRIVATE + gtest_main) + +target_include_directories(extempore_cpp_unit PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src) + +target_compile_features(extempore_cpp_unit PRIVATE cxx_std_17) + +include(GoogleTest) +gtest_discover_tests(extempore_cpp_unit + PROPERTIES LABELS "cpp-unit") +``` + +- [ ] **Step 4: Create a skeleton `tests/cpp-unit/univ_test.cpp`** + +```cpp +#include + +TEST(UnivSmoke, PlaceholderUntilRealTests) { + EXPECT_EQ(1 + 1, 2); +} +``` + +(Real tests arrive in 2.2 onward.) + +- [ ] **Step 5: Build and run** + +```bash +cmake --build build --target extempore_cpp_unit --parallel 2 +ctest --test-dir build --label-regex cpp-unit --output-on-failure +``` + +Expected: the smoke test passes. + +- [ ] **Step 6: Add `cpp-unit` to the CI label regex** + +In `.github/workflows/build-and-test.yml:75`, change: + +```yaml +run: ctest --test-dir build --build-config Release --label-regex "libs-core|libs-external|compiler-unit|examples-core|examples-audio" --output-on-failure +``` + +to: + +```yaml +run: ctest --test-dir build --build-config Release --label-regex "libs-core|libs-external|compiler-unit|cpp-unit|examples-core|examples-audio" --output-on-failure +``` + +- [ ] **Step 7: Commit** + +```bash +git add CMakeLists.txt tests/cpp-unit/ .github/workflows/build-and-test.yml +git commit -m "test: add cpp-unit test target with GoogleTest" +``` + +### Task 2.2: Test + fix `sys_slurp_file` off-by-one and unchecked `fread` + +**Files:** `src/UNIV.cpp:349-372`, `tests/cpp-unit/univ_test.cpp` + +- [ ] **Step 1: Expose a testable signature** + +`sys_slurp_file` currently lives in the `extemp::UNIV` namespace (check `include/UNIV.h`). If it's already exposed as `extemp::UNIV::sys_slurp_file(const char*)`, no change needed. Otherwise add a declaration to `include/UNIV.h` in the namespace block (grep `rreplace` — it's declared nearby). + +- [ ] **Step 2: Write failing tests in `tests/cpp-unit/univ_test.cpp`** + +Replace the placeholder test: + +```cpp +#include +#include +#include +#include +#include +#include + +#include "UNIV.h" + +namespace { +std::string write_temp(const std::string& contents) { + auto p = std::filesystem::temp_directory_path() / + ("extempore_test_" + std::to_string(::getpid()) + "_" + + std::to_string(rand()) + ".txt"); + std::ofstream(p, std::ios::binary) << contents; + return p.string(); +} +} + +TEST(SysSlurpFile, PreservesAllBytes) { + auto path = write_temp("hello"); + char* out = extemp::UNIV::sys_slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_STREQ(out, "hello"); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SysSlurpFile, HandlesEmpty) { + auto path = write_temp(""); + char* out = extemp::UNIV::sys_slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_STREQ(out, ""); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SysSlurpFile, HandlesBinaryWithEmbeddedZero) { + auto path = write_temp(std::string("ab\0cd", 5)); + char* out = extemp::UNIV::sys_slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + // At minimum the last non-null byte must be preserved + EXPECT_EQ(out[3], 'c'); + EXPECT_EQ(out[4], 'd'); + std::free(out); + std::filesystem::remove(path); +} +``` + +- [ ] **Step 3: Run — the `PreservesAllBytes` test will fail because current impl truncates** + +```bash +cmake --build build --target extempore_cpp_unit +ctest --test-dir build --label-regex cpp-unit --output-on-failure +``` + +Expected: `PreservesAllBytes` reports `out = "hell"` (last byte truncated by `buf[file_size-1] = '\0'`). + +- [ ] **Step 4: Fix `sys_slurp_file` in `src/UNIV.cpp`** + +Replace the current body (around lines 349-372) with: + +```cpp +char* sys_slurp_file(const char* fname) { + std::FILE* fp = std::fopen(fname, "rb"); + if (!fp) return nullptr; + if (std::fseek(fp, 0, SEEK_END) != 0) { std::fclose(fp); return nullptr; } + long file_size = std::ftell(fp); + if (file_size < 0) { std::fclose(fp); return nullptr; } + std::rewind(fp); + char* buf = static_cast(std::malloc(static_cast(file_size) + 1)); + if (!buf) { std::fclose(fp); return nullptr; } + size_t read = std::fread(buf, 1, static_cast(file_size), fp); + std::fclose(fp); + if (read != static_cast(file_size)) { std::free(buf); return nullptr; } + buf[file_size] = '\0'; + return buf; +} +``` + +- [ ] **Step 5: Re-run tests** + +```bash +ctest --test-dir build --label-regex cpp-unit --output-on-failure +``` + +Expected: all three `SysSlurpFile` tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add src/UNIV.cpp tests/cpp-unit/univ_test.cpp +git commit -m "fix(univ): sys_slurp_file off-by-one and unchecked fread + +- Allocates file_size+1 bytes so the terminator doesn't overwrite content +- Checks fread's return to catch short reads instead of silently returning + a partially-populated buffer +- Adds cpp-unit tests covering empty, ASCII, and embedded-null inputs" +``` + +### Task 2.3: Test + fix `cname_decode` shadow + memory leak + +**Files:** `src/UNIV.cpp:159-210`, `tests/cpp-unit/univ_test.cpp` + +The outer `char* d2 = nullptr` is shadowed by an inner `char* d2 = ...` inside the padding branch (line ~173). The outer `free(d2)` (line 207) is always a no-op; the inner `d2` leaks. + +- [ ] **Step 1: Write a failing test** + +Add to `univ_test.cpp`: + +```cpp +TEST(CnameDecode, PadBranchDoesNotLeak) { + // Input that triggers the padding branch — base64 encoding length ≡ 2 mod 4 + std::string encoded = "YWI"; // "ab" encoded + char buf[16] = {}; + // Exact API signature may need adjustment based on actual declaration + extemp::UNIV::cname_decode(encoded.data(), encoded.size(), buf); + EXPECT_STREQ(buf, "ab"); + // Leak detection happens via ASan CI run (Phase 1.2 sanitizer build) +} +``` + +(If the real signature differs, adjust — check `include/UNIV.h` for the declaration.) + +- [ ] **Step 2: Run — expect a correctness failure or leak under ASan** + +```bash +cmake --build build-asan --target extempore_cpp_unit +ctest --test-dir build-asan --label-regex cpp-unit --output-on-failure +``` + +Expected: ASan reports `direct leak`. + +- [ ] **Step 3: Rename the inner variable in `src/UNIV.cpp`** + +Find the inner `char* d2` allocation (line ~173) and rename to `pad_buf`: + +```cpp +// original: +char* d2 = (char*)malloc(padded_len + 1); +// ... uses d2 ... + +// fixed: +char* pad_buf = (char*)malloc(padded_len + 1); +// ... uses pad_buf ... +// At the end of the padding branch, BEFORE overwriting the outer d2: +data = pad_buf; // or whatever the original assignment was +// Also ensure the outer free cleans up pad_buf: the outer free becomes +// if (pad_buf) free(pad_buf) — rename the outer d2 accordingly, +// OR keep pad_buf local and free it before the branch exits. +``` + +Exact structure depends on the function flow — read lines 159-210 carefully. The goal: outer scope variable names no longer shadow, every `malloc` has a matching `free`. + +- [ ] **Step 4: Re-run under ASan** + +```bash +cmake --build build-asan --target extempore_cpp_unit +ctest --test-dir build-asan --label-regex cpp-unit --output-on-failure +``` + +Expected: no leak reports. + +- [ ] **Step 5: Commit** + +```bash +git add src/UNIV.cpp tests/cpp-unit/univ_test.cpp +git commit -m "fix(univ): cname_decode pad-branch leak and variable shadow" +``` + +### Task 2.4: Test + fix `cname_decode` / `base64_decode` unsequenced side effects + +**Files:** `src/UNIV.cpp:193-196, 257-260`, `tests/cpp-unit/univ_test.cpp` + +The expression `data[i] == '=' ? 0 & i++ : table[unsigned(data[i++])]` evaluates `i++` in multiple sub-expressions without sequencing → UB. + +- [ ] **Step 1: Fix by splitting into explicit statements** + +For both sites, rewrite: + +```cpp +// before: +foo = data[i] == '=' ? 0 & i++ : base64_decoding_table[unsigned(data[i++])]; + +// after: +unsigned char ch = static_cast(data[i++]); +foo = (ch == '=') ? 0 : base64_decoding_table[ch]; +``` + +- [ ] **Step 2: Add a UBSan-exercising test** + +```cpp +TEST(Base64Decode, HandlesPaddingWithoutUB) { + // Exact input that hits the '=' branch; run under UBSan + std::string encoded = "YWI="; + char buf[16] = {}; + extemp::UNIV::base64_decode(encoded.data(), encoded.size(), buf); + EXPECT_STREQ(buf, "ab"); +} +``` + +- [ ] **Step 3: Run under UBSan** + +```bash +rm -rf build-ubsan && cmake -B build-ubsan -G Ninja -DCMAKE_BUILD_TYPE=Debug -DEXTEMPORE_SANITIZE=ubsan +cmake --build build-ubsan --target extempore_cpp_unit +ctest --test-dir build-ubsan --label-regex cpp-unit --output-on-failure +``` + +Expected: pre-fix, UBSan reports unsequenced modification; post-fix, passes cleanly. + +- [ ] **Step 4: Commit** + +```bash +git add src/UNIV.cpp tests/cpp-unit/univ_test.cpp +git commit -m "fix(univ): sequence data[i++] reads in base64/cname decode" +``` + +### Task 2.5: Test + fix `rreplace` / `rsplit` buffer overrun + +**Files:** `src/UNIV.cpp:326-343`, `include/UNIV.h`, `tests/cpp-unit/univ_test.cpp` + +Current signatures write into a caller-supplied `char*` with a post-hoc 4096 check. Change to return `std::string`. + +- [ ] **Step 1: Add new overloads returning `std::string`** + +In `include/UNIV.h` namespace `UNIV`: + +```cpp +std::string rreplace(const std::string& pattern, + const std::string& replacement, + const std::string& input); +std::vector rsplit(const std::string& pattern, + const std::string& input); +``` + +In `src/UNIV.cpp`, implement both using `std::regex_replace` and `std::sregex_token_iterator` respectively. + +- [ ] **Step 2: Deprecate the raw-pointer versions** + +Mark the old signatures `[[deprecated("use std::string overload")]]`. + +- [ ] **Step 3: Write tests** + +```cpp +TEST(RReplace, BasicReplacement) { + EXPECT_EQ(extemp::UNIV::rreplace("foo", "bar", "a foo in a foo"), + "a bar in a bar"); +} +TEST(RSplit, BasicSplit) { + auto parts = extemp::UNIV::rsplit(",", "a,b,c"); + ASSERT_EQ(parts.size(), 3u); + EXPECT_EQ(parts[0], "a"); + EXPECT_EQ(parts[1], "b"); + EXPECT_EQ(parts[2], "c"); +} +TEST(RReplace, VeryLongInputDoesNotCrash) { + std::string big(100000, 'x'); + auto out = extemp::UNIV::rreplace("x", "yy", big); + EXPECT_EQ(out.size(), 200000u); +} +``` + +- [ ] **Step 4: Run, commit** + +```bash +cmake --build build --target extempore_cpp_unit +ctest --test-dir build --label-regex cpp-unit --output-on-failure +git add src/UNIV.cpp include/UNIV.h tests/cpp-unit/univ_test.cpp +git commit -m "feat(univ): add std::string rreplace/rsplit overloads + +Deprecates the char* versions that had an unchecked buffer-bound. +Call sites in libs/ and xtlang will migrate in a follow-up." +``` + +### Task 2.6: CI gate + +- [ ] **Step 1: Push** + +```bash +git push +``` + +- [ ] **Step 2: Watch CI** + +```bash +gh run watch --branch cpp-modernisation --exit-status +``` + +Expected: all 4 platforms green. `cpp-unit` tests now run as part of `ctest`. + +--- + +## Phase 3: Kill hand-rolled sync primitives + +**Files to delete:** `include/EXTMutex.h`, `src/EXTThread.cpp`, `include/EXTThread.h`, `include/EXTMonitor.h`, `include/EXTCondition.h` +**Files to modify:** every include site — use `grep -rn "EXTMutex\|EXTThread\|EXTMonitor\|EXTCondition" src/ include/` for the full list. Known users: +- `src/SchemeProcess.cpp`, `src/SchemeREPL.cpp`, `src/AudioDevice.cpp`, `src/TaskScheduler.cpp`, `src/EXTZones.cpp`, `src/EXTLLVM.cpp` +- `include/SchemeProcess.h`, `include/SchemeREPL.h`, `include/AudioDevice.h`, `include/TaskScheduler.h` + +### Task 3.1: Enumerate call sites + +- [ ] **Step 1: Map the usage** + +```bash +grep -rn "EXTMutex\|EXTMonitor\|EXTCondition\|EXTThread\|ScopedLock" src/ include/ > /tmp/sync_sites.txt +wc -l /tmp/sync_sites.txt +``` + +Review the file. Group by pattern: +1. `EXTMutex m; m.lock(); ...; m.unlock();` → `std::mutex` + `std::lock_guard` +2. `EXTMutex::ScopedLock l(m);` → `std::lock_guard l(m);` +3. `EXTMonitor` (mutex + condvar bundle) → `std::mutex` + `std::condition_variable` +4. `EXTCondition c; c.wait(m); c.signal();` → `std::condition_variable c; c.wait(lock); c.notify_one();` +5. `EXTThread t(fn, arg); t.start(); t.stop();` → `std::thread` + +### Task 3.2: Replace `EXTMutex` + +- [ ] **Step 1: Read `include/EXTMutex.h` to confirm it's a pure `std::recursive_mutex` wrapper** + +Confirm the semantics — it uses `std::recursive_mutex` unconditionally. Most call sites probably don't need recursion. Be careful: if a call site locks the same mutex from within the lock's critical section, we must keep `std::recursive_mutex` there. + +- [ ] **Step 2: For each call site, pick the right mutex type** + +Default: `std::mutex`. Use `std::recursive_mutex` only if you verify reentrancy. + +- [ ] **Step 3: Replace include + type** + +For example in `src/SchemeProcess.cpp`: + +```cpp +// before: +#include "EXTMutex.h" +extemp::EXTMutex m_mutex{"scheme-proc"}; +extemp::EXTMutex::ScopedLock lock(m_mutex); + +// after: +#include +std::mutex m_mutex; +std::lock_guard lock(m_mutex); +``` + +- [ ] **Step 4: Build after each file** + +```bash +cmake --build build --target extempore --parallel 2 +``` + +Commit every 1-3 files — small commits make bisection cheap. Suggested commit cadence: + +```bash +git add src/SchemeProcess.cpp include/SchemeProcess.h +git commit -m "refactor(sync): SchemeProcess uses std::mutex directly" +``` + +### Task 3.3: Replace `EXTMonitor` / `EXTCondition` + +Same pattern as 3.2. `EXTMonitor` becomes `std::mutex m; std::condition_variable cv;` as members; `wait` becomes `cv.wait(lock, pred)`. + +Commit per-file. + +### Task 3.4: Replace `EXTThread` + +- [ ] **Step 1: Read `src/EXTThread.cpp` and `include/EXTThread.h`** + +Identify its API surface — most likely `start(fn, arg)`, `stop()`, `join()`, `detach()`, priority setting. Note any custom-stack-size or realtime-priority logic — `std::thread` doesn't natively support RT priority, so check if that needs preserving via `pthread_setschedparam` on the native handle. + +- [ ] **Step 2: Pick a migration strategy** + +If `EXTThread` does have RT-priority logic we need, don't delete the file yet — reduce it to a thin helper around `std::thread::native_handle()`. Name it `ext::rt_thread` and keep the file if it's genuinely pulling weight. Otherwise delete. + +- [ ] **Step 3: Replace call sites** + +Typical: + +```cpp +// before: +auto* t = new EXTThread(&func, arg, "name"); +t->start(); + +// after: +std::jthread t([arg] { func(arg); }); +// (assuming func signature suits the lambda) +``` + +For long-lived threads owned by singletons, prefer `std::jthread` (stops on destruction) over raw `std::thread`. + +### Task 3.5: Delete the headers + +- [ ] **Step 1: Confirm zero remaining references** + +```bash +grep -rn "EXTMutex\|EXTMonitor\|EXTCondition\|EXTThread" src/ include/ +``` + +Expected: nothing except file-deletion candidates themselves. + +- [ ] **Step 2: Delete** + +```bash +git rm include/EXTMutex.h include/EXTMonitor.h include/EXTCondition.h include/EXTThread.h src/EXTThread.cpp +``` + +Remove these from any `EXTEMPORE_SOURCES` list in `CMakeLists.txt` or explicit source lists. + +- [ ] **Step 3: Commit** + +```bash +git add -u +git commit -m "refactor(sync): remove hand-rolled EXT{Mutex,Monitor,Condition,Thread} + +All call sites now use the standard library directly. The EXT wrappers +added no value over std::mutex / std::condition_variable / std::jthread, +used std::recursive_mutex unconditionally (masking reentrancy bugs), +and predated std::lock_guard/std::scoped_lock." +``` + +### Task 3.6: Run .xtm integration tests locally + +- [ ] **Step 1: Build and test** + +```bash +cmake --build build --target extempore --parallel 2 +ctest --test-dir build --label-regex "libs-core|compiler-unit|cpp-unit" -j4 --output-on-failure +``` + +Expected: green. If failures occur, most likely a subtle lock-scope change during migration. + +### Task 3.7: CI gate + +- [ ] **Step 1: Push and watch** + +```bash +git push +gh run watch --branch cpp-modernisation --exit-status +``` + +Expected: all 4 platforms green. Windows is highest risk — `std::thread::native_handle()` and recursive mutex semantics differ subtly. Read any Windows failure output carefully. + +--- + +## Phase 4: Networking safety + +**Files:** +- Modify: `src/OSC.cpp:1109`, `src/SchemeREPL.cpp:135`, `src/EXTLLVM.cpp:286`, `src/LinenoiseREPL.cpp:21` +- Modify: `src/Extempore.cpp:112-117` (signal handler) and `src/Extempore.cpp:176` (`freopen(stderr)`) +- Modify: `src/SchemeREPL.cpp:208-222` (FD sentinel) + +### Task 4.1: Replace `gethostbyname` with `getaddrinfo` + +- [ ] **Step 1: Write a helper in `include/UNIV.h` / `src/UNIV.cpp`** + +```cpp +// include/UNIV.h, in the UNIV namespace: +// Resolves `host` (IPv4 only, to match existing behaviour) to a +// network-order uint32_t. Returns 0 on failure. +uint32_t resolve_ipv4(const char* host); +``` + +```cpp +// src/UNIV.cpp: +#include +#ifdef _WIN32 + #include + #include +#else + #include + #include + #include +#endif + +uint32_t resolve_ipv4(const char* host) { + struct addrinfo hints{}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo* res = nullptr; + if (getaddrinfo(host, nullptr, &hints, &res) != 0 || !res) return 0; + uint32_t addr = reinterpret_cast(res->ai_addr)->sin_addr.s_addr; + freeaddrinfo(res); + return addr; +} +``` + +- [ ] **Step 2: Write a unit test** + +```cpp +TEST(ResolveIpv4, LocalhostResolves) { + uint32_t a = extemp::UNIV::resolve_ipv4("127.0.0.1"); + EXPECT_NE(a, 0u); + // 127.0.0.1 in network byte order + EXPECT_EQ(a, htonl(0x7f000001)); +} +TEST(ResolveIpv4, BogusHostReturnsZero) { + uint32_t a = extemp::UNIV::resolve_ipv4("this.host.definitely.does.not.resolve.invalid"); + EXPECT_EQ(a, 0u); +} +``` + +- [ ] **Step 3: Replace each call site** + +`src/OSC.cpp:1109`, `src/EXTLLVM.cpp:286`, `src/SchemeREPL.cpp:135`, `src/LinenoiseREPL.cpp:21`: + +```cpp +// before: +hen = gethostbyname(host); +if (hen == nullptr) return ...; +memcpy(&addr.sin_addr, hen->h_addr, hen->h_length); + +// after: +uint32_t resolved = extemp::UNIV::resolve_ipv4(host); +if (resolved == 0) return ...; +addr.sin_addr.s_addr = resolved; +``` + +- [ ] **Step 4: Build, test, commit** + +```bash +cmake --build build --target extempore extempore_cpp_unit +ctest --test-dir build --label-regex cpp-unit --output-on-failure +git add src/ include/UNIV.h tests/cpp-unit/univ_test.cpp +git commit -m "refactor(net): replace gethostbyname with getaddrinfo helper + +gethostbyname is deprecated, not thread-safe, and IPv4-only. +Consolidates four call sites behind extemp::UNIV::resolve_ipv4. +IPv6 support is future work — the helper signature stays IPv4 to +preserve existing callers' expected layout." +``` + +### Task 4.2: Fix FD sentinel in `SchemeREPL.cpp` + +- [ ] **Step 1: Find the default initialiser** + +```bash +grep -n "m_serverSocket" src/SchemeREPL.cpp include/SchemeREPL.h +``` + +If `m_serverSocket` defaults to `0`, change to `-1`. Update every check from `m_serverSocket > 0` or `!= 0` to `!= -1`. + +- [ ] **Step 2: Wrap the close path** + +In `closeREPL` (lines ~208-222), change `close(m_serverSocket)` to guard against `-1`: + +```cpp +if (m_serverSocket != -1) { + ::shutdown(m_serverSocket, SHUT_RDWR); + ::close(m_serverSocket); + m_serverSocket = -1; +} +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/SchemeREPL.cpp include/SchemeREPL.h +git commit -m "fix(repl): use -1 as socket sentinel to avoid closing stdin" +``` + +### Task 4.3: Signal-handler safety + +**File:** `src/Extempore.cpp:112-117` + +- [ ] **Step 1: Rewrite `sig_handler` to be async-signal-safe** + +```cpp +static void sig_handler(int sig) { + // Async-signal-safe: no printf, no malloc, no exit() + const char msg[] = "\nextempore: caught signal, exiting\n"; + ssize_t r = ::write(STDERR_FILENO, msg, sizeof(msg) - 1); + (void)r; // ignore return; there's nothing useful to do + _Exit(128 + sig); +} +``` + +Include `` (POSIX) and `` (for `_Exit`). On Windows, keep whatever the existing `#ifdef _WIN32` guard expects. + +- [ ] **Step 2: Commit** + +```bash +git add src/Extempore.cpp +git commit -m "fix: make sig_handler async-signal-safe" +``` + +### Task 4.4: Remove or gate the stderr silencing + +**File:** `src/Extempore.cpp:176` + +- [ ] **Step 1: Gate behind a command-line flag** + +Add a `--quiet` option or simply remove the line. The line silences all LLVM diagnostics for the life of the process, making crash investigation nearly impossible. + +Recommendation: remove outright. If a user wants quiet stderr, they can shell-redirect. + +- [ ] **Step 2: Commit** + +```bash +git add src/Extempore.cpp +git commit -m "fix: stop silencing stderr (was hiding LLVM diagnostics)" +``` + +### Task 4.5: CI gate + +- [ ] **Step 1: Push and watch** + +```bash +git push +gh run watch --branch cpp-modernisation --exit-status +``` + +Expected: all green. Network-stack changes are the most platform-sensitive work in the whole plan; Windows `winsock2.h` header ordering is fragile. Be ready to add a `#define WIN32_LEAN_AND_MEAN` or reorder includes if Windows explodes. + +--- + +## Phase 5: Convert deferred findings to backlog tasks + +For the audit items too large or design-sensitive for this refactor, create focused backlog tasks so they aren't forgotten. + +### Task 5.1: Create backlog entries + +- [ ] **Step 1: Create one backlog task per deferred item** + +```bash +backlog task create "RAII conversion: SchemeProcess/REPL/AudioDevice thread ownership" \ + --priority medium -l cpp,refactor \ + -d "Convert remaining owning raw pointers to unique_ptr/value types: +- AudioDevice::m_threads[MAX_RT_AUDIO_THREADS] → std::array +- SchemeProcess/SchemeREPL singletons → Meyers singletons +- SchemeTask::m_ptr → std::variant +- Per-thread thread_local PRNGs (EXTLLVM.cpp:392,397) → values not pointers +See docs/superpowers/plans/2026-04-19-cpp-modernisation.md §5." + +backlog task create "Audit LLVM_SCHEME_FF_MAP and FFI table thread safety" \ + --priority high -l cpp,concurrency \ + -d "src/EXTLLVM.cpp:137 (LLVM_SCHEME_FF_MAP) and src/SchemeS7.cpp:25-57 +(s_ffiCount/s_ffiTable) are acknowledged in comments as not thread safe. +Audit real access patterns and guard with shared_mutex or make atomic +where appropriate." + +backlog task create "Zone allocator mutex on audio thread" \ + --priority high -l cpp,rt-audio \ + -d "src/EXTZones.cpp:69-73 takes a global std::mutex on every allocation +in the audio-callback path. This is an RT-audio anti-pattern (priority +inversion). Investigate lock-free zone design or per-thread zones." + +backlog task create "Consolidate src/OSC.cpp (UDP/TCP × Win/POSIX duplication)" \ + --priority medium -l cpp,refactor \ + -d "1700+ lines with UDP/TCP paths duplicated across Windows and POSIX. +Byte-swap helpers reimplement __builtin_bswap*. Large standalone project." + +backlog task create "Audio thread dispatcher race conditions" \ + --priority high -l cpp,rt-audio,concurrency \ + -d "src/AudioDevice.cpp:169-170, 391 — multi-thread dispatch uses +spin-sleep on atomics and an unsynchronised m_numThreads read (comment +at 391 says 'this is a race :('). Replace with std::counting_semaphore +or condvar. TSan run required." + +backlog task create "Modernise llvm_scheme_env_set dispatch" \ + --priority low -l cpp,refactor \ + -d "src/EXTLLVM.cpp:448-590 is a 140-line strcmp-dispatched type handler. +Replace with enum + table or std::unordered_map." + +backlog task create "Replace select() with poll/epoll in SchemeProcess::serverImpl" \ + --priority low -l cpp,refactor \ + -d "src/SchemeProcess.cpp:469-602 uses select() (1024-fd limit). +Consider poll on POSIX, WSAPoll on Windows, or the existing asio +networking-TS bundle." +``` + +- [ ] **Step 2: Commit the plan document itself** + +```bash +git add docs/superpowers/plans/2026-04-19-cpp-modernisation.md backlog/tasks/ +git commit -m "docs: C++ modernisation refactor plan and follow-up tasks" +``` + +### Task 5.2: Final CI validation + +- [ ] **Step 1: One more full CI check on the branch** + +```bash +gh run watch --branch cpp-modernisation --exit-status +``` + +- [ ] **Step 2: Open PR** + +```bash +gh pr create --title "C++ modernisation: phases 1-4 (build hygiene, UNIV fixes, sync, networking)" \ + --body "$(cat <<'EOF' +## Summary +Applies phases 1-4 of the 2026-04-19 C++ audit as a single coordinated branch. + +- **Phase 1** — build hygiene: -Wall/-Wextra with bug-finding errors; sanitizer build option; .clang-format; .clang-tidy. +- **Phase 2** — UNIV.cpp correctness: sys_slurp_file off-by-one, cname_decode leak/shadow, base64 unsequenced side effects, rreplace/rsplit bounds. Adds a cpp-unit test target with GoogleTest. +- **Phase 3** — deletes EXTMutex/EXTMonitor/EXTCondition/EXTThread in favour of std::mutex / std::condition_variable / std::jthread. +- **Phase 4** — networking safety: getaddrinfo replaces gethostbyname; FD sentinel fix; async-signal-safe sig_handler; removes stderr silencing. + +See `docs/superpowers/plans/2026-04-19-cpp-modernisation.md` for the phased plan and `backlog/tasks/` for deferred follow-ups. + +## Test plan +- [x] CI green on Linux x64 / Linux arm64 / macOS arm64 / Windows x64 +- [x] cpp-unit tests pass under ASan and UBSan locally +- [x] All existing libs-core and libs-external ctest labels remain green +- [x] Manual smoke: `./extempore --batch "(begin (println 'hello) (quit 0))"` +EOF +)" +``` + +--- + +## Out of scope for this plan + +Deliberately left for the follow-up backlog tasks created in Task 5.1: + +- Broad RAII refactor of audio/scheme subsystem thread ownership +- Audio-thread race conditions in `AudioDevice.cpp` +- Zone allocator realtime-safety redesign +- `LLVM_SCHEME_FF_MAP` / s7 FFI table synchronisation +- `OSC.cpp` consolidation +- `llvm_scheme_env_set` dispatch table +- `select()` → `poll`/`epoll` +- Mass clang-format of the codebase (already tracked as TASK-023) +- Full clang-tidy fixup wave (already tracked as TASK-030) +- `typedef` → `using`, `NULL` → `nullptr` cleanup (mechanical, large diff — separate pass) + +These are each independently valuable and each warrants its own focused PR. diff --git a/extras/cmake/tests.cmake b/extras/cmake/tests.cmake index a0896845..288400e9 100644 --- a/extras/cmake/tests.cmake +++ b/extras/cmake/tests.cmake @@ -7,6 +7,8 @@ endif() include(CTest) +add_subdirectory(tests/cpp-unit) + set(EXTEMPORE_TEST_PORT_COUNTER 17099 CACHE INTERNAL "") function(extempore_get_next_port OUT_VAR) diff --git a/include/EXTCondition.h b/include/EXTCondition.h deleted file mode 100644 index a9cc92a5..00000000 --- a/include/EXTCondition.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2011, Andrew Sorensen - * - * All rights reserved. - * - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the authors nor other contributors may be used to endorse - * or promote products derived from this software without specific prior written - * permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef EXT_CONDITION -#define EXT_CONDITION - -#include -#include "EXTMutex.h" - -namespace extemp -{ - -class EXTCondition -{ -private: - std::condition_variable_any m_cond; -public: - EXTCondition() { - } - - void wait(EXTMutex* Mutex) { - std::unique_lock lock(Mutex->m_mutex); - m_cond.wait(lock); - } - - void signal() { - m_cond.notify_one(); - } -}; - -} //End Namespace - -#endif diff --git a/include/EXTLLVM.h b/include/EXTLLVM.h index 7bd73c98..d1dec3f3 100644 --- a/include/EXTLLVM.h +++ b/include/EXTLLVM.h @@ -37,7 +37,6 @@ #include #include -#include #include #include diff --git a/include/EXTMonitor.h b/include/EXTMonitor.h deleted file mode 100644 index 5a04d175..00000000 --- a/include/EXTMonitor.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2011, Andrew Sorensen - * - * All rights reserved. - * - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the authors nor other contributors may be used to endorse - * or promote products derived from this software without specific prior written - * permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef EXT_MONITOR_H -#define EXT_MONITOR_H - -#include - -#include "EXTMutex.h" -#include "EXTCondition.h" - -namespace extemp -{ -class EXTMonitor -{ -public: - class ScopedLock - { - private: - EXTMonitor& m_monitor; - bool m_signal; - public: - ScopedLock(EXTMonitor& Monitor, bool Signal = false): m_monitor(Monitor), m_signal(Signal) { - m_monitor.lock(); - } - ~ScopedLock() { - if (m_signal) { - m_monitor.signal(); - } - m_monitor.unlock(); - } - }; -private: - std::string m_name; - EXTMutex m_mutex; - EXTCondition m_condition; -public: - EXTMonitor(const std::string& Name): m_name(Name), m_mutex(Name) { - } - void lock() { - m_mutex.lock(); - } - void unlock() { - m_mutex.unlock(); - } - void wait() { - m_condition.wait(&m_mutex); - } - void signal() { - m_condition.signal(); - } -}; - -} //End Namespace - -#endif diff --git a/include/EXTMutex.h b/include/EXTMutex.h deleted file mode 100644 index 4a95977c..00000000 --- a/include/EXTMutex.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2011, Andrew Sorensen - * - * All rights reserved. - * - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the authors nor other contributors may be used to endorse - * or promote products derived from this software without specific prior written - * permission. - * - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef EXT_MUTEX -#define EXT_MUTEX - -#include -#include - -namespace extemp -{ - -class EXTMutex -{ -public: - class ScopedLock - { - private: - EXTMutex& m_mutex; - public: - ScopedLock(EXTMutex& Mutex): m_mutex(Mutex) { - m_mutex.lock(); - } - ~ScopedLock() { - m_mutex.unlock(); - } - }; -private: - std::string m_name; - std::recursive_mutex m_mutex; -public: - EXTMutex(const std::string& Name = std::string()): m_name(Name) { - } - - void lock() { m_mutex.lock(); } - void unlock() { m_mutex.unlock(); } - bool try_lock() { return m_mutex.try_lock(); } - - friend class EXTCondition; -}; - -} //End Namespace - -#endif diff --git a/include/PriorityQueue.h b/include/PriorityQueue.h index 2ace72f6..25778a90 100644 --- a/include/PriorityQueue.h +++ b/include/PriorityQueue.h @@ -40,7 +40,6 @@ #include -#include "EXTMutex.h" #include "Task.h" #include "BranchPrediction.h" diff --git a/include/SchemeProcess.h b/include/SchemeProcess.h index c0664fec..1f690818 100644 --- a/include/SchemeProcess.h +++ b/include/SchemeProcess.h @@ -40,6 +40,9 @@ #include "SchemeS7Private.h" #include #include "Task.h" +#include "EXTThread.h" +#include +#include #include #include #include @@ -67,8 +70,8 @@ class SchemeTask { Type m_type; // 0 = repl task, 1 = callback task, 2 = destroy env task void* m_ptr2; public: - SchemeTask(uint64_t Time, uint64_t MaxDuration, void* Ptr, const std::string& Label, Type Type, void* Ptr2 = 0): - m_time(Time), m_maxDuration(MaxDuration), m_ptr(Ptr), m_label(Label), m_type(Type), m_ptr2(Ptr2) { + SchemeTask(uint64_t Time, uint64_t MaxDuration, void* Ptr, const std::string& Label, Type TaskType, void* Ptr2 = 0): + m_time(Time), m_maxDuration(MaxDuration), m_ptr(Ptr), m_label(Label), m_type(TaskType), m_ptr2(Ptr2) { } uint64_t getTime() const { return m_time; } @@ -90,7 +93,8 @@ class SchemeProcess { bool m_banner; std::string m_initExpr; bool m_libsLoaded; - EXTMonitor m_guard; + std::recursive_mutex m_guardMutex; + std::condition_variable_any m_guardCond; bool m_running; EXTThread m_threadTask; EXTThread m_threadServer; diff --git a/include/SchemeREPL.h b/include/SchemeREPL.h index e3288695..72009625 100644 --- a/include/SchemeREPL.h +++ b/include/SchemeREPL.h @@ -34,7 +34,7 @@ */ #include "UNIV.h" -#include "EXTMutex.h" +#include #include #include @@ -70,7 +70,7 @@ class SchemeREPL { char m_buf[BUFLENGTH]; bool m_connected; bool m_active; - EXTMutex m_writeLock; + std::recursive_mutex m_writeLock; static repls_type sm_repls; public: diff --git a/include/SchemeS7Private.h b/include/SchemeS7Private.h index 1f0b59af..85d48d18 100644 --- a/include/SchemeS7Private.h +++ b/include/SchemeS7Private.h @@ -3,7 +3,6 @@ #include "SchemeS7.h" #include "EXTThread.h" -#include "EXTMonitor.h" #include "s7.h" #include diff --git a/include/TaskScheduler.h b/include/TaskScheduler.h index 77453ae5..6f5bacd4 100644 --- a/include/TaskScheduler.h +++ b/include/TaskScheduler.h @@ -37,13 +37,23 @@ #define TASK_SCHEDULER_H #include "PriorityQueue.h" -#include "EXTMonitor.h" #include "EXTThread.h" #include "UNIV.h" +#include +#include + namespace extemp { -class EXTMonitor; +// Paired mutex + condvar used by the scheduler and its clients. +struct Monitor { + std::recursive_mutex mutex; + std::condition_variable_any cond; + void lock() { mutex.lock(); } + void unlock() { mutex.unlock(); } + void wait() { std::unique_lock lk(mutex); cond.wait(lk); } + void signal() { cond.notify_one(); } +}; class TaskScheduler { @@ -51,8 +61,8 @@ class TaskScheduler unsigned m_numFrames; PriorityQueue m_queue; EXTThread m_queueThread; - EXTMonitor m_guard; - EXTMutex m_queueMutex; + Monitor m_guard; + std::recursive_mutex m_queueMutex; static TaskScheduler sm_instance; private: @@ -67,10 +77,10 @@ class TaskScheduler void start() { m_queueThread.start(); } void setFrames(unsigned Frames) { m_numFrames = Frames; } - EXTMonitor& getGuard() { return m_guard; } + Monitor& getGuard() { return m_guard; } void add(TaskI* Task) { - EXTMutex::ScopedLock lock(m_queueMutex); + std::lock_guard lock(m_queueMutex); m_queue.add(Task); } template diff --git a/include/ext/FileUtil.h b/include/ext/FileUtil.h new file mode 100644 index 00000000..a55c5d43 --- /dev/null +++ b/include/ext/FileUtil.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace extemp { +namespace file_util { + +inline char* slurp_file(const char* path) { + std::FILE* fp = std::fopen(path, "rb"); + if (!fp) return nullptr; + if (std::fseek(fp, 0, SEEK_END) != 0) { std::fclose(fp); return nullptr; } + long sz = std::ftell(fp); + if (sz < 0) { std::fclose(fp); return nullptr; } + std::rewind(fp); + char* buf = static_cast(std::malloc(static_cast(sz) + 1)); + if (!buf) { std::fclose(fp); return nullptr; } + size_t got = std::fread(buf, 1, static_cast(sz), fp); + std::fclose(fp); + if (got != static_cast(sz)) { std::free(buf); return nullptr; } + buf[sz] = '\0'; + return buf; +} + +} // namespace file_util +} // namespace extemp diff --git a/include/ext/NetUtil.h b/include/ext/NetUtil.h new file mode 100644 index 00000000..408711e2 --- /dev/null +++ b/include/ext/NetUtil.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +#include +#endif + +namespace extemp { +namespace net_util { + +// On Windows, getaddrinfo silently returns WSANOTINITIALISED until +// WSAStartup has been called. The extempore binary initialises Winsock +// elsewhere, but standalone consumers (e.g. the cpp-unit tests) don't. +// Do it here, once, on the first resolve_ipv4 call. +inline void ensure_winsock_initialised() { +#ifdef _WIN32 + static std::once_flag flag; + std::call_once(flag, []() { + WSADATA wsa; + WSAStartup(MAKEWORD(2, 2), &wsa); + }); +#endif +} + +// Resolves `host` (IPv4) to an address in network byte order. +// Returns 0 on failure. Thread-safe (uses getaddrinfo, not gethostbyname). +inline uint32_t resolve_ipv4(const char* host) { + if (!host) return 0; + ensure_winsock_initialised(); + struct addrinfo hints{}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo* res = nullptr; + if (getaddrinfo(host, nullptr, &hints, &res) != 0 || !res) return 0; + uint32_t addr = reinterpret_cast(res->ai_addr)->sin_addr.s_addr; + freeaddrinfo(res); + return addr; +} + +} // namespace net_util +} // namespace extemp diff --git a/src/AudioDevice.cpp b/src/AudioDevice.cpp index a06936db..db4553e3 100644 --- a/src/AudioDevice.cpp +++ b/src/AudioDevice.cpp @@ -47,7 +47,6 @@ #include "AudioDevice.h" #include "TaskScheduler.h" -#include "EXTMonitor.h" #include "EXTLLVM.h" #include "SchemeFFI.h" #include "BranchPrediction.h" diff --git a/src/EXTLLVM.cpp b/src/EXTLLVM.cpp index 2c838965..d727cfaa 100644 --- a/src/EXTLLVM.cpp +++ b/src/EXTLLVM.cpp @@ -81,8 +81,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -280,20 +282,17 @@ EXPORT void llvm_send_udp(char* host, int port, void* message, int message_lengt #else struct sockaddr_in sa; - struct hostent* hen; /* host-to-IP translation */ - /* Address resolution stage */ - hen = gethostbyname(host); - if (!hen) { - printf("OSC Error: Could no resolve host name\n"); + uint32_t resolved = extemp::net_util::resolve_ipv4(host); + if (!resolved) { + printf("OSC Error: Could not resolve host name\n"); return; } memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; sa.sin_port = htons(port); - memcpy(&sa.sin_addr.s_addr, hen->h_addr_list[0], hen->h_length); + sa.sin_addr.s_addr = resolved; #endif @@ -837,34 +836,37 @@ EXPORT double audio_clock_now() } // CATEGORY: native mutex +// +// xtlang code stores the returned void* and may call lock more than once +// from the same thread without a matching unlock, so recursive semantics +// are preserved. EXPORT void* mutex_create() { - auto mutex(new EXTMutex); - return mutex; + return new std::recursive_mutex; } EXPORT int mutex_destroy(void* Mutex) { - delete reinterpret_cast(Mutex); + delete reinterpret_cast(Mutex); return 0; } EXPORT int mutex_lock(void* Mutex) { - reinterpret_cast(Mutex)->lock(); + reinterpret_cast(Mutex)->lock(); return 0; } EXPORT int mutex_unlock(void* Mutex) { - reinterpret_cast(Mutex)->unlock(); + reinterpret_cast(Mutex)->unlock(); return 0; } EXPORT int mutex_trylock(void* Mutex) { - return reinterpret_cast(Mutex)->try_lock(); + return reinterpret_cast(Mutex)->try_lock(); } // CATEGORY: native thread diff --git a/src/EXTZones.cpp b/src/EXTZones.cpp index 31350ed8..b220f030 100644 --- a/src/EXTZones.cpp +++ b/src/EXTZones.cpp @@ -1,10 +1,10 @@ #include -#include #include #include #include #include +#include #define DEBUG_ZONE_ALLOC 0 #define DEBUG_ZONE_STACK 0 @@ -66,11 +66,8 @@ llvm_zone_t* llvm_zone_reset(llvm_zone_t* Zone) EXPORT void* llvm_zone_malloc(llvm_zone_t* zone, uint64_t size) { - static std::unique_ptr alloc_mutex = []() { - std::unique_ptr m(new extemp::EXTMutex("alloc mutex")); - return m; - }(); - extemp::EXTMutex::ScopedLock lock(*alloc_mutex); + static std::recursive_mutex alloc_mutex; + std::lock_guard lock(alloc_mutex); #if DEBUG_ZONE_ALLOC printf("MallocZone: %p:%p:%lld:%lld:%lld\n",zone,zone->memory,zone->offset,zone->size,size); #endif diff --git a/src/Extempore.cpp b/src/Extempore.cpp index b46a1fa0..e86756e5 100644 --- a/src/Extempore.cpp +++ b/src/Extempore.cpp @@ -49,6 +49,8 @@ #ifndef _WIN32 #include "LinenoiseREPL.h" #include +#include +#include #else #undef min #undef max @@ -107,13 +109,20 @@ BOOL CtrlHandler(DWORD fdwCtrlType) void sig_handler(int Signo) { + // Async-signal-safe: write(2) and _Exit are on the POSIX safe list, + // printf / exit are not. We drop the helpful "(SIGINT), exiting..." + // prefix in favour of correctness. if (Signo == SIGINT) { - printf("\nReceived interrupt signal (SIGINT), exiting Extempore...\n"); - _exit(0); + static const char msg[] = "\nextempore: SIGINT, exiting\n"; + ssize_t r = ::write(STDERR_FILENO, msg, sizeof(msg) - 1); + (void)r; + _Exit(128 + SIGINT); } else if (Signo == SIGTERM) { - printf("\nReceived termination signal (SIGTERM), exiting Extempore...\n"); - exit(0); + static const char msg[] = "\nextempore: SIGTERM, exiting\n"; + ssize_t r = ::write(STDERR_FILENO, msg, sizeof(msg) - 1); + (void)r; + _Exit(128 + SIGTERM); } } @@ -172,10 +181,7 @@ EXPORT int extempore_init(int argc, char** argv) int utility_port = 7098; bool repl_mode = false; #ifndef _WIN32 - // redirect stderr to nullptr - freopen("/dev/null", "w", stderr); - - // signal handlers for OSX/Linux + // signal handlers for OSX/Linux if (signal(SIGINT, sig_handler) == SIG_ERR) { printf("\nWarning: can't catch SIGINT.\n"); } diff --git a/src/LinenoiseREPL.cpp b/src/LinenoiseREPL.cpp index 4d89c4c5..9618a0a5 100644 --- a/src/LinenoiseREPL.cpp +++ b/src/LinenoiseREPL.cpp @@ -2,12 +2,12 @@ #include "LinenoiseREPL.h" #include "linenoise/linenoise.h" +#include "ext/NetUtil.h" #include #include #include #include -#include #include #include #include @@ -18,8 +18,8 @@ static int connect_to_server(const std::string& host, int port) { - struct hostent* hen = gethostbyname(host.c_str()); - if (!hen) { + uint32_t resolved = extemp::net_util::resolve_ipv4(host.c_str()); + if (!resolved) { fprintf(stderr, "repl: could not resolve host '%s'\n", host.c_str()); return -1; } @@ -28,7 +28,7 @@ static int connect_to_server(const std::string& host, int port) memset(&sa, 0, sizeof(sa)); sa.sin_family = AF_INET; sa.sin_port = htons(port); - memcpy(&sa.sin_addr.s_addr, hen->h_addr_list[0], hen->h_length); + sa.sin_addr.s_addr = resolved; int retries = 30; for (int i = 0; i < retries; i++) { diff --git a/src/OSC.cpp b/src/OSC.cpp index 74386c7a..39b43757 100644 --- a/src/OSC.cpp +++ b/src/OSC.cpp @@ -35,6 +35,7 @@ #include "OSC.h" #include "SchemeProcess.h" +#include "ext/NetUtil.h" #include #include #include @@ -667,10 +668,10 @@ namespace extemp { int res = select(highest_fd, &c_rfd, nullptr, nullptr, &pause); if(res >= 0) { }else{ - struct stat buf; + struct stat st; std::vector::iterator pos = client_sockets.begin(); while(pos != client_sockets.end()) { - int result = fstat(*pos,&buf); + int result = fstat(*pos,&st); if(result < 0) { FD_CLR(*pos,&rfd); client_sockets.erase(pos); @@ -960,8 +961,8 @@ namespace extemp { #endif *i = swap32i(*i); char* byte_array = (char*) i; - for(int i=0;i<4;++i) { - data[i] = byte_array[i]; + for(int j=0;j<4;++j) { + data[j] = byte_array[j]; } return 4; } @@ -1103,21 +1104,18 @@ namespace extemp { std::experimental::net::ip::udp::endpoint sa = *iter; #else struct sockaddr_in sa; - struct hostent* hen; /* host-to-IP translation */ - /* Address resolution stage */ - hen = gethostbyname(host); - if (!hen) { - printf("OSC Error: Could no resolve host name\n"); + uint32_t resolved = extemp::net_util::resolve_ipv4(host); + if (!resolved) { + printf("OSC Error: Could not resolve host name\n"); delete t->getArg(); return; } memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; sa.sin_port = htons(port); - memcpy(&sa.sin_addr.s_addr, hen->h_addr_list[0], hen->h_length); + sa.sin_addr.s_addr = resolved; #endif diff --git a/src/SchemeFFI.cpp b/src/SchemeFFI.cpp index b697898c..4fdfe2c4 100644 --- a/src/SchemeFFI.cpp +++ b/src/SchemeFFI.cpp @@ -132,7 +132,6 @@ CMRC_DECLARE(xtm); #include //#include -#include #include namespace extemp { namespace SchemeFFI { static llvm::Module* jitCompile(const std::string& String); diff --git a/src/SchemeProcess.cpp b/src/SchemeProcess.cpp index 77983696..c07f25ae 100644 --- a/src/SchemeProcess.cpp +++ b/src/SchemeProcess.cpp @@ -107,7 +107,7 @@ const char* SchemeProcess::sm_banner = "\n" SchemeProcess::SchemeProcess(const std::string& LoadPath, const std::string& Name, int ServerPort, bool Banner, const std::string& InitExpr): m_loadPath(LoadPath), m_name(Name), m_serverPort(ServerPort), - m_banner(Banner), m_initExpr(InitExpr), m_libsLoaded(false), m_guard("scheme_server_guard"), + m_banner(Banner), m_initExpr(InitExpr), m_libsLoaded(false), m_running(true), m_threadTask(&taskTrampoline, this, "SP_task"), m_threadServer(&serverTrampoline, this, "SP_server") { @@ -204,17 +204,19 @@ void SchemeProcess::stop() void SchemeProcess::addCallback(TaskI* TaskAdd, SchemeTask::Type Type) { - EXTMonitor::ScopedLock lock(m_guard, true); + std::lock_guard lock(m_guardMutex); auto currentTime(TaskAdd->getStartTime()); auto duration(TaskAdd->getDuration()); auto task(static_cast*>(TaskAdd)); m_taskQueue.push(SchemeTask(currentTime, duration, task->getArg(), "tmp_label", Type)); + m_guardCond.notify_one(); } void SchemeProcess::createSchemeTask(void* Arg, const std::string& Label, SchemeTask::Type Type) { - EXTMonitor::ScopedLock lock(m_guard, true); + std::lock_guard lock(m_guardMutex); m_taskQueue.push(SchemeTask(extemp::UNIV::TIME, m_maxDuration, Arg, Label, Type)); + m_guardCond.notify_one(); } bool SchemeProcess::loadFile(const std::string& File, const std::string& Path) @@ -292,7 +294,7 @@ void* SchemeProcess::taskImpl() std::this_thread::sleep_for(std::chrono::seconds(2)); // give time for NSApp etc. to init // only load extempore.xtm in primary process if (m_name == "primary") { - EXTMonitor::ScopedLock lock(m_guard); + std::lock_guard lock(m_guardMutex); #ifdef DYLIB auto fs = cmrc::xtm::get_filesystem(); auto data = fs.open("runtime/init.ll"); @@ -341,10 +343,12 @@ void* SchemeProcess::taskImpl() continue; } while (likely(!m_taskQueue.empty() && m_running)) { - m_guard.lock(); - SchemeTask task = m_taskQueue.front(); - m_taskQueue.pop(); - m_guard.unlock(); + SchemeTask task = [&]() { + std::lock_guard lock(m_guardMutex); + SchemeTask t = m_taskQueue.front(); + m_taskQueue.pop(); + return t; + }(); switch (task.getType()) { case SchemeTask::Type::DESTROY_ENV: m_scheme->imp_env.erase(reinterpret_cast(task.getPtr())); @@ -394,8 +398,9 @@ void* SchemeProcess::taskImpl() auto minutes(time / UNIV::MINUTE()); time -= minutes * UNIV::MINUTE(); auto seconds(time / UNIV::SECOND()); - char prompt[32]; - snprintf(prompt, sizeof(prompt), "\n[extempore %.2u:%.2u:%.2u]: ", unsigned(hours), unsigned(minutes), unsigned(seconds)); + char prompt[64]; + snprintf(prompt, sizeof(prompt), "\n[extempore %.2u:%.2u:%.2u]: ", + unsigned(hours % 100), unsigned(minutes % 60), unsigned(seconds % 60)); ss << prompt; } UNIV::printSchemeCell(m_scheme, ss, m_scheme->value); @@ -499,19 +504,19 @@ void* SchemeProcess::serverImpl() if (unlikely(FD_ISSET(m_serverSocket, &curReadFds))) { //check if we have any new accepts on our server socket sockaddr_in client_address; socklen_t clientAddressSize(sizeof(client_address)); - auto res(accept(m_serverSocket, reinterpret_cast(&client_address), &clientAddressSize)); - if (unlikely(res < 0)) { + auto sock(accept(m_serverSocket, reinterpret_cast(&client_address), &clientAddressSize)); + if (unlikely(sock < 0)) { std::cout << "Bad Accept in Server Socket Handling" << std::endl; continue; //continue on error } - numFds = int(res) + 1; - FD_SET(res, &readFds); //add new socket to the FD_SET + numFds = int(sock) + 1; + FD_SET(sock, &readFds); //add new socket to the FD_SET ascii_info(); printf("INFO:"); ascii_default(); std::cout << " server: accepted new connection to " << m_name << " process" << std::endl; - clientSockets.push_back(res); - inStrings[res].clear(); + clientSockets.push_back(sock); + inStrings[sock].clear(); std::string outString; if (m_banner) { outString += sm_banner; @@ -521,13 +526,14 @@ void* SchemeProcess::serverImpl() auto minutes(time / UNIV::MINUTE()); time -= minutes * UNIV::MINUTE(); auto seconds(time / UNIV::SECOND()); - char prompt[32]; - snprintf(prompt, sizeof(prompt), "[extempore %.2u:%.2u:%.2u]: ", unsigned(hours), unsigned(minutes), unsigned(seconds)); + char prompt[64]; + snprintf(prompt, sizeof(prompt), "[extempore %.2u:%.2u:%.2u]: ", + unsigned(hours % 100), unsigned(minutes % 60), unsigned(seconds % 60)); outString += prompt; } else { outString += "Welcome to extempore!"; } - send(res, outString.c_str(), int(outString.length() + 1), 0); + send(sock, outString.c_str(), int(outString.length() + 1), 0); continue; } for (unsigned index = 0; index < clientSockets.size(); ++index) { @@ -577,12 +583,13 @@ void* SchemeProcess::serverImpl() std::string::size_type pos = 0; std::string::size_type end = evalStr.find_first_of('\x0d', pos); for (; end != std::string::npos; pos = end + 2, end = evalStr.find_first_of('\x0d', pos)) { - EXTMonitor::ScopedLock lock(m_guard, true); + std::lock_guard lock(m_guardMutex); char c[16]; snprintf(c, sizeof(c), "%i", int(sock)); std::string* s = new std::string(evalStr.substr(pos, end - pos + 1)); // std::cout << extemp::UNIV::TIME << "> SCHEME TASK WITH SUBEXPR:" << *s << std::endl; m_taskQueue.push(SchemeTask(extemp::UNIV::TIME, m_maxDuration, s, c, SchemeTask::Type::REPL)); + m_guardCond.notify_one(); } } } diff --git a/src/SchemeREPL.cpp b/src/SchemeREPL.cpp index 1f5e9193..e6978434 100644 --- a/src/SchemeREPL.cpp +++ b/src/SchemeREPL.cpp @@ -34,8 +34,8 @@ */ #include "SchemeREPL.h" -#include "EXTMutex.h" #include "UNIV.h" +#include "ext/NetUtil.h" #include @@ -65,7 +65,12 @@ namespace extemp { std::unordered_map SchemeREPL::sm_repls; SchemeREPL::SchemeREPL(const std::string& Title, SchemeProcess* Process): m_title(Title), m_process(Process), - m_serverSocket(0), m_connected(false), m_active(true), m_writeLock("repl_lock") +#ifdef _WIN32 + m_serverSocket(nullptr), +#else + m_serverSocket(-1), +#endif + m_connected(false), m_active(true) { ascii_info(); printf("INFO:"); @@ -76,10 +81,16 @@ SchemeREPL::SchemeREPL(const std::string& Title, SchemeProcess* Process): m_titl void SchemeREPL::writeString(std::string&& String) { +#ifdef _WIN32 if (!m_serverSocket) { return; } - EXTMutex::ScopedLock lock(m_writeLock); +#else + if (m_serverSocket == -1) { + return; + } +#endif + std::lock_guard lock(m_writeLock); String.push_back('\r'); String.push_back('\n'); int length = String.length(); @@ -130,10 +141,9 @@ bool SchemeREPL::connectToProcessAtHostname(const std::string& hostname, int por //std::cout << "resolved: " << ep << std::endl << std::flush; if(iter == end) { #else - struct sockaddr_in sa; - struct hostent* hen; /* host-to-IP translation */ - hen = gethostbyname(hostname.c_str()); - if (!hen) { + struct sockaddr_in sa; + uint32_t resolved = extemp::net_util::resolve_ipv4(hostname.c_str()); + if (!resolved) { #endif ascii_error(); printf("Could not resolve host name\n"); @@ -160,7 +170,7 @@ bool SchemeREPL::connectToProcessAtHostname(const std::string& hostname, int por memset(&sa, 0, sizeof(sa)); sa.sin_family = AF_INET; sa.sin_port = htons(port); - memcpy(&sa.sin_addr.s_addr, hen->h_addr_list[0], hen->h_length); + sa.sin_addr.s_addr = resolved; m_serverSocket = socket(AF_INET, SOCK_STREAM, 0); if (m_serverSocket < 0) { @@ -209,15 +219,20 @@ void SchemeREPL::closeREPL() { m_active = false; #ifdef _WIN32 - m_serverSocket->close(); - delete m_serverSocket; + if (m_serverSocket) { + m_serverSocket->close(); + delete m_serverSocket; + m_serverSocket = nullptr; + } delete m_serverIoService; - m_serverIoService = 0; + m_serverIoService = nullptr; #else - shutdown(m_serverSocket, SHUT_RDWR); - close(m_serverSocket); + if (m_serverSocket != -1) { + shutdown(m_serverSocket, SHUT_RDWR); + close(m_serverSocket); + m_serverSocket = -1; + } #endif - m_serverSocket = 0; m_connected = false; } diff --git a/src/TaskScheduler.cpp b/src/TaskScheduler.cpp index 7632a251..40cc6ebe 100644 --- a/src/TaskScheduler.cpp +++ b/src/TaskScheduler.cpp @@ -34,7 +34,6 @@ */ #include "TaskScheduler.h" -#include "EXTMonitor.h" #include "AudioDevice.h" #include #include @@ -44,8 +43,7 @@ namespace extemp { TaskScheduler TaskScheduler::sm_instance; -TaskScheduler::TaskScheduler(): m_numFrames(0), m_queueThread(TaskScheduler::queueThread, this, "scheduler"), - m_guard("task_scheduler_guard"), m_queueMutex("taskQueue") +TaskScheduler::TaskScheduler(): m_numFrames(0), m_queueThread(TaskScheduler::queueThread, this, "scheduler") { } diff --git a/src/UNIV.cpp b/src/UNIV.cpp index e4d1af0f..f20a9d4d 100644 --- a/src/UNIV.cpp +++ b/src/UNIV.cpp @@ -44,6 +44,7 @@ #include #include "SchemeFFI.h" #include "SchemeS7Private.h" +#include "ext/FileUtil.h" #ifdef __APPLE__ #include @@ -166,46 +167,50 @@ EXPORT char* cname_decode(char *data, size_t input_length, size_t *output_length { if (base64_codesafe_decoding_table == nullptr) base64_codesafe_build_decoding_table(); - char* d2 = nullptr; - // pad with $'s + // Pad with $'s to align to 4 bytes. pad_buf is separate from the outer + // scope so we always know which buffer to free at the end. + char* pad_buf = nullptr; if (input_length % 4 != 0) { - int lgthdiff = (4-(input_length % 4)); - char* d2 = (char*) malloc(input_length+lgthdiff); - memcpy(d2,data,input_length); - input_length = input_length+lgthdiff; - for(int i=0;i uint32_t { + unsigned char c = static_cast(data[i++]); + return c == '$' ? 0u : base64_codesafe_decoding_table[c]; + }; - uint32_t sextet_a = data[i] == '$' ? 0 & i++ : base64_codesafe_decoding_table[unsigned(data[i++])]; - uint32_t sextet_b = data[i] == '$' ? 0 & i++ : base64_codesafe_decoding_table[unsigned(data[i++])]; - uint32_t sextet_c = data[i] == '$' ? 0 & i++ : base64_codesafe_decoding_table[unsigned(data[i++])]; - uint32_t sextet_d = data[i] == '$' ? 0 & i++ : base64_codesafe_decoding_table[unsigned(data[i++])]; + for (unsigned i = 0, j = 0; i < input_length;) { + uint32_t sextet_a = fetch(i); + uint32_t sextet_b = fetch(i); + uint32_t sextet_c = fetch(i); + uint32_t sextet_d = fetch(i); uint32_t triple = (sextet_a << 3 * 6) - + (sextet_b << 2 * 6) - + (sextet_c << 1 * 6) - + (sextet_d << 0 * 6); + + (sextet_b << 2 * 6) + + (sextet_c << 1 * 6) + + (sextet_d << 0 * 6); if (j < *output_length) decoded_data[j++] = (triple >> 2 * 8) & 0xFF; if (j < *output_length) decoded_data[j++] = (triple >> 1 * 8) & 0xFF; if (j < *output_length) decoded_data[j++] = (triple >> 0 * 8) & 0xFF; } - if (d2) free(d2); - //printf("DECODE: %d:%s\n",*output_length,decoded_data); + if (pad_buf) free(pad_buf); return decoded_data; } @@ -252,12 +257,17 @@ EXPORT unsigned char* base64_decode(const char *data, size_t input_length, size_ unsigned char *decoded_data = (unsigned char*) malloc(*output_length); if (decoded_data == nullptr) return nullptr; + auto fetch = [&](unsigned& i) -> uint32_t { + unsigned char c = static_cast(data[i++]); + return c == '=' ? 0u : base64_std_decoding_table[c]; + }; + for (unsigned i = 0, j = 0; i < input_length;) { - uint32_t sextet_a = data[i] == '=' ? 0 & i++ : base64_std_decoding_table[unsigned(data[i++])]; - uint32_t sextet_b = data[i] == '=' ? 0 & i++ : base64_std_decoding_table[unsigned(data[i++])]; - uint32_t sextet_c = data[i] == '=' ? 0 & i++ : base64_std_decoding_table[unsigned(data[i++])]; - uint32_t sextet_d = data[i] == '=' ? 0 & i++ : base64_std_decoding_table[unsigned(data[i++])]; + uint32_t sextet_a = fetch(i); + uint32_t sextet_b = fetch(i); + uint32_t sextet_c = fetch(i); + uint32_t sextet_d = fetch(i); uint32_t triple = (sextet_a << 3 * 6) + (sextet_b << 2 * 6) @@ -305,14 +315,18 @@ EXPORT int64_t rmatches(char* regex, char* str, char** results, int64_t maxnum) EXPORT bool rsplit(const char* regex, const char* str, char* a, char* b) { + // Callers in libs/core/adt.xtm allocate 2048-byte buffers for a and b. + // Bail rather than write past the buffer. A proper API with explicit + // capacity is tracked as a follow-up backlog task. + constexpr size_t kAssumedBufSize = 2048; try { - std::string s(str); std::regex re(regex); std::cmatch m; if (!std::regex_search(str, m, re) || m.size() != 1) return false; - int range = static_cast(m.position(0)); - int range2 = range + static_cast(m.length(0)); - int length = static_cast(strlen(str)); + size_t range = static_cast(m.position(0)); + size_t range2 = range + static_cast(m.length(0)); + size_t length = strlen(str); + if (range >= kAssumedBufSize || (length - range2) >= kAssumedBufSize) return false; memcpy(a, str, range); a[range] = '\0'; memcpy(b, str + range2, length - range2); @@ -348,27 +362,12 @@ EXPORT const char* sys_sharedir(){ EXPORT char* sys_slurp_file(const char* fname) { - std::string filename(fname); - std::string sharedir_filename(extemp::UNIV::SHARE_DIR + "/" + filename); - - // check raw path first, then prepend SHARE_DIR - std::FILE *fp = std::fopen(filename.c_str(), "rb"); - if (!fp) { - fp = std::fopen(sharedir_filename.c_str(), "rb"); + // Try the raw path first, then prepend SHARE_DIR. + if (char* buf = extemp::file_util::slurp_file(fname)) { + return buf; } - - if(fp){ - std::fseek(fp, 0, SEEK_END); - size_t file_size = std::ftell(fp); - char* buf = (char*)malloc(file_size*sizeof(char)); - std::rewind(fp); - std::fread(buf, 1, file_size, fp); - std::fclose(fp); - - buf[file_size-1] = '\0'; - return buf; - } - return nullptr; + std::string sharedir_filename(extemp::UNIV::SHARE_DIR + "/" + fname); + return extemp::file_util::slurp_file(sharedir_filename.c_str()); } EXPORT int register_for_window_events() diff --git a/tests/cpp-unit/CMakeLists.txt b/tests/cpp-unit/CMakeLists.txt new file mode 100644 index 00000000..15c1f388 --- /dev/null +++ b/tests/cpp-unit/CMakeLists.txt @@ -0,0 +1,24 @@ +add_executable(extempore_cpp_unit + univ_test.cpp + netutil_test.cpp) + +target_link_libraries(extempore_cpp_unit PRIVATE + gtest_main) + +if(WIN32) + target_link_libraries(extempore_cpp_unit PRIVATE ws2_32) +endif() + +target_include_directories(extempore_cpp_unit PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/src) + +target_compile_features(extempore_cpp_unit PRIVATE cxx_std_17) + +if(UNIX AND NOT APPLE) + target_link_libraries(extempore_cpp_unit PRIVATE pthread) +endif() + +include(GoogleTest) +gtest_discover_tests(extempore_cpp_unit + PROPERTIES LABELS "cpp-unit") diff --git a/tests/cpp-unit/netutil_test.cpp b/tests/cpp-unit/netutil_test.cpp new file mode 100644 index 00000000..c4fdd2ef --- /dev/null +++ b/tests/cpp-unit/netutil_test.cpp @@ -0,0 +1,20 @@ +#include + +#include "ext/NetUtil.h" + +TEST(ResolveIpv4, LocalhostResolves) { + uint32_t a = extemp::net_util::resolve_ipv4("127.0.0.1"); + EXPECT_NE(a, 0u); + // 127.0.0.1 in network byte order + EXPECT_EQ(a, htonl(0x7f000001)); +} + +TEST(ResolveIpv4, NullReturnsZero) { + EXPECT_EQ(extemp::net_util::resolve_ipv4(nullptr), 0u); +} + +TEST(ResolveIpv4, BogusHostReturnsZero) { + // Using a TLD that cannot resolve per RFC 2606 / RFC 6761. + uint32_t a = extemp::net_util::resolve_ipv4("does-not-resolve.invalid"); + EXPECT_EQ(a, 0u); +} diff --git a/tests/cpp-unit/univ_test.cpp b/tests/cpp-unit/univ_test.cpp new file mode 100644 index 00000000..d9880cb4 --- /dev/null +++ b/tests/cpp-unit/univ_test.cpp @@ -0,0 +1,75 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ext/FileUtil.h" + +namespace { + +std::string write_temp(const std::string& contents) { + static std::atomic counter{0}; + auto p = std::filesystem::temp_directory_path() / + ("extempore_test_" + std::to_string(counter.fetch_add(1)) + ".bin"); + std::ofstream f(p, std::ios::binary); + f.write(contents.data(), static_cast(contents.size())); + f.close(); + return p.string(); +} + +} // namespace + +TEST(SlurpFile, ReadsAsciiFullyWithNullTerminator) { + auto path = write_temp("hello"); + char* out = extemp::file_util::slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_EQ(std::string(out), "hello"); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SlurpFile, HandlesEmpty) { + auto path = write_temp(""); + char* out = extemp::file_util::slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_EQ(out[0], '\0'); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SlurpFile, PreservesLastByte) { + // The pre-fix implementation wrote the terminator *into* the last content + // byte, truncating files by one character. This test guards that regression. + auto path = write_temp("abcde"); + char* out = extemp::file_util::slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_EQ(out[4], 'e'); + EXPECT_EQ(out[5], '\0'); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SlurpFile, PreservesBytesAcrossEmbeddedNull) { + std::string contents("ab\0cd", 5); + auto path = write_temp(contents); + char* out = extemp::file_util::slurp_file(path.c_str()); + ASSERT_NE(out, nullptr); + EXPECT_EQ(out[0], 'a'); + EXPECT_EQ(out[1], 'b'); + EXPECT_EQ(out[2], '\0'); + EXPECT_EQ(out[3], 'c'); + EXPECT_EQ(out[4], 'd'); + EXPECT_EQ(out[5], '\0'); + std::free(out); + std::filesystem::remove(path); +} + +TEST(SlurpFile, MissingFileReturnsNull) { + char* out = extemp::file_util::slurp_file("/nonexistent/path/that/does/not/exist"); + EXPECT_EQ(out, nullptr); +}