From 0a6175c390c112bed3c88570278e05fef78804e5 Mon Sep 17 00:00:00 2001 From: Yuansheng Wang Date: Sun, 17 May 2026 22:38:05 +0800 Subject: [PATCH 1/3] bench: replace dense-100k with github-100k scenario Replace the synthetic dense payload (many tiny k/v pairs) with a GitHub Issues API simulation that better reflects real-world API responses: nested user objects, labels arrays, realistic URL and timestamp strings, ~3-5% structural density. The new scenario provides a more meaningful benchmark for typical REST API parsing workloads. --- benches/lua_bench.lua | 134 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 116 insertions(+), 18 deletions(-) diff --git a/benches/lua_bench.lua b/benches/lua_bench.lua index 29db44b..659415f 100644 --- a/benches/lua_bench.lua +++ b/benches/lua_bench.lua @@ -24,6 +24,73 @@ end -- the final image falls through to `math.max(1024, remaining)` — undershoot -- is at most a few hundred bytes; worst-case overshoot is ~1 KB (only when -- `remaining < 1024`, which the seed=42 walk does not hit for our ladder). +-- GitHub-style payload: simulates /repos/{owner}/{repo}/issues response. +-- Each issue has ~20 fields including nested user object, labels array, +-- and realistic string lengths (URLs, timestamps, markdown body). +-- Structural density ~3-5%, matching real GitHub API responses. +local function make_github_issues_payload(target_bytes) + local issues = {} + local current = 2 -- outer envelope: [...] + + local issue_num = 1 + while current < target_bytes do + local labels = {} + local label_count = (issue_num % 4) -- 0-3 labels per issue + for i = 1, label_count do + labels[#labels + 1] = string.format( + '{"id":%d,"name":"label-%d","color":"%06x","description":"Label description for categorization"}', + 10000 + issue_num * 10 + i, i, (issue_num * 12345 + i) % 0xFFFFFF) + end + + local body_len = 200 + (issue_num % 5) * 100 -- 200-600 chars + local body = string.rep("Lorem ipsum dolor sit amet. ", math.ceil(body_len / 29)):sub(1, body_len) + + local issue = string.format([[{ +"id":%d, +"number":%d, +"title":"Issue title describing the problem or feature request #%d", +"body":"%s", +"state":"%s", +"locked":%s, +"comments":%d, +"user":{"login":"user%d","id":%d,"avatar_url":"https://avatars.githubusercontent.com/u/%d?v=4","type":"User","site_admin":false}, +"labels":[%s], +"assignees":[], +"milestone":null, +"created_at":"2024-%02d-%02dT%02d:%02d:%02dZ", +"updated_at":"2024-%02d-%02dT%02d:%02d:%02dZ", +"closed_at":null, +"author_association":"CONTRIBUTOR", +"html_url":"https://github.com/example/repo/issues/%d", +"url":"https://api.github.com/repos/example/repo/issues/%d", +"repository_url":"https://api.github.com/repos/example/repo", +"labels_url":"https://api.github.com/repos/example/repo/issues/%d/labels{/name}", +"comments_url":"https://api.github.com/repos/example/repo/issues/%d/comments", +"events_url":"https://api.github.com/repos/example/repo/issues/%d/events" +}]], + 1000000 + issue_num, + issue_num, + issue_num, + body, + issue_num % 3 == 0 and "closed" or "open", + issue_num % 7 == 0 and "true" or "false", + issue_num % 50, + issue_num % 100, 100000 + issue_num, 100000 + issue_num, + table.concat(labels, ","), + (issue_num % 12) + 1, (issue_num % 28) + 1, issue_num % 24, issue_num % 60, issue_num % 60, + (issue_num % 12) + 1, (issue_num % 28) + 1, (issue_num + 1) % 24, (issue_num + 5) % 60, (issue_num + 10) % 60, + issue_num, issue_num, issue_num, issue_num, issue_num) + + -- Remove newlines for compact JSON + issue = issue:gsub("\n", "") + issues[#issues + 1] = issue + current = current + #issue + 1 + issue_num = issue_num + 1 + end + + return "[" .. table.concat(issues, ",") .. "]" +end + local function make_payload(target_bytes) local rng_state = 42 local function rng_range(lo, hi) @@ -94,9 +161,49 @@ local function bench(name, iters, fn) name, median, mean, lo, hi, mem_after - mem_before)) end +-- Default accessors for multimodal payloads +local function default_cjson_access(obj) + local _ = obj.model + local _ = obj.temperature + local _ = obj.messages and obj.messages[1] and obj.messages[1].role +end + +local function default_qd_access(d) + local _ = d:get_str("model") + local _ = d:get_f64("temperature") + local _ = d:get_str("messages[0].role") +end + +local function default_table_access(t) + local _ = t.model + local _ = t.temperature + local _ = t.messages and t.messages[1] and t.messages[1].role +end + +-- GitHub issues accessors: array of issues, access first issue's fields +local function github_cjson_access(obj) + local _ = obj[1] and obj[1].id + local _ = obj[1] and obj[1].title + local _ = obj[1] and obj[1].user and obj[1].user.login +end + +local function github_qd_access(d) + local _ = d:get_i64("[0].id") + local _ = d:get_str("[0].title") + local _ = d:get_str("[0].user.login") +end + +local function github_table_access(t) + local _ = t[1] and t[1].id + local _ = t[1] and t[1].title + local _ = t[1] and t[1].user and t[1].user.login +end + local scenarios = { {name = "small", iters = 5000, payload = read_file("benches/fixtures/small_api.json")}, {name = "medium", iters = 500, payload = read_file("benches/fixtures/medium_resp.json")}, + {name = "github-100k", iters = 100, payload = make_github_issues_payload(100 * 1024), + cjson_access = github_cjson_access, qd_access = github_qd_access, table_access = github_table_access}, {name = "100k", iters = 100, payload = make_payload(100 * 1024)}, {name = "200k", iters = 50, payload = make_payload(200 * 1024)}, {name = "500k", iters = 20, payload = make_payload(500 * 1024)}, @@ -114,45 +221,36 @@ local pooled_decoder = has_pooled_api and qd.new_decoder() or nil for _, s in ipairs(scenarios) do print(string.format("=== %s (%d bytes) ===", s.name, #s.payload)) + local cjson_access = s.cjson_access or default_cjson_access + local qd_access = s.qd_access or default_qd_access + local table_access = s.table_access or default_table_access + bench("cjson.decode + access 3 fields", s.iters, function() local obj = cjson.decode(s.payload) - local _ = obj.model - local _ = obj.temperature - local _ = obj.messages and obj.messages[1] and obj.messages[1].role + cjson_access(obj) end) bench("quickdecode.parse + access 3 fields", s.iters, function() local d = qd.parse(s.payload) - local _ = d:get_str("model") - local _ = d:get_f64("temperature") - local _ = d:get_str("messages[0].role") + qd_access(d) end) if has_pooled_api then bench("quickdecode pooled :parse + access 3 fields", s.iters, function() local d = pooled_decoder:parse(s.payload) - local _ = d:get_str("model") - local _ = d:get_f64("temperature") - local _ = d:get_str("messages[0].role") + qd_access(d) end) - -- One-shot-per-request pattern: each iter creates a fresh decoder, - -- parses once, and lets both decoder and doc fall to GC. No reuse. - -- This is the typical "user does not cache the decoder" path. bench("quickdecode new_decoder()+parse (one-shot)", s.iters, function() local dec = qd.new_decoder() local d = dec:parse(s.payload) - local _ = d:get_str("model") - local _ = d:get_f64("temperature") - local _ = d:get_str("messages[0].role") + qd_access(d) end) end bench("qd.decode + t.field x3", s.iters, function() local t = qd.decode(s.payload) - local _ = t.model - local _ = t.temperature - local _ = t.messages and t.messages[1] and t.messages[1].role + table_access(t) end) bench("qd.decode + qd.encode (unmodified)", s.iters, function() From 83cab1e105d56d2ca278c6c21ebd7924c3a73173 Mon Sep 17 00:00:00 2001 From: Yuansheng Wang Date: Sun, 17 May 2026 22:40:58 +0800 Subject: [PATCH 2/3] docs: add github-100k scenario to benchmarks Document the new GitHub Issues API simulation scenario that provides a realistic REST API benchmark with ~3-5% structural density. --- docs/benchmarks.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 35b0e2a..ea4eae9 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -48,6 +48,12 @@ until the target size is reached. The image size sequence comes from a Park–Miller LCG with `seed=42` rather than `math.random` so the payload is byte-identical across hosts. +A separate `github-100k` scenario simulates a GitHub Issues API response +(`/repos/{owner}/{repo}/issues`) with ~100 KB of realistic REST API +structure: nested user objects, labels arrays, URLs, timestamps, and +markdown body text. This provides a benchmark for typical REST API +parsing workloads with ~3-5% structural density. + ### Workload — what each row does | Row | What it does | Notes | @@ -94,6 +100,7 @@ Each row is "parse + access 3 fields" on the named payload. |---|---:|---:|---:|---:|---:|---:| | small | 2.1 KB | 39,414 | 54,395 | 117,233 | 126,807 | 268,240 | | medium | 60.4 KB | 5,600 | 40,180 | 90,074 | 120,627 | 126,263 | +| github-100k | 100 KB | 5,373 | — | 27,020 | 27,367 | 36,430 | | 100k | 100 KB | 2,589 | 19,944 | 72,202 | 61,162 | 80,257 | | 200k | 200 KB | 1,414 | 14,397 | 57,670 | 48,031 | 58,548 | | 500k | 500 KB | 722 | 5,882 | 34,602 | 33,167 | 36,900 | @@ -109,6 +116,7 @@ Each row is "parse + access 3 fields" on the named payload. |---|---:|---:|---:|---:| | small | 1.4× | 3.0× | 2.2× | 3.2× | | medium | 7.2× | 16.1× | 2.2× | 21.5× | +| github-100k | — | 5.0× | — | 5.1× | | 100k | 7.7× | 27.9× | 3.6× | 23.6× | | 200k | 10.2× | 40.8× | 4.0× | 34.0× | | 500k | 8.1× | 47.9× | 5.9× | 45.9× | @@ -127,6 +135,7 @@ collected before the snapshot. |---|---:|---:|---:|---:|---:| | small | +15,881 | +16,284 | +1,338 | +4,337 | +11,140 | | medium | +1,955 | +2,661 | +66 | +500 | +1,120 | +| github-100k | +12,867 | — | +19 | +592 | +273 | | 100k | +601 | +950 | +18 | +429 | +229 | | 200k | +505 | +722 | +7 | +206 | +112 | | 500k | +648 | +757 | +3 | +83 | +45 | @@ -188,6 +197,11 @@ later asks. Captures the upper bound of the lazy win. because the Lua table tree stays GC-rooted until the next collection. The 10 MB case retains ~11 MB for `cjson` / `simdjson`, ~3 KB for `qd.parse`. +6. **REST API payloads (github-100k) show a 5× speedup** — lower than the + multimodal payloads because the structural density is higher (~3-5% vs + <0.1%). However, memory savings remain dramatic: 677× less retention + (12.8 MB → 19 KB) because `cjson` must materialize every nested object + and string into the Lua heap. ## When to pick which From f61149ae6c367f099d09d3cd65fbfa65f1cd5636 Mon Sep 17 00:00:00 2001 From: Yuansheng Wang Date: Sun, 17 May 2026 22:45:32 +0800 Subject: [PATCH 3/3] bench: use realistic pseudo-random base64 for image payloads Replace string.rep("A", size) with a pre-generated 64 KB block of pseudo-random base64 characters (deterministic LCG, seed=12345). Larger payloads repeat the block to reach target size. This provides more realistic character distribution for CPU branch prediction and cache behavior, while keeping O(1) payload generation. --- benches/lua_bench.lua | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/benches/lua_bench.lua b/benches/lua_bench.lua index 659415f..7f2c1de 100644 --- a/benches/lua_bench.lua +++ b/benches/lua_bench.lua @@ -91,6 +91,31 @@ local function make_github_issues_payload(target_bytes) return "[" .. table.concat(issues, ",") .. "]" end +-- Pre-generate a 64 KB block of pseudo-random base64 characters. +-- Reused via repetition for larger image payloads to avoid O(n) generation. +local function make_b64_block() + local b64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + local rng = 12345 + local t = {} + for i = 1, 64 * 1024 do + rng = (rng * 48271) % 2147483647 + local idx = (rng % 64) + 1 + t[i] = b64_chars:sub(idx, idx) + end + return table.concat(t) +end + +local B64_BLOCK = make_b64_block() +local B64_BLOCK_LEN = #B64_BLOCK + +local function make_b64(size) + if size <= B64_BLOCK_LEN then + return B64_BLOCK:sub(1, size) + end + local reps = math.ceil(size / B64_BLOCK_LEN) + return string.rep(B64_BLOCK, reps):sub(1, size) +end + local function make_payload(target_bytes) local rng_state = 42 local function rng_range(lo, hi) @@ -118,7 +143,7 @@ local function make_payload(target_bytes) local upper = math.min(500 * 1024, remaining) img_size = rng_range(50 * 1024, upper) end - local b64 = string.rep("A", img_size) + local b64 = make_b64(img_size) local img_part = '{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,' .. b64 .. '"}}' parts[#parts + 1] = img_part