From ffc2131f3af037a1891865d6765343d84e3cfb23 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 06:52:45 +0000 Subject: [PATCH 01/46] Prevent ?? in redirectCanonical --- danode/router.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/danode/router.d b/danode/router.d index 385f90c..8486d6a 100644 --- a/danode/router.d +++ b/danode/router.d @@ -143,7 +143,7 @@ class Router { // Perform a canonical redirect of a non-existing page to the index script void redirectCanonical(WebConfig config, ref Request request, ref Response response){ log(Level.Trace, "Router: [T] Redirecting canonical url to the index page"); - request.url = format("%s?%s", config.index, request.query); + request.url = config.index ~ request.query; return deliver(request, response, true); } } From 55aa50b82ce9ed9ad7a567cba63a55b5b07f9445 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 07:06:10 +0000 Subject: [PATCH 02/46] Fix: Upload size limit inconsistency --- danode/client.d | 4 +--- danode/post.d | 8 ++++---- danode/webconfig.d | 3 +++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/danode/client.d b/danode/client.d index 2dc0b9d..5b2ef82 100644 --- a/danode/client.d +++ b/danode/client.d @@ -39,15 +39,13 @@ class Client { response.kill(); // kill any running CGI process } size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); - size_t uploadLimit = serverConfig.get("max_upload_size", 100 * 1024 * 1024); - size_t requestLimit = serverConfig.get("max_request_size", 2 * 1024 * 1024); while (running) { if (driver.receive(driver.socket) > 0) { // We've received new data if (!driver.hasHeader()) { if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } } else { if (driver.endOfHeader > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } - size_t limit = (driver.header.indexOf("multipart/") >= 0)? uploadLimit: requestLimit; + size_t limit = (driver.header.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize; if (driver.inbuffer.data.length > limit) { driver.sendPayloadTooLarge(response); stop(); continue; } } // Parse the data and try to create a response (Could fail multiple times) diff --git a/danode/post.d b/danode/post.d index b05c4ba..373e941 100644 --- a/danode/post.d +++ b/danode/post.d @@ -43,19 +43,19 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f return(response.setPayload(StatusCode.BadRequest, "400 - Bad Request\n", "text/plain")); } string content = request.body; + string contenttype = from(request.headers, "Content-Type"); + log(Level.Trace, "content type: %s", contenttype); + size_t limit = (contenttype.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize; if (expectedlength == 0) { log(Level.Trace, "Post: [T] Content-Length not specified (or 0), length: %s", content.length); return(response.havepost = true); // When we don't receive any post data it is meaningless to scan for any content - } else if (expectedlength > serverConfig.get("max_request_size", 2 * 1024 * 1024)) { + } else if (expectedlength > limit) { log(Level.Verbose, "Post: [W] Upload too large: %d bytes from %s", expectedlength, request.ip); return(response.setPayload(StatusCode.PayloadTooLarge, "413 - Payload Too Large\n", "text/plain")); } log(Level.Trace, "Post: [T] Received %s of %s", content.length, expectedlength); if(content.length < expectedlength) return(false); - string contenttype = from(request.headers, "Content-Type"); - log(Level.Trace, "content type: %s", contenttype); - if (contenttype.indexOf(XFORMHEADER) >= 0) { log(Level.Verbose, "XFORM: [I] parsing %d bytes", expectedlength); request.parseXform(content); diff --git a/danode/webconfig.d b/danode/webconfig.d index 23c2124..db5cca2 100644 --- a/danode/webconfig.d +++ b/danode/webconfig.d @@ -45,6 +45,9 @@ struct ServerConfig { config.mtime = timeLastModified(path); } + @property size_t maxUploadSize() { return(get("max_upload_size", 100 * 1024 * 1024)); } + @property size_t maxRequestSize() { return(get("max_request_size", 2 * 1024 * 1024)); } + T get(T)(string key, T def) { synchronized(serverConfigMutex) { try { return to!T(data.from(key, to!string(def))); }catch (Exception e) { return def; } } } From 4de66e3caa5211f62313cdfdc328a2a10f11a20a Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 07:29:38 +0000 Subject: [PATCH 03/46] Fix: Response header pollution on CGI fallback --- danode/response.d | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/danode/response.d b/danode/response.d index 3613c14..c8b8c82 100644 --- a/danode/response.d +++ b/danode/response.d @@ -37,29 +37,32 @@ struct Response { // Generate a HTTP header for the response @property final char[] header() { - if (hdr.data) { return(hdr.data); /* Header was constructed */ } + if (hdr.data) { return(hdr.data); } // Header was already constructed // Scripts are allowed to have/send their own header if (payload.type == PayloadType.Script) { CGI script = to!CGI(payload); + if (buildScriptHeader(hdr, connection, script, protocol)) return(hdr.data); + // Fallback: Populate headers from script output foreach (line; script.fullHeader().split("\n")) { auto v = line.split(": "); - if(v.length == 2) this.headers[v[0]] = chomp(v[1]); + if(v.length == 2) headers[v[0]] = chomp(v[1]); } - if (buildScriptHeader(hdr, connection, script, protocol)) return(hdr.data); } - // Server-generated header - hdr.put(format("%s %d %s\r\n", protocol, statuscode, statuscode.reason)); - foreach (key, value; headers) { hdr.put(format("%s: %s\r\n", key, value)); } - hdr.put(format("Date: %s\r\n", htmltime())); + // Server always owns these, overwrite anything the script has set + headers["Connection"] = connection; + headers["Date"] = htmltime(); + if (payload.mtime != SysTime.init) headers["Last-Modified"] = htmltime(payload.mtime); if (payload.type != PayloadType.Script && !noBody(statuscode)) { - long contentLength = isRange ? (rangeEnd - rangeStart + 1) : payload.length; - hdr.put(format("Content-Length: %d\r\n", contentLength)); - hdr.put(format("Content-Type: %s\r\n", payload.mimetype)); - if (maxage > 0) { hdr.put(format("Cache-Control: max-age=%d, public\r\n", maxage)); } + headers["Content-Length"] = to!string(isRange ? (rangeEnd - rangeStart + 1) : payload.length); + headers["Content-Type"] = payload.mimetype; + if (maxage > 0) headers["Cache-Control"] = format("max-age=%d, public", maxage); } - if (payload.mtime != SysTime.init) { hdr.put(format("Last-Modified: %s\r\n", htmltime(payload.mtime))); } - hdr.put(format("Connection: %s\r\n\r\n", connection)); + + // Header emit loop + hdr.put(format("%s %d %s\r\n", protocol, statuscode, statuscode.reason)); + foreach (key, value; headers) { hdr.put(format("%s: %s\r\n", key, value)); } + hdr.put("\r\n"); return(hdr.data); } From 56f9f78d8b85527888d05a78091f26dabf4de8f1 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 07:51:10 +0000 Subject: [PATCH 04/46] Fix: No more logConnection called twice for keepalive connections (I hope) --- danode/client.d | 1 + danode/request.d | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/danode/client.d b/danode/client.d index 5b2ef82..c53f64f 100644 --- a/danode/client.d +++ b/danode/client.d @@ -82,6 +82,7 @@ class Client { } void logConnection(in Request rq, in Response rs) { + if(request.starttime == SysTime.init) return; string uri; try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); diff --git a/danode/request.d b/danode/request.d index 37a49c5..8dcd8ef 100644 --- a/danode/request.d +++ b/danode/request.d @@ -51,8 +51,8 @@ struct Request { HTTPVersion protocol; /// protocol requested string[string] headers; /// Associative array holding the header values SysTime starttime; /// start time of the Request - PostItem[string] postinfo; /// Associative array holding the post parameters and values - long maxtime; /// Maximum time in ms before the request is discarded + PostItem[string] postinfo; /// Associative array holding the post parameters and values + long maxtime; /// Maximum time in ms before the request is discarded // Start a new Request, and parseHeader on the DriverInterface final void initialize(const DriverInterface driver) { From 949b1f098740ad8a0b26f23bbd783511923bcb25 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 07:52:39 +0000 Subject: [PATCH 05/46] Make sure to synchronize this.completed = true; --- danode/process.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/danode/process.d b/danode/process.d index 594f577..b3407c5 100644 --- a/danode/process.d +++ b/danode/process.d @@ -182,7 +182,7 @@ class Process : Thread { log(Level.Trace, "removing process input file %s ? %s", inputfile, removeInput); if(removeInput) remove(inputfile); } catch(Exception e) { error("process.d, exception: '%s'", e.msg); } - this.completed = true; + synchronized { this.completed = true; } } } From 57b7685904e4e2de3eeb0265c29952e3848b4bb2 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 07:54:51 +0000 Subject: [PATCH 06/46] Make sure to synchronize updates to some more fields --- danode/process.d | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/danode/process.d b/danode/process.d index b3407c5..2766256 100644 --- a/danode/process.d +++ b/danode/process.d @@ -121,7 +121,7 @@ class Process : Thread { while (lastmodified < maxtime && buffer.data.length < maxOutput) { n = fread(tmp.ptr, 1, tmp.sizeof, fp); if (n > 0) { - modified = Clock.currTime(); + synchronized { modified = Clock.currTime(); } buffer.put(tmp[0 .. n]); } else { break; } } @@ -144,8 +144,7 @@ class Process : Thread { try { if( !exists(inputfile) ) { log(Level.Verbose, "no input path: %s", inputfile); - this.process.terminated = true; - this.completed = true; + synchronized { this.process.terminated = true; this.completed = true; } return; } fStdIn = File(inputfile, "r"); @@ -162,7 +161,7 @@ class Process : Thread { while (running && lastmodified < maxtime) { drainPipes(); - process = cast(WaitResult) tryWait(cpid); + synchronized { process = cast(WaitResult) tryWait(cpid); } Thread.sleep(msecs(1)); } if (!process.terminated) { From 37e30f67f88effc7c398daa5e6ee3b157bff5b6f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:09:22 +0000 Subject: [PATCH 07/46] Allow multi-colon header values --- danode/cgi.d | 4 ++-- danode/response.d | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/danode/cgi.d b/danode/cgi.d index 6e20194..1e2ace5 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -60,8 +60,8 @@ class CGI : Payload { final T getHeader(T)(string key, T def = T.init) const { if (endOfHeader > 0) { foreach (line; fullHeader().split("\n")) { - string[] elems = line.split(": "); - if (elems.length == 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(elems[1]))); } + string[] elems = line.split(":"); + if (elems.length >= 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(join(elems[1 .. $], ":")))); } } } return(def); diff --git a/danode/response.d b/danode/response.d index c8b8c82..8705eb1 100644 --- a/danode/response.d +++ b/danode/response.d @@ -45,8 +45,8 @@ struct Response { if (buildScriptHeader(hdr, connection, script, protocol)) return(hdr.data); // Fallback: Populate headers from script output foreach (line; script.fullHeader().split("\n")) { - auto v = line.split(": "); - if(v.length == 2) headers[v[0]] = chomp(v[1]); + auto v = line.split(":"); + if(v.length >= 2) this.headers[v[0]] = strip(join(v[1 .. $], ":")); } } // Server always owns these, overwrite anything the script has set @@ -113,9 +113,8 @@ bool buildScriptHeader(ref Appender!(char[]) hdr, ref string connection, CGI scr hdr.put(format("%s %d %s\r\n", protocol, script.statuscode, script.statuscode.reason)); } foreach (line; scriptheader.split("\n")) { - auto stripped = strip(line); - auto parts = stripped.split(":"); - if (stripped.length > 0 && parts.length > 0 && icmp(parts[0], "connection") != 0) { hdr.put(line ~ "\n"); } + auto parts = strip(line).split(":"); + if (parts.length > 0 && parts[0].length > 0 && icmp(parts[0], "connection") != 0) { hdr.put(line ~ "\n"); } } hdr.put(format("Connection: %s\r\n\r\n", connection)); return true; From a5aa139f816b0a49e305c404de37e17ab5132440 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:18:22 +0000 Subject: [PATCH 08/46] Parse IPv6 host & port --- danode/client.d | 2 +- danode/request.d | 34 +++++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/danode/client.d b/danode/client.d index c53f64f..5908fa2 100644 --- a/danode/client.d +++ b/danode/client.d @@ -82,7 +82,7 @@ class Client { } void logConnection(in Request rq, in Response rs) { - if(request.starttime == SysTime.init) return; + if(rq.starttime == SysTime.init) return; string uri; try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); diff --git a/danode/request.d b/danode/request.d index 8dcd8ef..1963457 100644 --- a/danode/request.d +++ b/danode/request.d @@ -103,17 +103,27 @@ struct Request { final @property @nogc bool hasRange() const nothrow { return headers.from("Range").startsWith("bytes="); } // The Host header requested in the request - final @property @nogc string host() const nothrow { - ptrdiff_t i = headers.from("Host").indexOf(":"); - if (i > 0) { return(headers.from("Host")[0 .. i]); } - return(headers.from("Host")); + final @property @nogc string host() const nothrow { + ptrdiff_t i; + string h = headers.from("Host"); + if (h.startsWith("[")) { // IPv6: [::1]:8080 + i = h.indexOf("]"); return((i > 0)? h[0 .. i+1] : h); + } + i = h.indexOf(":"); return((i > 0)? h[0 .. i] : h); } // The Port from the Host header in the request final @property ushort serverport() const { - ptrdiff_t i = headers.from("Host").indexOf(":"); - if (i > 0) { return( to!ushort(headers.from("Host")[(i+1) .. $])); } - return(isSecure ? to!ushort(443) : to!ushort(80)); // return the default ports + ptrdiff_t i; + string h = headers.from("Host"); + if (h.startsWith("[")) { // IPv6: [::1]:8080 + i = h.indexOf("]:"); + if (i > 0) { return(to!ushort(h[i+2 .. $])); } + return(isSecure ? to!ushort(443) : to!ushort(80)); + } + i = h.indexOf(":"); + if (i > 0) { return(to!ushort(h[i+1 .. $])); } + return(isSecure ? to!ushort(443) : to!ushort(80)); } // Input file generated storing the headers of the request @@ -269,4 +279,14 @@ unittest { Request r11; r11.headers["Range"] = "bytes=abc-def"; assert(r11.range() == [-1, -1], "malformed range must return [-1, -1]"); + + Request r_ipv6; + r_ipv6.headers["Host"] = "[::1]:8080"; + assert(r_ipv6.host == "[::1]", "IPv6 host must include brackets"); + assert(r_ipv6.serverport() == 8080, "IPv6 port must be 8080"); + + Request r_ipv6b; + r_ipv6b.headers["Host"] = "[::1]"; + assert(r_ipv6b.host == "[::1]", "IPv6 without port must return host"); + assert(r_ipv6b.serverport() == 80, "IPv6 without port must return default"); } From 72794cc612efd7b09f12f6b2a803b2e2b891408f Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:25:38 +0000 Subject: [PATCH 09/46] ready() now returns bool --- danode/cgi.d | 2 +- danode/files.d | 4 ++-- danode/payload.d | 4 ++-- danode/response.d | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/danode/cgi.d b/danode/cgi.d index 1e2ace5..74bfb6a 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -28,7 +28,7 @@ class CGI : Payload { final @property PayloadType type() const { return(PayloadType.Script); } // Ready to start sending ? - final @property long ready() { + final @property bool ready() { if (external.finished) return true; if (mimetype == "text/event-stream") return (endOfHeader > 0); return false; diff --git a/danode/files.d b/danode/files.d index b166bea..b9ce9a9 100644 --- a/danode/files.d +++ b/danode/files.d @@ -30,7 +30,7 @@ class FileStream : Payload { } final @property PayloadType type() const { return PayloadType.File; } - final @property long ready() { return payload.ready(); } + final @property bool ready() { return payload.ready(); } final @property ptrdiff_t length() const { return payload.length(); } final @property SysTime mtime() { return payload.mtime(); } final @property string mimetype() const { return payload.mimetype(); } @@ -126,7 +126,7 @@ class FilePayload : Payload { final @property bool hasEncodedVersion() const { return(encbuf !is null); } final @property bool isStaticFile() { return(!path.isCGI()); } final @property SysTime mtime() const { try { return path.timeLastModified(); }catch (Exception e) { return btime; } } - final @property long ready() { return(true); } + final @property bool ready() { return(true); } final @property PayloadType type() const { return(PayloadType.File); } final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); } final @property long buffersize() const { return cast(long)(buf.length); } diff --git a/danode/payload.d b/danode/payload.d index 7a63e8e..337be61 100644 --- a/danode/payload.d +++ b/danode/payload.d @@ -14,7 +14,7 @@ enum HeaderType { None, FastCGI, HTTP10, HTTP11 } /* Payload interface, Payload is carried by the Response structure, not the Request structure */ interface Payload { public: - @property long ready(); + @property bool ready(); @property StatusCode statuscode() const; @property PayloadType type() const; @property ptrdiff_t length() const; @@ -39,7 +39,7 @@ class Message : Payload { } final @property PayloadType type() const { return(PayloadType.Message); } - final @property long ready() { return(true); } + final @property bool ready() { return(true); } final @property ptrdiff_t length() const { return(message.length); } final @property SysTime mtime() { return SysTime.init; } final @property string mimetype() const { return mime; } diff --git a/danode/response.d b/danode/response.d index 8705eb1..58dbd71 100644 --- a/danode/response.d +++ b/danode/response.d @@ -82,7 +82,7 @@ struct Response { } @property final bool isSSE() const { return(payload !is null && payload.mimetype == "text/event-stream"); } - @property final bool scriptCompleted() { return(canComplete && payload.type == PayloadType.Script && payload.ready > 0 && index >= length); } + @property final bool scriptCompleted() { return(canComplete && payload.type == PayloadType.Script && payload.ready && index >= length); } @property final bool canComplete() const { return(payload !is null && payload.length >= 0); } // Stream of bytes (header + stream of bytes) From 1d17cce221799a7b161c0057214011b363dc5d92 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:28:44 +0000 Subject: [PATCH 10/46] Style: use !is null --- danode/client.d | 4 ++-- danode/interfaces.d | 2 +- danode/post.d | 6 +++--- danode/response.d | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/danode/client.d b/danode/client.d index 5908fa2..5e2ca7d 100644 --- a/danode/client.d +++ b/danode/client.d @@ -85,8 +85,8 @@ class Client { if(rq.starttime == SysTime.init) return; string uri; try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } - long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); - int code = cast(int)(rs.payload ? rs.statuscode.code : 0); + long bytes = (rs.payload !is null && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); + int code = cast(int)((rs.payload !is null)? rs.statuscode.code : 0); long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime); tag(Level.Always, format("%d", code), "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), ip, port, rq.shorthost, uri.replace("%", "%%"), requests, bytes/1024f, ms); diff --git a/danode/interfaces.d b/danode/interfaces.d index 76cdc08..941bf21 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -77,7 +77,7 @@ abstract class DriverInterface { // serve a 408 connection timed out page void sendTimedOut(ref DriverInterface driver, ref Response response) { - if(response.payload && response.payload.type == PayloadType.Script){ to!CGI(response.payload).notifyovertime(); } + if(response.payload !is null && response.payload.type == PayloadType.Script){ to!CGI(response.payload).notifyovertime(); } response.setPayload(StatusCode.TimedOut, "408 - Connection Timed Out\n", "text/plain"); driver.send(response, driver.socket); } diff --git a/danode/post.d b/danode/post.d index 373e941..1a1ff1c 100644 --- a/danode/post.d +++ b/danode/post.d @@ -148,13 +148,13 @@ final void serverAPI(in FileSystem filesystem, in WebConfig config, in Request r foreach (c; request.cookies.split("; ")) { content.put(format("C=%s\n", chomp(c)) ); } content.put(format("S=SERVER_SOFTWARE=%s\n", serverConfig.get("serverinfo", "DaNode/0.0.3"))); try{ - content.put(format("S=SERVER_NAME=%s\n", (response.address)? response.address.toHostNameString() : "localhost")); + content.put(format("S=SERVER_NAME=%s\n", (response.address !is null)? response.address.toHostNameString() : "localhost")); }catch(Exception e){ error("Exception while trying to call: toHostNameString()"); content.put("S=SERVER_NAME=localhost\n"); } - content.put(format("S=SERVER_ADDR=%s\n", (response.address)? response.address.toAddrString() : "127.0.0.1")); - content.put(format("S=SERVER_PORT=%s\n", (response.address)? response.address.toPortString() : "80")); + content.put(format("S=SERVER_ADDR=%s\n", (response.address !is null)? response.address.toAddrString() : "127.0.0.1")); + content.put(format("S=SERVER_PORT=%s\n", (response.address !is null)? response.address.toPortString() : "80")); content.put(format("S=DOCUMENT_ROOT=%s\n", filesystem.localroot(request.shorthost()))); content.put(format("S=GATEWAY_INTERFACE=%s\n", "CGI/1.1")); content.put(format("S=PHP_SELF=%s\n", request.path)); diff --git a/danode/response.d b/danode/response.d index 58dbd71..01b1d62 100644 --- a/danode/response.d +++ b/danode/response.d @@ -68,7 +68,7 @@ struct Response { // Propagate shutdown through the chain to kill Process final void kill() { - if (payload && payload.type == PayloadType.Script) { to!CGI(payload).notifyovertime(); } + if (payload !is null && payload.type == PayloadType.Script) { to!CGI(payload).notifyovertime(); } } @property final StatusCode statuscode() const { From b2e4e7bb3778346d8b2e5ac11f4633f368d01492 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:31:12 +0000 Subject: [PATCH 11/46] Remove dead code --- danode/request.d | 3 --- 1 file changed, 3 deletions(-) diff --git a/danode/request.d b/danode/request.d index 1963457..6e49408 100644 --- a/danode/request.d +++ b/danode/request.d @@ -136,9 +136,6 @@ struct Request { return format("%s/%s.up", filesystem.localroot(shorthost()), md5UUID(format("%s-%s", this.id, name))); } - // Get parameters as associative array - final string[string] get() const { return parseQueryString(query[1 .. $]); } - // List of filenames uploaded by the user final @property string[] postfiles() const { string[] files; From 69e2ca638b3e52e040825fb1f1f9e2a3b1288393 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:31:46 +0000 Subject: [PATCH 12/46] Minor --- danode/request.d | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/danode/request.d b/danode/request.d index 6e49408..e5104dd 100644 --- a/danode/request.d +++ b/danode/request.d @@ -139,9 +139,7 @@ struct Request { // List of filenames uploaded by the user final @property string[] postfiles() const { string[] files; - foreach (p; postinfo) { - if(p.type == PostType.File && p.size > 0) files ~= p.value; - } + foreach (p; postinfo) { if(p.type == PostType.File && p.size > 0) { files ~= p.value; } } return(files); } From a22922decd82afecd0c3ce92cb7514716b7210d2 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:34:41 +0000 Subject: [PATCH 13/46] Minor --- danode/post.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/danode/post.d b/danode/post.d index 1a1ff1c..065aaa4 100644 --- a/danode/post.d +++ b/danode/post.d @@ -69,7 +69,7 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f log(Level.Verbose, "MPART: [I] # of items: %s", request.postinfo.length); } else if (contenttype.indexOf(JSON) >= 0) { log(Level.Verbose, "JSON: [I] Parsing %d bytes", expectedlength); - //request.postinfo["php://input"] = PostItem(PostType.File, "stdin", "php://input", content, JSON, content.length); + // JSON body is passed raw to the script via stdin - no server-side parsing needed } else { error("parsePost: Unsupported POST content type: %s [%s] -> %s", contenttype, expectedlength, content); request.parseXform(content); From 6c6021f3647081cc88900610cfc749ad6b7eed92 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:36:41 +0000 Subject: [PATCH 14/46] Layout --- danode/post.d | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/danode/post.d b/danode/post.d index 065aaa4..a13a38e 100644 --- a/danode/post.d +++ b/danode/post.d @@ -182,14 +182,14 @@ unittest { FileSystem fs = new FileSystem("./www/"); // extractQuoted - assert(extractQuoted("name=\"hello\"", "name") == "hello", "extractQuoted must get name"); + assert(extractQuoted("name=\"hello\"", "name") == "hello", "extractQuoted must get name"); assert(extractQuoted("filename=\"test.txt\"", "filename") == "test.txt", "extractQuoted must get filename"); - assert(extractQuoted("name=\"\"", "name") == "", "extractQuoted empty value"); - assert(extractQuoted("nothing here", "name") == "", "extractQuoted missing key"); + assert(extractQuoted("name=\"\"", "name") == "", "extractQuoted empty value"); + assert(extractQuoted("nothing here", "name") == "", "extractQuoted missing key"); // findBodyLine assert(findBodyLine(["Content-Disposition: form-data", "", "value"]) == 2, "findBodyLine must find blank line"); - assert(findBodyLine(["Content-Disposition: form-data"]) == -1, "findBodyLine no blank must return -1"); + assert(findBodyLine(["Content-Disposition: form-data"]) == -1, "findBodyLine no blank must return -1"); // parseXform via runRequest auto router = new Router("./www/", Address.init); From 21624ccf96f7803e1f7a29720a402838c13a6cae Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 08:44:17 +0000 Subject: [PATCH 15/46] Exit handler for windows and linux --- danode/server.d | 7 ++----- danode/signals.d | 24 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/danode/server.d b/danode/server.d index 924d1f8..88fa771 100644 --- a/danode/server.d +++ b/danode/server.d @@ -9,7 +9,7 @@ import danode.functions : Msecs, sISelect, resolveFolder; import danode.interfaces : DriverInterface; import danode.http : HTTP; import danode.router : Router; -import danode.signals : shutdownSignal; +import danode.signals : shutdownSignal, registerExitHandler; import danode.workerpool : WorkerPool; import danode.webconfig : serverConfig, ServerConfig, serverConfigMutex; @@ -139,10 +139,7 @@ void main(string[] args) { "verbose|v", &verbose); // Verbose level (via commandline) atomicStore(cv, verbose); synchronized(serverConfigMutex) { serverConfig = ServerConfig(wwwFolder ~ "server.config"); } - version(Posix) { - import danode.signals : setupPosix; - setupPosix(); - } + registerExitHandler(); auto server = new Server(port, backlog, wwwFolder, sslFolder, sslKey, accountKey); version (SSL) { diff --git a/danode/signals.d b/danode/signals.d index 57f216f..71948b2 100644 --- a/danode/signals.d +++ b/danode/signals.d @@ -2,11 +2,11 @@ * License: GPLv3 (https://github.com/DannyArends/DaNode) - Danny Arends **/ module danode.signals; +import danode.imports; + shared bool shutdownSignal = false; version(Posix) { - import danode.imports; - import core.sys.posix.sys.resource; import core.sys.posix.signal : signal, SIGPIPE, SIGTERM, SIGINT; import core.sys.posix.unistd : write; @@ -27,8 +27,23 @@ version(Posix) { break; } } +} + +version(Windows) { + import core.sys.windows.wincon : SetConsoleCtrlHandler, CTRL_C_EVENT, CTRL_BREAK_EVENT, CTRL_CLOSE_EVENT; + import core.sys.windows.windef : BOOL, DWORD, TRUE, FALSE; + + extern(Windows) BOOL handleConsoleCtrl(DWORD ctrlType) nothrow { + switch (ctrlType) { + case CTRL_C_EVENT, CTRL_BREAK_EVENT, CTRL_CLOSE_EVENT: atomicStore(shutdownSignal, true); return TRUE; + default: return FALSE; + } + } +} - void setupPosix() { +void registerExitHandler(){ + version(Windows){ SetConsoleCtrlHandler(&handleConsoleCtrl, TRUE); } + version(Posix) { rlimit rl; getrlimit(RLIMIT_NOFILE, &rl); rl.rlim_cur = rl.rlim_max; @@ -38,5 +53,4 @@ version(Posix) { signal(SIGTERM, &handleSignal); signal(SIGINT, &handleSignal); } -} - +} \ No newline at end of file From 1ca34bdecb4fdbc7969bcd7a16ee3b7fef4e02c2 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 10:28:45 +0000 Subject: [PATCH 16/46] Shutdown on ctrl+c in unittests with removal of *.in files --- danode/cgi.d | 2 ++ danode/client.d | 19 ++++++++++--------- danode/functions.d | 3 +++ danode/process.d | 12 +++++++----- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/danode/cgi.d b/danode/cgi.d index 74bfb6a..c3d8dce 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -140,7 +140,9 @@ class CGI : Payload { unittest { import danode.router : Router, runRequest; import danode.interfaces : StringDriver; + import danode.signals : registerExitHandler; + registerExitHandler(); tag(Level.Always, "FILE", "%s", __FILE__); auto router = new Router("./www/", Address.init); diff --git a/danode/client.d b/danode/client.d index 5e2ca7d..7adaf8c 100644 --- a/danode/client.d +++ b/danode/client.d @@ -12,6 +12,7 @@ import danode.response : Response; import danode.request : Request; import danode.log : log, tag, Level; import danode.webconfig : serverConfig; +import danode.signals : shutdownSignal; class Client { private: @@ -28,17 +29,17 @@ class Client { } final void run() { + Request request; + Response response; log(Level.Trace, "New connection established %s:%d", ip(), port() ); + scope (exit) { + if (driver.socketReady()) driver.closeConnection(); // Close connection + request.clearUploadFiles(); // Clean uploaded files + response.kill(); // Kill any running CGI process + } try { if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; } - Request request; - Response response; - scope (exit) { - if (driver.socketReady()) driver.closeConnection(); // Close connection - request.clearUploadFiles(); // Clean uploaded files - response.kill(); // kill any running CGI process - } - size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); + size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); while (running) { if (driver.receive(driver.socket) > 0) { // We've received new data if (!driver.hasHeader()) { @@ -93,7 +94,7 @@ class Client { } // Is the client still running, if the socket was gone it's not otherwise check the terminated flag - final @property bool running() const { return(!atomicLoad(terminated) && driver.socketReady()); } + final @property bool running() const { return(!atomicLoad(terminated) && !atomicLoad(shutdownSignal) && driver.socketReady()); } // Stop the client by setting the terminated flag final void stop() { diff --git a/danode/functions.d b/danode/functions.d index 07076b6..6fef4aa 100644 --- a/danode/functions.d +++ b/danode/functions.d @@ -43,6 +43,9 @@ string resolveFolder(string path) { return(path); } +void safeClose(ref File f) nothrow { try { if (f.isOpen()) { f.close(); } } catch(Exception e) {} } +void safeRemove(string path) nothrow { try { if (exists(path)) { remove(path); } } catch(Exception e) {} } + // Returns null if path escapes root string safePath(in string root, in string path) { if (path.canFind("..")) return null; diff --git a/danode/process.d b/danode/process.d index 2766256..f74c1fd 100644 --- a/danode/process.d +++ b/danode/process.d @@ -4,7 +4,7 @@ module danode.process; import danode.imports; -import danode.functions : Msecs; +import danode.functions : Msecs, safeClose, safeRemove; import danode.log : log, tag, error, Level; import danode.webconfig : serverConfig; @@ -141,6 +141,12 @@ class Process : Thread { // execute the process and wait until maxtime has finished or the process returns // inputfile is removed when the run() returns succesfully, on error, it is kept final void run() { + scope(exit) { + log(Level.Always, "Removing process input file %s ? %s", inputfile, removeInput); + safeClose(fStdIn); + if (removeInput) safeRemove(inputfile); + synchronized { this.completed = true; } + } try { if( !exists(inputfile) ) { log(Level.Verbose, "no input path: %s", inputfile); @@ -177,11 +183,7 @@ class Process : Thread { // Close the file handles fStdIn.close(); fStdOut.close(); fStdErr.close(); - - log(Level.Trace, "removing process input file %s ? %s", inputfile, removeInput); - if(removeInput) remove(inputfile); } catch(Exception e) { error("process.d, exception: '%s'", e.msg); } - synchronized { this.completed = true; } } } From f61b3dc1352bda412e102d97707f3a8fa013d771 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 10:46:52 +0000 Subject: [PATCH 17/46] Quit running tests after ctrl+c --- danode/cgi.d | 5 ++--- danode/process.d | 2 +- danode/response.d | 6 +++++- danode/router.d | 2 ++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/danode/cgi.d b/danode/cgi.d index c3d8dce..912b4d3 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -67,9 +67,8 @@ class CGI : Payload { return(def); } - private const(char)[] rawOutput() const { - return cast(const(char)[]) external.output(0); - } + private const(char)[] rawOutput() const { return cast(const(char)[]) external.output(0); } + void joinThread() { external.join(); } // Type of header returned by the script: FastCGI, HTTP10, HTTP11 @property final HeaderType headerType() const { diff --git a/danode/process.d b/danode/process.d index f74c1fd..3043e9a 100644 --- a/danode/process.d +++ b/danode/process.d @@ -142,7 +142,7 @@ class Process : Thread { // inputfile is removed when the run() returns succesfully, on error, it is kept final void run() { scope(exit) { - log(Level.Always, "Removing process input file %s ? %s", inputfile, removeInput); + log(Level.Verbose, "Removing process input file %s ? %s", inputfile, removeInput); safeClose(fStdIn); if (removeInput) safeRemove(inputfile); synchronized { this.completed = true; } diff --git a/danode/response.d b/danode/response.d index 01b1d62..9c95930 100644 --- a/danode/response.d +++ b/danode/response.d @@ -68,7 +68,11 @@ struct Response { // Propagate shutdown through the chain to kill Process final void kill() { - if (payload !is null && payload.type == PayloadType.Script) { to!CGI(payload).notifyovertime(); } + if (payload !is null && payload.type == PayloadType.Script) { + auto cgi = to!CGI(payload); + cgi.notifyovertime(); + cgi.joinThread(); + } } @property final StatusCode statuscode() const { diff --git a/danode/router.d b/danode/router.d index 8486d6a..8093790 100644 --- a/danode/router.d +++ b/danode/router.d @@ -15,6 +15,7 @@ import danode.functions : isCGI, isFILE, isDIR, isAllowed, safePath; import danode.filesystem : FileSystem; import danode.post : parsePost; import danode.log : log, tag, Level; +import danode.signals : shutdownSignal; version(SSL) { import danode.ssl : hasCertificate; @@ -150,6 +151,7 @@ class Router { // Helper function used to make calls during a unittest, setup a driver, a client and run the request StringDriver runRequest(Router router, string request = "GET /dmd.d HTTP/1.1\nHost: localhost\n\n", long maxtime = 1000) { + if(atomicLoad(shutdownSignal)) { exit(1); } tag(Level.Verbose, "runRequest", "%s", request); auto driver = new StringDriver(request); auto client = new Client(router, driver, maxtime); From f35cb85ff7d107def0fa5af8cc8def6f7db1e693 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 10:59:23 +0000 Subject: [PATCH 18/46] Eliminating some dead code, and some other minor fixes --- danode/cgi.d | 1 - danode/files.d | 1 + danode/filesystem.d | 6 ++---- danode/http.d | 4 ++-- danode/https.d | 4 ++-- danode/interfaces.d | 8 +++----- danode/request.d | 2 -- 7 files changed, 10 insertions(+), 16 deletions(-) diff --git a/danode/cgi.d b/danode/cgi.d index 912b4d3..3641b14 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -77,7 +77,6 @@ class CGI : Payload { if (values.length >= 3 && values[0] == "HTTP/1.0") return HeaderType.HTTP10; if (values.length >= 3 && values[0] == "HTTP/1.1") return HeaderType.HTTP11; if (getHeader("Status", "") != "") return HeaderType.FastCGI; - //if (getHeader("Content-Type", "") != "") return HeaderType.FastCGI; return HeaderType.None; } diff --git a/danode/files.d b/danode/files.d index b9ce9a9..7107e76 100644 --- a/danode/files.d +++ b/danode/files.d @@ -68,6 +68,7 @@ class FilePayload : Payload { this(string path, size_t buffermaxsize) { this.path = path; this.buffermaxsize = buffermaxsize; + this.buffer(); } /* Does the file require to be updated before sending ? */ diff --git a/danode/filesystem.d b/danode/filesystem.d index 004c82c..efe4269 100644 --- a/danode/filesystem.d +++ b/danode/filesystem.d @@ -16,9 +16,9 @@ import danode.log : log, tag, error, Level; Note 2: ./www/localhost existing is required for unit testing */ struct Domain { FilePayload[string] files; - long entries; - long buffered; + @property long entries() const { return files.length; } + @property long buffered() const { long n = 0; foreach(ref f; files.byValue) { if(f.isBuffered) n++; } return n; } @property long buffersize() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].buffersize(); } return sum; } @property long size() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].length(); } return sum; } } @@ -58,8 +58,6 @@ class FileSystem { log(Level.Trace, "File: '%s' as '%s'", f.name, shortname); if (!domain.files.has(shortname)) { domain.files[shortname] = new FilePayload(f.name, maxsize); - domain.entries++; - if(domain.files[shortname].buffer()) { domain.buffered++; } } } } diff --git a/danode/http.d b/danode/http.d index 77bceb9..8069db3 100644 --- a/danode/http.d +++ b/danode/http.d @@ -11,10 +11,10 @@ import danode.log : log, tag, error, Level; class HTTP : DriverInterface { public: - this(Socket socket, bool blocking = false) { super(socket, blocking); } + this(Socket socket) { super(socket); } // Open the connection by setting the socket to non blocking I/O, and registering the origin address - override bool openConnection() { + override bool openConnection(bool blocking = false) { try { socket.blocking = blocking; } catch(Exception e) { error("Unable to accept socket: %s", e.msg); return(false); } diff --git a/danode/https.d b/danode/https.d index e102fea..d224cf8 100644 --- a/danode/https.d +++ b/danode/https.d @@ -19,7 +19,7 @@ version(SSL) { SSL* ssl = null; public: - this(Socket socket, bool blocking = false) { super(socket, blocking); } + this(Socket socket) { super(socket); } // Perform the SSL handshake bool performHandshake() { @@ -40,7 +40,7 @@ version(SSL) { } // Open the connection by setting the socket to non blocking I/O, and registering the origin address - override bool openConnection() { + override bool openConnection(bool blocking = false) { log(Level.Verbose, "Opening HTTPS connection"); if (contexts.length > 0) { log(Level.Trace, "Number of SSL contexts: %d", contexts.length); diff --git a/danode/interfaces.d b/danode/interfaces.d index 941bf21..d12395a 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -22,12 +22,10 @@ abstract class DriverInterface { SysTime systime; /// Time in ms since this process came alive SysTime modtime; /// Time in ms since this process was last modified Address address; /// Private address field - bool blocking = false; /// Blocking communication ? - this(Socket socket, bool blocking = false) { + this(Socket socket) { this.socket = socket; this.socketSet = new SocketSet(); - this.blocking = blocking; systime = Clock.currTime(); touch(); } @@ -51,7 +49,7 @@ abstract class DriverInterface { } long receiveData(ref char[] buffer); - bool openConnection(); + bool openConnection(bool blocking = false); void closeConnection(); @nogc bool isSecure() const nothrow; @@ -104,7 +102,7 @@ class StringDriver : DriverInterface { public: this(string input) { super(null); inbuffer ~= input; } - override bool openConnection() { return(true); } + override bool openConnection(bool blocking = false) { return(true); } override void closeConnection() nothrow { } override bool socketReady() const { return inbuffer.data.length > 0; } @nogc override bool isSecure() const nothrow { return(false); } diff --git a/danode/request.d b/danode/request.d index e5104dd..42e93e4 100644 --- a/danode/request.d +++ b/danode/request.d @@ -52,7 +52,6 @@ struct Request { string[string] headers; /// Associative array holding the header values SysTime starttime; /// start time of the Request PostItem[string] postinfo; /// Associative array holding the post parameters and values - long maxtime; /// Maximum time in ms before the request is discarded // Start a new Request, and parseHeader on the DriverInterface final void initialize(const DriverInterface driver) { @@ -61,7 +60,6 @@ struct Request { this.body = driver.body; this.isSecure = driver.isSecure; this.starttime = Clock.currTime(); - this.maxtime = serverConfig.get("request_timeout", 5000L); this.id = md5UUID(format("%s:%d-%s", driver.ip, driver.port, starttime)); this.isValid = this.parseHeader(driver.header); log(Level.Verbose, "request: %s to %s from %s:%d - %s", method, uri, this.ip, this.port, this.id); From b88005636f00d1ee327f22cf1ba2c418d3d726e2 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 11:04:09 +0000 Subject: [PATCH 19/46] Minor smell, move havepost into request and call it postParsed --- danode/post.d | 8 ++++---- danode/request.d | 1 + danode/response.d | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/danode/post.d b/danode/post.d index a13a38e..d980cb7 100644 --- a/danode/post.d +++ b/danode/post.d @@ -34,7 +34,7 @@ struct PostItem { // when the entire request body is not yet available. POST data supplied in Multipart // and X-form post formats are currently supported final bool parsePost(ref Request request, ref Response response, in FileSystem filesystem) { - if (response.havepost || request.method != RequestMethod.POST) { return(response.havepost = true); } + if (request.postParsed || request.method != RequestMethod.POST) { return(request.postParsed = true); } long expectedlength; try { @@ -48,7 +48,7 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f size_t limit = (contenttype.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize; if (expectedlength == 0) { log(Level.Trace, "Post: [T] Content-Length not specified (or 0), length: %s", content.length); - return(response.havepost = true); // When we don't receive any post data it is meaningless to scan for any content + return(request.postParsed = true); // When we don't receive any post data it is meaningless to scan for any content } else if (expectedlength > limit) { log(Level.Verbose, "Post: [W] Upload too large: %d bytes from %s", expectedlength, request.ip); return(response.setPayload(StatusCode.PayloadTooLarge, "413 - Payload Too Large\n", "text/plain")); @@ -62,7 +62,7 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f log(Level.Verbose, "XFORM: [T] # of items: %s", request.postinfo.length); } else if (contenttype.indexOf(MPHEADER) >= 0) { auto parts = split(contenttype, "boundary="); - if (parts.length < 2) return(response.havepost = true); + if (parts.length < 2) return(request.postParsed = true); string mpid = parts[1]; log(Level.Verbose, "MPART: [I] header: %s, parsing %d bytes", mpid, expectedlength); request.parseMultipart(filesystem, content, mpid); @@ -74,7 +74,7 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f error("parsePost: Unsupported POST content type: %s [%s] -> %s", contenttype, expectedlength, content); request.parseXform(content); } - return(response.havepost = true); + return(request.postParsed = true); } // Parse X-form content in the body of the request diff --git a/danode/request.d b/danode/request.d index 42e93e4..68c2ee0 100644 --- a/danode/request.d +++ b/danode/request.d @@ -52,6 +52,7 @@ struct Request { string[string] headers; /// Associative array holding the header values SysTime starttime; /// start time of the Request PostItem[string] postinfo; /// Associative array holding the post parameters and values + bool postParsed = false; // Start a new Request, and parseHeader on the DriverInterface final void initialize(const DriverInterface driver) { diff --git a/danode/response.d b/danode/response.d index 9c95930..76c4822 100644 --- a/danode/response.d +++ b/danode/response.d @@ -24,7 +24,6 @@ struct Response { string[string] headers; Payload payload; bool created = false; - bool havepost = false; bool routed = false; bool completed = false; Appender!(char[]) hdr; @@ -142,7 +141,7 @@ Response create(in Request request, Address address, in StatusCode statuscode = bool setPayload(ref Response response, StatusCode code, string msg = "", in string mimetype = UNSUPPORTED_FILE) { response.payload = new Message(code, msg, mimetype); - return(response.ready = response.havepost = true); + return(response.ready = true); } // send a redirect permanently response From a4fd0f670f49b9a9b75e68cf7bec44325830d392 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 11:37:19 +0000 Subject: [PATCH 20/46] Remove the false --- danode/server.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/danode/server.d b/danode/server.d index 88fa771..c798347 100644 --- a/danode/server.d +++ b/danode/server.d @@ -73,8 +73,8 @@ class Server { string ip = accepted.remoteAddress().toAddrString(); bool isLoopback = (ip == "127.0.0.1" || ip == "::1"); DriverInterface driver = null; - if (!secure) driver = new HTTP(accepted, false); - version(SSL) { if (secure) driver = new HTTPS(accepted, false); } + if (!secure) driver = new HTTP(accepted); + version(SSL) { if (secure) driver = new HTTPS(accepted); } if (driver is null) { accepted.close(); return; } if (!pool.push(driver, ip, isLoopback)) { log(Level.Always, "Rate limit or capacity exceeded [%s]", ip); From a65f1a32715341a67063860ce8f8f65af391aae9 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 12:42:22 +0000 Subject: [PATCH 21/46] Multipart streaming --- danode/interfaces.d | 9 ++++ danode/multipart.d | 129 ++++++++++++++++++++++++++++++++++++++++++++ danode/post.d | 110 +++++++++++-------------------------- danode/request.d | 4 +- danode/router.d | 2 +- 5 files changed, 173 insertions(+), 81 deletions(-) create mode 100644 danode/multipart.d diff --git a/danode/interfaces.d b/danode/interfaces.d index d12395a..d471a70 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -37,6 +37,15 @@ abstract class DriverInterface { } catch(Exception e) { error("Exception closing socket: %s", e.msg); } } + final void trimToHeader() { + ptrdiff_t hsize = bodyStart(); + if (hsize > 0 && hsize <= inbuffer.data.length) { + auto header = inbuffer.data[0 .. hsize].dup; + inbuffer.clear(); + inbuffer.put(header); + } + } + // Receive upto maxsize of bytes from the client into the input buffer ptrdiff_t receive(Socket socket, ptrdiff_t maxsize = 4096) { if (!socketReady()) return(-1); diff --git a/danode/multipart.d b/danode/multipart.d new file mode 100644 index 0000000..ffaa05f --- /dev/null +++ b/danode/multipart.d @@ -0,0 +1,129 @@ +/** danode/server.d - Entry point: socket setup, connection acceptance, rate limiting + * License: GPLv3 (https://github.com/DannyArends/DaNode) - Danny Arends **/ +module danode.multipart; + +import danode.imports; + +import danode.request : Request; +import danode.post : PostItem, PostType; +import danode.log : log, tag, error, Level; + +enum MPState { INIT, HEADER, BODY } + +struct MultipartParser { + string boundary; /// "--boundary" + string uploadDir; /// directory for .up files + MPState state = MPState.INIT; + char[] tail; /// leftover bytes from previous chunk (boundary detection) + File outfile; /// current open output file + string currentPath; /// current .up file path + string currentMime; /// current part mime type + string currentName; /// current field name + string currentFname; /// current filename + Appender!(char[]) hdrbuf; /// accumulating part header bytes + bool done = false; /// final boundary seen + + @property bool isActive() const { return boundary.length > 0; } + + bool feed(ref Request request, const(char)[] chunk) { + // Prepend any leftover tail from previous chunk + char[] data = tail ~ chunk; + tail = []; + + while (data.length > 0 && !done) { + final switch (state) { + case MPState.INIT: // Find opening boundary + ptrdiff_t i = indexOf(data, boundary ~ "\r\n"); + if (i < 0) { tail = data.dup; return(false); } + data = data[i + boundary.length + 2 .. $]; + state = MPState.HEADER; + break; + + case MPState.HEADER: // Accumulate until \r\n\r\n + ptrdiff_t i = indexOf(data, "\r\n\r\n"); + if (i < 0) { hdrbuf.put(data); tail = []; return(false); } + hdrbuf.put(data[0 .. i]); + data = data[i + 4 .. $]; + // Parse headers + parsePartHeader(request); + hdrbuf.clear(); + state = MPState.BODY; + break; + + case MPState.BODY: // Look for \r\n--boundary + string delim = "\r\n" ~ boundary; + ptrdiff_t i = indexOf(data, delim); + if (i < 0) { // No boundary found - write all but tail + ptrdiff_t safe = cast(ptrdiff_t)data.length - cast(ptrdiff_t)delim.length; + if (safe > 0) { writeChunk(data[0 .. safe]); data = data[safe .. $]; } + tail = data.dup; + return(false); + } + // Boundary found - write up to it and close part + writeChunk(data[0 .. i]); + closePart(request); + data = data[i + delim.length .. $]; + // Check for final boundary (--) or next part (\r\n) + if (data.length >= 2 && data[0..2] == "--") { return(done = true); } + if (data.length >= 2 && data[0..2] == "\r\n") { data = data[2..$]; } + state = MPState.HEADER; + break; + } + } + return done; + } + + private void parsePartHeader(ref Request request) { + string header = to!string(hdrbuf.data); + currentName = extractQuoted(header, "name"); + currentFname = extractQuoted(header, "filename"); + currentMime = "application/octet-stream"; + foreach (line; header.split("\r\n")) { + if (line.toLower.startsWith("content-type:")) { currentMime = strip(line[line.indexOf(":")+1 .. $]); break; } + } + if (currentFname.length > 0) { + currentPath = uploadDir ~ md5UUID(format("%s-%s", request.id, currentName)).toString() ~ ".up"; + try { outfile = File(currentPath, "wb"); } + catch(Exception e) { error("MultipartParser: failed to open '%s': %s", currentPath, e.msg); } + log(Level.Verbose, "MPART: [I] streaming file %s -> %s", currentFname, currentPath); + } + } + + private void writeChunk(const(char)[] chunk) { + if (currentFname.length > 0 && outfile.isOpen()) { + try { outfile.rawWrite(chunk); }catch(Exception e) { error("MultipartParser: write failed: %s", e.msg); } + } + } + + private void closePart(ref Request request) { + if (currentFname.length > 0) { + if (outfile.isOpen()) outfile.close(); + long sz = currentPath.exists ? currentPath.getSize() : 0; + request.postinfo[currentName] = PostItem(PostType.File, currentName, currentFname, currentPath, currentMime, sz); + log(Level.Verbose, "MPART: [I] closed file %s, %d bytes", currentPath, sz); + currentPath = ""; currentFname = ""; currentMime = ""; + } else if (currentName.length > 0) { // Plain input field - body was accumulated in tail, store as value + request.postinfo[currentName] = PostItem(PostType.Input, currentName, "", to!string(hdrbuf.data)); + } + currentName = ""; + } +} + +// Extract value from: name="value" or filename="value" +pure string extractQuoted(string s, string key) nothrow { + ptrdiff_t i = s.indexOf(key ~ "=\""); + if (i < 0) return ""; + i += key.length + 2; + ptrdiff_t j = s.indexOf("\"", i); + return j > i ? s[i .. j] : ""; +} + +unittest { + tag(Level.Always, "FILE", "%s", __FILE__); + + // extractQuoted + assert(extractQuoted("name=\"hello\"", "name") == "hello", "extractQuoted must get name"); + assert(extractQuoted("filename=\"test.txt\"", "filename") == "test.txt", "extractQuoted must get filename"); + assert(extractQuoted("name=\"\"", "name") == "", "extractQuoted empty value"); + assert(extractQuoted("nothing here", "name") == "", "extractQuoted missing key"); +} diff --git a/danode/post.d b/danode/post.d index d980cb7..fff5345 100644 --- a/danode/post.d +++ b/danode/post.d @@ -6,7 +6,7 @@ import danode.imports; import danode.cgi : CGI; import danode.statuscode : StatusCode; -import danode.interfaces : StringDriver; +import danode.interfaces : StringDriver, DriverInterface; import danode.request : Request, RequestMethod; import danode.response : Response, setPayload; import danode.webconfig : WebConfig; @@ -15,6 +15,7 @@ import danode.filesystem : FileSystem; import danode.functions : from, writeFile, parseQueryString; import danode.log : log, tag, error, Level; import danode.webconfig : serverConfig; +import danode.multipart : MultipartParser; immutable string MPHEADER = "multipart/form-data"; /// Multipart header id immutable string XFORMHEADER = "application/x-www-form-urlencoded"; /// X-form header id @@ -33,45 +34,52 @@ struct PostItem { // Parse the POST request data from the client, or waits (returning false) for more data // when the entire request body is not yet available. POST data supplied in Multipart // and X-form post formats are currently supported -final bool parsePost(ref Request request, ref Response response, in FileSystem filesystem) { +final bool parsePost(ref Request request, ref Response response, in FileSystem filesystem, DriverInterface driver = null) { if (request.postParsed || request.method != RequestMethod.POST) { return(request.postParsed = true); } long expectedlength; try { expectedlength = to!long(request.headers.from("Content-Length", "0")); - } catch (Exception e) { - return(response.setPayload(StatusCode.BadRequest, "400 - Bad Request\n", "text/plain")); - } - string content = request.body; + } catch (Exception e) { return(response.setPayload(StatusCode.BadRequest, "400 - Bad Request\n", "text/plain")); } + string contenttype = from(request.headers, "Content-Type"); - log(Level.Trace, "content type: %s", contenttype); size_t limit = (contenttype.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize; - if (expectedlength == 0) { - log(Level.Trace, "Post: [T] Content-Length not specified (or 0), length: %s", content.length); - return(request.postParsed = true); // When we don't receive any post data it is meaningless to scan for any content - } else if (expectedlength > limit) { + + if (expectedlength == 0) { return(request.postParsed = true); } + if (expectedlength > limit) { log(Level.Verbose, "Post: [W] Upload too large: %d bytes from %s", expectedlength, request.ip); return(response.setPayload(StatusCode.PayloadTooLarge, "413 - Payload Too Large\n", "text/plain")); } + + if (contenttype.indexOf(MPHEADER) >= 0) { + auto parts = split(contenttype, "boundary="); + if (parts.length < 2) return(request.postParsed = true); + // Initialize parser on first call + if (!request.mpParser.isActive) { + string mpid = "--" ~ parts[1]; + request.mpParser = MultipartParser(mpid, filesystem.localroot(request.shorthost()) ~ "/"); + log(Level.Verbose, "MPART: [I] streaming mode activated, boundary: %s", mpid); + } + // Feed current body bytes to parser + if (driver !is null) { + auto bodyData = driver.inbuffer.data[driver.bodyStart .. $]; + if (request.mpParser.feed(request, bodyData)) { return(request.postParsed = true); } + driver.trimToHeader(); + return false; + } + } + + // Non-multipart: wait for full body as before + string content = request.body; log(Level.Trace, "Post: [T] Received %s of %s", content.length, expectedlength); if(content.length < expectedlength) return(false); if (contenttype.indexOf(XFORMHEADER) >= 0) { - log(Level.Verbose, "XFORM: [I] parsing %d bytes", expectedlength); request.parseXform(content); - log(Level.Verbose, "XFORM: [T] # of items: %s", request.postinfo.length); - } else if (contenttype.indexOf(MPHEADER) >= 0) { - auto parts = split(contenttype, "boundary="); - if (parts.length < 2) return(request.postParsed = true); - string mpid = parts[1]; - log(Level.Verbose, "MPART: [I] header: %s, parsing %d bytes", mpid, expectedlength); - request.parseMultipart(filesystem, content, mpid); - log(Level.Verbose, "MPART: [I] # of items: %s", request.postinfo.length); } else if (contenttype.indexOf(JSON) >= 0) { - log(Level.Verbose, "JSON: [I] Parsing %d bytes", expectedlength); - // JSON body is passed raw to the script via stdin - no server-side parsing needed + log(Level.Verbose, "JSON: [I] passing %d bytes raw to script", expectedlength); } else { - error("parsePost: Unsupported POST content type: %s [%s] -> %s", contenttype, expectedlength, content); + error("parsePost: Unsupported POST content type: %s [%s]", contenttype, expectedlength); request.parseXform(content); } return(request.postParsed = true); @@ -82,61 +90,11 @@ final void parseXform(ref Request request, const string content) { foreach (k, v; parseQueryString(content)) { request.postinfo[k] = PostItem(PostType.Input, k, "", v); } } -// Extract value from: name="value" or filename="value" -pure string extractQuoted(string s, string key) nothrow { - ptrdiff_t i = s.indexOf(key ~ "=\""); - if (i < 0) return ""; - i += key.length + 2; - ptrdiff_t j = s.indexOf("\"", i); - return j > i ? s[i .. j] : ""; -} - @nogc pure ptrdiff_t findBodyLine(in string[] lines) nothrow { foreach (i, line; lines) { if (strip(line).length == 0) return i + 1; } return -1; } -// Parse Multipart content in the body of the request -final void parseMultipart(ref Request request, in FileSystem filesystem, const string content, const string mpid) { - int[string] keys; - foreach (part; chomp(content).split(mpid)) { - string[] elem = strip(part).split("\r\n"); - if (elem.length < 2 || elem[0] == "--") { - log(Level.Verbose, "MPART: [I] ID element: %s", elem.length > 0 ? elem[0] : ""); continue; - } - - string[] mphdr = elem[0].split("; "); - if (mphdr.length < 2) continue; - - string key = extractQuoted(elem[0], "name"); - if (key.length == 0) continue; - - ptrdiff_t bodyLine = findBodyLine(elem); - if (bodyLine < 0 || bodyLine >= elem.length) continue; - - if (mphdr.length == 2) { - request.postinfo[key] = PostItem(PostType.Input, key, "", join(elem[bodyLine .. ($-1)])); - } else if (mphdr.length >= 3) { - string fname = extractQuoted(elem[0], "filename"); - log(Level.Verbose, "MPART: [I] found on key %s file %s", key, fname); - bool isarraykey = key.length > 2 && key[($-2) .. $] == "[]"; - keys[key] = keys.get(key, -1) + 1; - log(Level.Verbose, "MPART: [I] found on key %s #%d file %s", key, keys[key], fname); - if (fname != "") { - string fkey = isarraykey ? key ~ to!string(keys[key]) : key; - string skey = isarraykey ? key[0 .. $-2] : key; - string localpath = request.uploadfile(filesystem, fkey); - string mpcontent = join(elem[bodyLine .. ($-1)], "\r\n"); - auto mimeParts = split(elem[bodyLine-1], ": "); - string fileMime = mimeParts.length >= 2 ? mimeParts[1] : "application/octet-stream"; - request.postinfo[fkey] = PostItem(PostType.File, skey, fname, localpath, fileMime, mpcontent.length); - localpath.writeFile(mpcontent); - log(Level.Verbose, "MPART: [I] Wrote %d bytes to file %s", mpcontent.length, localpath); - } else { request.postinfo[key] = PostItem(PostType.Input, key, ""); } - } - } -} - /* The serverAPI functions prepares and writes out the input file for external process execution The inputfile contains the SERVER, COOKIES, POST, and FILES information that can be used by the external script This data is picked-up by the different CGI APIs, and presented to the client in the regular way */ @@ -181,12 +139,6 @@ unittest { FileSystem fs = new FileSystem("./www/"); - // extractQuoted - assert(extractQuoted("name=\"hello\"", "name") == "hello", "extractQuoted must get name"); - assert(extractQuoted("filename=\"test.txt\"", "filename") == "test.txt", "extractQuoted must get filename"); - assert(extractQuoted("name=\"\"", "name") == "", "extractQuoted empty value"); - assert(extractQuoted("nothing here", "name") == "", "extractQuoted missing key"); - // findBodyLine assert(findBodyLine(["Content-Disposition: form-data", "", "value"]) == 2, "findBodyLine must find blank line"); assert(findBodyLine(["Content-Disposition: form-data"]) == -1, "findBodyLine no blank must return -1"); diff --git a/danode/request.d b/danode/request.d index 68c2ee0..7858962 100644 --- a/danode/request.d +++ b/danode/request.d @@ -10,6 +10,7 @@ import danode.functions : interpreter, from, parseHtmlDate, parseQueryString; import danode.webconfig : WebConfig, serverConfig; import danode.post : PostItem, PostType; import danode.log : log, tag, error, Level; +import danode.multipart : MultipartParser; // The Request-Method indicates which method is to be performed on the specified resource enum RequestMethod : string { @@ -53,7 +54,8 @@ struct Request { SysTime starttime; /// start time of the Request PostItem[string] postinfo; /// Associative array holding the post parameters and values bool postParsed = false; - + MultipartParser mpParser; /// streaming multipart parser + // Start a new Request, and parseHeader on the DriverInterface final void initialize(const DriverInterface driver) { this.ip = driver.ip; diff --git a/danode/router.d b/danode/router.d index 8093790..9df00a8 100644 --- a/danode/router.d +++ b/danode/router.d @@ -46,7 +46,7 @@ class Router { // Route a request based on the request header final void route(DriverInterface driver, ref Request request, ref Response response) { if (!response.routed && parse(driver, request, response)) { - if (request.parsePost(response, filesystem)) { deliver(request, response); } + if (request.parsePost(response, filesystem, driver)) { deliver(request, response); } } } From c185fc93a78d4994a87d85f4b9e606499cebb45e Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 13:11:04 +0000 Subject: [PATCH 22/46] Adding unittests and debug --- danode/multipart.d | 166 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 135 insertions(+), 31 deletions(-) diff --git a/danode/multipart.d b/danode/multipart.d index ffaa05f..fd20b15 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -11,17 +11,18 @@ import danode.log : log, tag, error, Level; enum MPState { INIT, HEADER, BODY } struct MultipartParser { - string boundary; /// "--boundary" - string uploadDir; /// directory for .up files - MPState state = MPState.INIT; - char[] tail; /// leftover bytes from previous chunk (boundary detection) - File outfile; /// current open output file - string currentPath; /// current .up file path - string currentMime; /// current part mime type - string currentName; /// current field name - string currentFname; /// current filename + string boundary; /// "--boundary" + string uploadDir; /// directory for .up files + MPState state = MPState.INIT; + char[] tail; /// leftover bytes from previous chunk (boundary detection) + File outfile; /// current open output file + string currentPath; /// current .up file path + string currentMime; /// current part mime type + string currentName; /// current field name + string currentFname; /// current filename Appender!(char[]) hdrbuf; /// accumulating part header bytes - bool done = false; /// final boundary seen + bool done = false; /// final boundary seen + Appender!(char[]) valuebuf; // accumulates plain field value @property bool isActive() const { return boundary.length > 0; } @@ -39,35 +40,43 @@ struct MultipartParser { state = MPState.HEADER; break; - case MPState.HEADER: // Accumulate until \r\n\r\n - ptrdiff_t i = indexOf(data, "\r\n\r\n"); - if (i < 0) { hdrbuf.put(data); tail = []; return(false); } - hdrbuf.put(data[0 .. i]); - data = data[i + 4 .. $]; - // Parse headers - parsePartHeader(request); - hdrbuf.clear(); - state = MPState.BODY; - break; + case MPState.HEADER: // Accumulate until \r\n\r\n + hdrbuf.put(data); + ptrdiff_t i = indexOf(cast(string)hdrbuf.data, "\r\n\r\n"); + if (i < 0) { tail = []; return(false); } + data = hdrbuf.data[i + 4 .. $].dup; + hdrbuf.shrinkTo(i); + parsePartHeader(request); + hdrbuf.clear(); + state = MPState.BODY; + break; case MPState.BODY: // Look for \r\n--boundary string delim = "\r\n" ~ boundary; ptrdiff_t i = indexOf(data, delim); if (i < 0) { // No boundary found - write all but tail - ptrdiff_t safe = cast(ptrdiff_t)data.length - cast(ptrdiff_t)delim.length; - if (safe > 0) { writeChunk(data[0 .. safe]); data = data[safe .. $]; } - tail = data.dup; + ptrdiff_t keep = 0; + foreach_reverse (k; 1 .. min(delim.length, data.length) + 1) { + if (data[$ - k .. $] == delim[0 .. k]) { keep = k; break; } + } + ptrdiff_t safe = cast(ptrdiff_t)data.length - keep; + if (safe > 0) { writeChunk(data[0 .. safe]); } + tail = data[safe .. $].dup; return(false); } + if (i + delim.length + 2 > data.length) { tail = data[i .. $].dup; if (i > 0) writeChunk(data[0 .. i]); return false; } // Boundary found - write up to it and close part writeChunk(data[0 .. i]); closePart(request); + //writefln("POST-BOUNDARY data='%s' len=%d", data, data.length); data = data[i + delim.length .. $]; - // Check for final boundary (--) or next part (\r\n) + // Check for final boundary (--) or next part (\r\n) + if (data.length == 0) { tail = cast(char[])[]; return false; } if (data.length >= 2 && data[0..2] == "--") { return(done = true); } - if (data.length >= 2 && data[0..2] == "\r\n") { data = data[2..$]; } + if (data[0] == '-') { tail = data.dup; return false; } + if (data.length >= 2 && data[0..2] == "\r\n") { data = data[2..$]; state = MPState.HEADER; break; } + else if (data[0] == '\r') { tail = data.dup; return false; } state = MPState.HEADER; - break; } } return done; @@ -91,8 +100,8 @@ struct MultipartParser { private void writeChunk(const(char)[] chunk) { if (currentFname.length > 0 && outfile.isOpen()) { - try { outfile.rawWrite(chunk); }catch(Exception e) { error("MultipartParser: write failed: %s", e.msg); } - } + try { outfile.rawWrite(chunk); } catch(Exception e) { error("MultipartParser: write failed: %s", e.msg); } + } else if (currentFname.length == 0 && currentName.length > 0) { valuebuf.put(chunk); } } private void closePart(ref Request request) { @@ -102,8 +111,9 @@ struct MultipartParser { request.postinfo[currentName] = PostItem(PostType.File, currentName, currentFname, currentPath, currentMime, sz); log(Level.Verbose, "MPART: [I] closed file %s, %d bytes", currentPath, sz); currentPath = ""; currentFname = ""; currentMime = ""; - } else if (currentName.length > 0) { // Plain input field - body was accumulated in tail, store as value - request.postinfo[currentName] = PostItem(PostType.Input, currentName, "", to!string(hdrbuf.data)); + } else if (currentName.length > 0) { // Plain input field -accumulated valuebuf + request.postinfo[currentName] = PostItem(PostType.Input, currentName, "", to!string(valuebuf.data)); + valuebuf.clear(); } currentName = ""; } @@ -119,6 +129,9 @@ pure string extractQuoted(string s, string key) nothrow { } unittest { + import danode.request : Request; + import danode.filesystem : FileSystem; + tag(Level.Always, "FILE", "%s", __FILE__); // extractQuoted @@ -126,4 +139,95 @@ unittest { assert(extractQuoted("filename=\"test.txt\"", "filename") == "test.txt", "extractQuoted must get filename"); assert(extractQuoted("name=\"\"", "name") == "", "extractQuoted empty value"); assert(extractQuoted("nothing here", "name") == "", "extractQuoted missing key"); -} + + // Helper to build a multipart body + string buildMultipart(string boundary, string[2][] textFields, string[3][] fileFields) { + string body; + foreach (f; textFields) { + body ~= "--" ~ boundary ~ "\r\n"; + body ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"\r\n\r\n"; + body ~= f[1] ~ "\r\n"; + } + foreach (f; fileFields) { + body ~= "--" ~ boundary ~ "\r\n"; + body ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"; filename=\"" ~ f[1] ~ "\"\r\n"; + body ~= "Content-Type: application/octet-stream\r\n\r\n"; + body ~= f[2] ~ "\r\n"; + } + body ~= "--" ~ boundary ~ "--\r\n"; + return body; + } + + FileSystem fs = new FileSystem("./www/"); + string uploadDir = fs.localroot("localhost") ~ "/"; + string boundary = "testboundary123"; + + // Test 1: single text field + { + Request r; + r.id = md5UUID("test1"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string body = buildMultipart(boundary, [["name", "danny"]], []); + //writefln("TEST1 body='%s' len=%d", body, body.length); + bool result = parser.feed(r, body); + //writefln("TEST1 result=%s state=%s done=%s postinfo=%s", result, parser.state, parser.done, r.postinfo); + assert(result, "single text field must complete"); + } + + // Test 2: single file upload + { + Request r; + r.id = md5UUID("test2"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string body = buildMultipart(boundary, [], [["file", "test.txt", "hello world"]]); + assert(parser.feed(r, body), "single file must complete"); + assert("file" in r.postinfo, "file must be in postinfo"); + assert(r.postinfo["file"].type == PostType.File, "must be File type"); + assert(r.postinfo["file"].filename == "test.txt", "filename must match"); + assert(r.postinfo["file"].size == "hello world".length, "size must match"); + // cleanup + if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); + } + + // Test 3: mixed text + file + { + Request r; + r.id = md5UUID("test3"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string body = buildMultipart(boundary, [["name", "danny"]], [["file", "data.bin", "binarydata"]]); + assert(parser.feed(r, body), "mixed must complete"); + assert(r.postinfo["name"].value == "danny", "text field must parse"); + assert(r.postinfo["file"].type == PostType.File, "file field must parse"); + if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); + } + + // Test 4: cross-chunk boundary detection - feed 1 byte at a time + { + Request r; + r.id = md5UUID("test4"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string body = buildMultipart(boundary, [["field", "value"]], []); + bool done = false; + foreach (i; 0 .. body.length) { + done = parser.feed(r, body[i..i+1]); + //writefln("byte %d '%s' state=%s tail='%s' done=%s", i, body[i], parser.state, parser.tail, done); + if (done) break; + } + //writefln("final state=%s postinfo=%s", parser.state, r.postinfo); + assert(done, "byte-by-byte feed must complete"); + } + + // Test 5: binary file with = and \r\n in content + { + Request r; + r.id = md5UUID("test5"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string binaryContent = "data=with=equals\r\nand newlines\r\nmore data"; + string body = buildMultipart(boundary, [], [["bin", "binary.bin", binaryContent]]); + assert(parser.feed(r, body), "binary content must complete"); + assert(r.postinfo["bin"].size == binaryContent.length, "binary size must match"); + string written = cast(string) read(r.postinfo["bin"].value); + assert(written == binaryContent, "binary content must be preserved exactly"); + if (r.postinfo["bin"].value.exists) remove(r.postinfo["bin"].value); + } +} \ No newline at end of file From 928392718327311b35c65f16bcd1417f61c9b0e3 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 13:13:37 +0000 Subject: [PATCH 23/46] Remove debug --- danode/multipart.d | 5 ----- 1 file changed, 5 deletions(-) diff --git a/danode/multipart.d b/danode/multipart.d index fd20b15..1c4f88d 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -68,7 +68,6 @@ struct MultipartParser { // Boundary found - write up to it and close part writeChunk(data[0 .. i]); closePart(request); - //writefln("POST-BOUNDARY data='%s' len=%d", data, data.length); data = data[i + delim.length .. $]; // Check for final boundary (--) or next part (\r\n) if (data.length == 0) { tail = cast(char[])[]; return false; } @@ -168,9 +167,7 @@ unittest { r.id = md5UUID("test1"); auto parser = MultipartParser("--" ~ boundary, uploadDir); string body = buildMultipart(boundary, [["name", "danny"]], []); - //writefln("TEST1 body='%s' len=%d", body, body.length); bool result = parser.feed(r, body); - //writefln("TEST1 result=%s state=%s done=%s postinfo=%s", result, parser.state, parser.done, r.postinfo); assert(result, "single text field must complete"); } @@ -210,10 +207,8 @@ unittest { bool done = false; foreach (i; 0 .. body.length) { done = parser.feed(r, body[i..i+1]); - //writefln("byte %d '%s' state=%s tail='%s' done=%s", i, body[i], parser.state, parser.tail, done); if (done) break; } - //writefln("final state=%s postinfo=%s", parser.state, r.postinfo); assert(done, "byte-by-byte feed must complete"); } From 36883d7a145317810c6e079d9747144a0e268310 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 17:34:20 +0000 Subject: [PATCH 24/46] Some code updates --- danode/interfaces.d | 25 ++++++++++++++++------ danode/multipart.d | 52 ++++++++++++++++++++++++--------------------- danode/post.d | 7 ++---- 3 files changed, 49 insertions(+), 35 deletions(-) diff --git a/danode/interfaces.d b/danode/interfaces.d index d471a70..a887d65 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -37,13 +37,26 @@ abstract class DriverInterface { } catch(Exception e) { error("Exception closing socket: %s", e.msg); } } - final void trimToHeader() { - ptrdiff_t hsize = bodyStart(); - if (hsize > 0 && hsize <= inbuffer.data.length) { - auto header = inbuffer.data[0 .. hsize].dup; - inbuffer.clear(); - inbuffer.put(header); + // Receive a raw chunk without buffering into inbuffer - for streaming use + final const(char)[] receiveChunk(ptrdiff_t maxsize = 65536) { + // Drain any body bytes already in inbuffer first + ptrdiff_t bs = bodyStart(); + if (bs >= 0 && bs < inbuffer.data.length) { + auto buffered = inbuffer.data[bs .. $].dup; + ptrdiff_t hsize = bodyStart(); + if (hsize > 0 && hsize <= inbuffer.data.length) { + auto header = inbuffer.data[0 .. hsize].dup; + inbuffer.clear(); + inbuffer.put(header); + } + return buffered; } + if (!socketReady()) return []; + if (socketSet.sISelect(socket, false, 25) <= 0) return []; + char[] tmpbuffer = new char[](maxsize); + ptrdiff_t received = receiveData(tmpbuffer); + if (received > 0) { touch(); return tmpbuffer[0 .. received]; } + return []; } // Receive upto maxsize of bytes from the client into the input buffer diff --git a/danode/multipart.d b/danode/multipart.d index 1c4f88d..bf696f9 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -11,31 +11,37 @@ import danode.log : log, tag, error, Level; enum MPState { INIT, HEADER, BODY } struct MultipartParser { - string boundary; /// "--boundary" - string uploadDir; /// directory for .up files - MPState state = MPState.INIT; - char[] tail; /// leftover bytes from previous chunk (boundary detection) - File outfile; /// current open output file - string currentPath; /// current .up file path - string currentMime; /// current part mime type - string currentName; /// current field name - string currentFname; /// current filename - Appender!(char[]) hdrbuf; /// accumulating part header bytes - bool done = false; /// final boundary seen - Appender!(char[]) valuebuf; // accumulates plain field value + string boundary; /// "--boundary" + string uploadDir; /// directory for .up files + MPState state = MPState.INIT; /// State of the parser + char[] tail; /// leftover bytes from previous chunk (boundary detection) + File outfile; /// current open output file + string currentPath; /// current .up file path + string currentMime; /// current part mime type + string currentName; /// current field name + string currentFname; /// current filename + Appender!(char[]) hdrbuf; /// accumulating part header bytes + bool done = false; /// final boundary seen + string delim; /// "\r\n--boundary", cached + Appender!(char[]) valuebuf; /// accumulates plain field value + + this (string boundary, string uploadDir) { + this.boundary = boundary; + this.uploadDir = uploadDir; + this.delim = "\r\n" ~ boundary; + } @property bool isActive() const { return boundary.length > 0; } bool feed(ref Request request, const(char)[] chunk) { - // Prepend any leftover tail from previous chunk - char[] data = tail ~ chunk; + char[] data = tail ~ chunk; // Prepend any leftover tail from previous chunk tail = []; while (data.length > 0 && !done) { final switch (state) { case MPState.INIT: // Find opening boundary ptrdiff_t i = indexOf(data, boundary ~ "\r\n"); - if (i < 0) { tail = data.dup; return(false); } + if (i < 0) { return(saveTail(data)); } data = data[i + boundary.length + 2 .. $]; state = MPState.HEADER; break; @@ -52,7 +58,6 @@ struct MultipartParser { break; case MPState.BODY: // Look for \r\n--boundary - string delim = "\r\n" ~ boundary; ptrdiff_t i = indexOf(data, delim); if (i < 0) { // No boundary found - write all but tail ptrdiff_t keep = 0; @@ -61,26 +66,25 @@ struct MultipartParser { } ptrdiff_t safe = cast(ptrdiff_t)data.length - keep; if (safe > 0) { writeChunk(data[0 .. safe]); } - tail = data[safe .. $].dup; - return(false); + return(saveTail(data)); } if (i + delim.length + 2 > data.length) { tail = data[i .. $].dup; if (i > 0) writeChunk(data[0 .. i]); return false; } // Boundary found - write up to it and close part writeChunk(data[0 .. i]); closePart(request); data = data[i + delim.length .. $]; - // Check for final boundary (--) or next part (\r\n) - if (data.length == 0) { tail = cast(char[])[]; return false; } - if (data.length >= 2 && data[0..2] == "--") { return(done = true); } - if (data[0] == '-') { tail = data.dup; return false; } - if (data.length >= 2 && data[0..2] == "\r\n") { data = data[2..$]; state = MPState.HEADER; break; } - else if (data[0] == '\r') { tail = data.dup; return false; } + if (data.length < 2) { return saveTail(data); } + if (data[0..2] == "--") { return(done = true); } + if (data[0..2] == "\r\n") { data = data[2..$]; } + else { return saveTail(data); } state = MPState.HEADER; } } return done; } + private bool saveTail(char[] d) { tail = d.dup; return false; } + private void parsePartHeader(ref Request request) { string header = to!string(hdrbuf.data); currentName = extractQuoted(header, "name"); diff --git a/danode/post.d b/danode/post.d index fff5345..49ce787 100644 --- a/danode/post.d +++ b/danode/post.d @@ -54,17 +54,14 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f if (contenttype.indexOf(MPHEADER) >= 0) { auto parts = split(contenttype, "boundary="); if (parts.length < 2) return(request.postParsed = true); - // Initialize parser on first call if (!request.mpParser.isActive) { string mpid = "--" ~ parts[1]; request.mpParser = MultipartParser(mpid, filesystem.localroot(request.shorthost()) ~ "/"); log(Level.Verbose, "MPART: [I] streaming mode activated, boundary: %s", mpid); } - // Feed current body bytes to parser if (driver !is null) { - auto bodyData = driver.inbuffer.data[driver.bodyStart .. $]; - if (request.mpParser.feed(request, bodyData)) { return(request.postParsed = true); } - driver.trimToHeader(); + auto chunk = driver.receiveChunk(); + if (chunk.length > 0 && request.mpParser.feed(request, chunk)) return(request.postParsed = true); return false; } } From 5fd3fe7939413f637161bfdbc3d480b9ddabc319 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 17:39:49 +0000 Subject: [PATCH 25/46] Fixing some minor bugs --- danode/interfaces.d | 5 ++--- danode/multipart.d | 28 +++++++++++++--------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/danode/interfaces.d b/danode/interfaces.d index a887d65..deedc62 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -43,9 +43,8 @@ abstract class DriverInterface { ptrdiff_t bs = bodyStart(); if (bs >= 0 && bs < inbuffer.data.length) { auto buffered = inbuffer.data[bs .. $].dup; - ptrdiff_t hsize = bodyStart(); - if (hsize > 0 && hsize <= inbuffer.data.length) { - auto header = inbuffer.data[0 .. hsize].dup; + if (bs > 0 && bs <= inbuffer.data.length) { + auto header = inbuffer.data[0 .. bs].dup; inbuffer.clear(); inbuffer.put(header); } diff --git a/danode/multipart.d b/danode/multipart.d index bf696f9..319dbea 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -1,4 +1,4 @@ -/** danode/server.d - Entry point: socket setup, connection acceptance, rate limiting +/** danode/multipart.d - Streaming multipart/form-data parser * License: GPLv3 (https://github.com/DannyArends/DaNode) - Danny Arends **/ module danode.multipart; @@ -45,18 +45,16 @@ struct MultipartParser { data = data[i + boundary.length + 2 .. $]; state = MPState.HEADER; break; - - case MPState.HEADER: // Accumulate until \r\n\r\n - hdrbuf.put(data); - ptrdiff_t i = indexOf(cast(string)hdrbuf.data, "\r\n\r\n"); - if (i < 0) { tail = []; return(false); } - data = hdrbuf.data[i + 4 .. $].dup; - hdrbuf.shrinkTo(i); - parsePartHeader(request); - hdrbuf.clear(); - state = MPState.BODY; - break; - + case MPState.HEADER: // Accumulate until \r\n\r\n + hdrbuf.put(data); + ptrdiff_t i = indexOf(cast(string)hdrbuf.data, "\r\n\r\n"); + if (i < 0) { tail = []; return(false); } + data = hdrbuf.data[i + 4 .. $].dup; + hdrbuf.shrinkTo(i); + parsePartHeader(request); + hdrbuf.clear(); + state = MPState.BODY; + break; case MPState.BODY: // Look for \r\n--boundary ptrdiff_t i = indexOf(data, delim); if (i < 0) { // No boundary found - write all but tail @@ -66,9 +64,9 @@ struct MultipartParser { } ptrdiff_t safe = cast(ptrdiff_t)data.length - keep; if (safe > 0) { writeChunk(data[0 .. safe]); } - return(saveTail(data)); + return(saveTail(data[safe .. $])); } - if (i + delim.length + 2 > data.length) { tail = data[i .. $].dup; if (i > 0) writeChunk(data[0 .. i]); return false; } + if (i + delim.length + 2 > data.length) { tail = data[i .. $].dup; if (i > 0) writeChunk(data[0 .. i]); return(false); } // Boundary found - write up to it and close part writeChunk(data[0 .. i]); closePart(request); From dec0fd7ec05483dd8e36fde973cb8ab8699c6b00 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 17:59:38 +0000 Subject: [PATCH 26/46] Extract a helper --- danode/client.d | 2 +- danode/interfaces.d | 37 +++++++++++++++++-------------------- danode/multipart.d | 4 ++-- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/danode/client.d b/danode/client.d index 7adaf8c..3644d82 100644 --- a/danode/client.d +++ b/danode/client.d @@ -41,7 +41,7 @@ class Client { if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; } size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); while (running) { - if (driver.receive(driver.socket) > 0) { // We've received new data + if (driver.receive() > 0) { // We've received new data if (!driver.hasHeader()) { if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } } else { diff --git a/danode/interfaces.d b/danode/interfaces.d index deedc62..4434a7c 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -31,6 +31,12 @@ abstract class DriverInterface { } bool socketReady() const { if (socket !is null) { return socket.isAlive(); } return false; }; /// Is the connection alive ? void touch() { modtime = Clock.currTime(); } + private ptrdiff_t readSocket(ref char[] tmpbuffer) { + if (!socketReady() || socketSet.sISelect(socket, false, 25) <= 0) return 0; + ptrdiff_t received = receiveData(tmpbuffer); + if (received > 0) { touch(); log(Level.Trace, "Received %d bytes of data", received); } + return received; + } void closeSocket() { try { if (socket !is null) { if (socket.isAlive()) { socket.shutdown(SocketShutdown.BOTH); } socket.close(); } @@ -39,33 +45,24 @@ abstract class DriverInterface { // Receive a raw chunk without buffering into inbuffer - for streaming use final const(char)[] receiveChunk(ptrdiff_t maxsize = 65536) { - // Drain any body bytes already in inbuffer first ptrdiff_t bs = bodyStart(); - if (bs >= 0 && bs < inbuffer.data.length) { + if (bs > 0 && bs < inbuffer.data.length) { auto buffered = inbuffer.data[bs .. $].dup; - if (bs > 0 && bs <= inbuffer.data.length) { - auto header = inbuffer.data[0 .. bs].dup; - inbuffer.clear(); - inbuffer.put(header); - } - return buffered; + auto header = inbuffer.data[0 .. bs].dup; + inbuffer.clear(); + inbuffer.put(header); + return(buffered); } - if (!socketReady()) return []; - if (socketSet.sISelect(socket, false, 25) <= 0) return []; char[] tmpbuffer = new char[](maxsize); - ptrdiff_t received = receiveData(tmpbuffer); - if (received > 0) { touch(); return tmpbuffer[0 .. received]; } - return []; + ptrdiff_t received = readSocket(tmpbuffer); + return(received > 0 ? tmpbuffer[0 .. received] : []); } // Receive upto maxsize of bytes from the client into the input buffer - ptrdiff_t receive(Socket socket, ptrdiff_t maxsize = 4096) { - if (!socketReady()) return(-1); - if (socketSet.sISelect(socket, false, 25) <= 0) return(0); - ptrdiff_t received; + ptrdiff_t receive(ptrdiff_t maxsize = 4096) { char[] tmpbuffer = new char[](maxsize); - if ((received = receiveData(tmpbuffer)) > 0) { inbuffer.put(tmpbuffer[0 .. received]); touch(); } - if(received > 0) log(Level.Trace, "Received %d bytes of data", received); + ptrdiff_t received = readSocket(tmpbuffer); + if (received > 0) inbuffer.put(tmpbuffer[0 .. received]); return(inbuffer.data.length); } @@ -128,7 +125,7 @@ class StringDriver : DriverInterface { override bool socketReady() const { return inbuffer.data.length > 0; } @nogc override bool isSecure() const nothrow { return(false); } override long receiveData(ref char[] buffer) { return(0); } // unused - overriding receive() directly - override ptrdiff_t receive(Socket socket, ptrdiff_t maxsize = 4096) { + override ptrdiff_t receive(ptrdiff_t maxsize = 4096) { if (inbuffer.data.length != 0) touch(); return(inbuffer.data.length); } diff --git a/danode/multipart.d b/danode/multipart.d index 319dbea..4448bbc 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -123,10 +123,10 @@ struct MultipartParser { // Extract value from: name="value" or filename="value" pure string extractQuoted(string s, string key) nothrow { ptrdiff_t i = s.indexOf(key ~ "=\""); - if (i < 0) return ""; + if (i < 0) return(""); i += key.length + 2; ptrdiff_t j = s.indexOf("\"", i); - return j > i ? s[i .. j] : ""; + return(j > i ? s[i .. j] : ""); } unittest { From 370500e0e407a00da921b51c22f444c4ee6c8bf4 Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 18:04:45 +0000 Subject: [PATCH 27/46] body = mp or content --- danode/interfaces.d | 2 +- danode/multipart.d | 42 +++++++++++++++++++++--------------------- danode/post.d | 9 ++++----- danode/request.d | 6 +++--- danode/router.d | 2 +- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/danode/interfaces.d b/danode/interfaces.d index 4434a7c..b13f3db 100644 --- a/danode/interfaces.d +++ b/danode/interfaces.d @@ -81,7 +81,7 @@ abstract class DriverInterface { final @property string header() const { return(fullheader(inbuffer.data)); } // Byte input converted to body as string - final @property string body() const { + final @property string content() const { if (bodyStart < 0 || bodyStart > inbuffer.data.length) return(""); return(to!string(inbuffer.data[bodyStart() .. $])); } diff --git a/danode/multipart.d b/danode/multipart.d index 4448bbc..07041e8 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -143,20 +143,20 @@ unittest { // Helper to build a multipart body string buildMultipart(string boundary, string[2][] textFields, string[3][] fileFields) { - string body; + string mp; foreach (f; textFields) { - body ~= "--" ~ boundary ~ "\r\n"; - body ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"\r\n\r\n"; - body ~= f[1] ~ "\r\n"; + mp ~= "--" ~ boundary ~ "\r\n"; + mp ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"\r\n\r\n"; + mp ~= f[1] ~ "\r\n"; } foreach (f; fileFields) { - body ~= "--" ~ boundary ~ "\r\n"; - body ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"; filename=\"" ~ f[1] ~ "\"\r\n"; - body ~= "Content-Type: application/octet-stream\r\n\r\n"; - body ~= f[2] ~ "\r\n"; + mp ~= "--" ~ boundary ~ "\r\n"; + mp ~= "Content-Disposition: form-data; name=\"" ~ f[0] ~ "\"; filename=\"" ~ f[1] ~ "\"\r\n"; + mp ~= "Content-Type: application/octet-stream\r\n\r\n"; + mp ~= f[2] ~ "\r\n"; } - body ~= "--" ~ boundary ~ "--\r\n"; - return body; + mp ~= "--" ~ boundary ~ "--\r\n"; + return mp; } FileSystem fs = new FileSystem("./www/"); @@ -168,8 +168,8 @@ unittest { Request r; r.id = md5UUID("test1"); auto parser = MultipartParser("--" ~ boundary, uploadDir); - string body = buildMultipart(boundary, [["name", "danny"]], []); - bool result = parser.feed(r, body); + string mp = buildMultipart(boundary, [["name", "danny"]], []); + bool result = parser.feed(r, mp); assert(result, "single text field must complete"); } @@ -178,8 +178,8 @@ unittest { Request r; r.id = md5UUID("test2"); auto parser = MultipartParser("--" ~ boundary, uploadDir); - string body = buildMultipart(boundary, [], [["file", "test.txt", "hello world"]]); - assert(parser.feed(r, body), "single file must complete"); + string mp = buildMultipart(boundary, [], [["file", "test.txt", "hello world"]]); + assert(parser.feed(r, mp), "single file must complete"); assert("file" in r.postinfo, "file must be in postinfo"); assert(r.postinfo["file"].type == PostType.File, "must be File type"); assert(r.postinfo["file"].filename == "test.txt", "filename must match"); @@ -193,8 +193,8 @@ unittest { Request r; r.id = md5UUID("test3"); auto parser = MultipartParser("--" ~ boundary, uploadDir); - string body = buildMultipart(boundary, [["name", "danny"]], [["file", "data.bin", "binarydata"]]); - assert(parser.feed(r, body), "mixed must complete"); + string mp = buildMultipart(boundary, [["name", "danny"]], [["file", "data.bin", "binarydata"]]); + assert(parser.feed(r, mp), "mixed must complete"); assert(r.postinfo["name"].value == "danny", "text field must parse"); assert(r.postinfo["file"].type == PostType.File, "file field must parse"); if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); @@ -205,10 +205,10 @@ unittest { Request r; r.id = md5UUID("test4"); auto parser = MultipartParser("--" ~ boundary, uploadDir); - string body = buildMultipart(boundary, [["field", "value"]], []); + string mp = buildMultipart(boundary, [["field", "value"]], []); bool done = false; - foreach (i; 0 .. body.length) { - done = parser.feed(r, body[i..i+1]); + foreach (i; 0 .. mp.length) { + done = parser.feed(r, mp[i..i+1]); if (done) break; } assert(done, "byte-by-byte feed must complete"); @@ -220,8 +220,8 @@ unittest { r.id = md5UUID("test5"); auto parser = MultipartParser("--" ~ boundary, uploadDir); string binaryContent = "data=with=equals\r\nand newlines\r\nmore data"; - string body = buildMultipart(boundary, [], [["bin", "binary.bin", binaryContent]]); - assert(parser.feed(r, body), "binary content must complete"); + string mp = buildMultipart(boundary, [], [["bin", "binary.bin", binaryContent]]); + assert(parser.feed(r, mp), "binary content must complete"); assert(r.postinfo["bin"].size == binaryContent.length, "binary size must match"); string written = cast(string) read(r.postinfo["bin"].value); assert(written == binaryContent, "binary content must be preserved exactly"); diff --git a/danode/post.d b/danode/post.d index 49ce787..cc4ef72 100644 --- a/danode/post.d +++ b/danode/post.d @@ -67,17 +67,16 @@ final bool parsePost(ref Request request, ref Response response, in FileSystem f } // Non-multipart: wait for full body as before - string content = request.body; - log(Level.Trace, "Post: [T] Received %s of %s", content.length, expectedlength); - if(content.length < expectedlength) return(false); + log(Level.Trace, "Post: [T] Received %s of %s", request.content.length, expectedlength); + if(request.content.length < expectedlength) return(false); if (contenttype.indexOf(XFORMHEADER) >= 0) { - request.parseXform(content); + request.parseXform(request.content); } else if (contenttype.indexOf(JSON) >= 0) { log(Level.Verbose, "JSON: [I] passing %d bytes raw to script", expectedlength); } else { error("parsePost: Unsupported POST content type: %s [%s]", contenttype, expectedlength); - request.parseXform(content); + request.parseXform(request.content); } return(request.postParsed = true); } diff --git a/danode/request.d b/danode/request.d index 7858962..e45e9bd 100644 --- a/danode/request.d +++ b/danode/request.d @@ -41,7 +41,7 @@ pure bool parseRequestLine(ref Request request, const string line) { struct Request { string ip; /// IP location of the client long port; /// Port at which the client is connected - string body; /// the body of the HTTP request + string content; /// the content of the HTTP request bool isSecure; /// was a secure request made bool isValid; /// Is the header valid ? UUID id; /// md5UUID for this request @@ -60,7 +60,7 @@ struct Request { final void initialize(const DriverInterface driver) { this.ip = driver.ip; this.port = driver.port; - this.body = driver.body; + this.content = driver.content; this.isSecure = driver.isSecure; this.starttime = Clock.currTime(); this.id = md5UUID(format("%s:%d-%s", driver.ip, driver.port, starttime)); @@ -87,7 +87,7 @@ struct Request { } // New input was obtained and / or the driver has been changed, update the driver - final void update(string body) { this.body = body; } + final void update(string content) { this.content = content; } // Parse Range header: "bytes=start-end" or "bytes=start-" final @property long[2] range() const { diff --git a/danode/router.d b/danode/router.d index 9df00a8..6c7498c 100644 --- a/danode/router.d +++ b/danode/router.d @@ -39,7 +39,7 @@ class Router { if (!response.created) { request.initialize(driver); response = request.create(this.address); - } else { request.update(driver.body); } + } else { request.update(driver.content); } return(true); } From 386f6b7e349dc91c461b5ae15efbdea76688005a Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 18:10:22 +0000 Subject: [PATCH 28/46] Minor --- danode/multipart.d | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/danode/multipart.d b/danode/multipart.d index 07041e8..de1aa98 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -162,9 +162,7 @@ unittest { FileSystem fs = new FileSystem("./www/"); string uploadDir = fs.localroot("localhost") ~ "/"; string boundary = "testboundary123"; - - // Test 1: single text field - { + { // Test 1: single text field Request r; r.id = md5UUID("test1"); auto parser = MultipartParser("--" ~ boundary, uploadDir); @@ -172,9 +170,7 @@ unittest { bool result = parser.feed(r, mp); assert(result, "single text field must complete"); } - - // Test 2: single file upload - { + { // Test 2: single file upload Request r; r.id = md5UUID("test2"); auto parser = MultipartParser("--" ~ boundary, uploadDir); @@ -187,9 +183,7 @@ unittest { // cleanup if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); } - - // Test 3: mixed text + file - { + { // Test 3: mixed text + file Request r; r.id = md5UUID("test3"); auto parser = MultipartParser("--" ~ boundary, uploadDir); @@ -199,9 +193,7 @@ unittest { assert(r.postinfo["file"].type == PostType.File, "file field must parse"); if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); } - - // Test 4: cross-chunk boundary detection - feed 1 byte at a time - { + { // Test 4: cross-chunk boundary detection - feed 1 byte at a time Request r; r.id = md5UUID("test4"); auto parser = MultipartParser("--" ~ boundary, uploadDir); @@ -213,9 +205,7 @@ unittest { } assert(done, "byte-by-byte feed must complete"); } - - // Test 5: binary file with = and \r\n in content - { + { // Test 5: binary file with = and \r\n in content Request r; r.id = md5UUID("test5"); auto parser = MultipartParser("--" ~ boundary, uploadDir); @@ -227,4 +217,4 @@ unittest { assert(written == binaryContent, "binary content must be preserved exactly"); if (r.postinfo["bin"].value.exists) remove(r.postinfo["bin"].value); } -} \ No newline at end of file +} From 8ff320b8f64cd76316005507025397e3532045ea Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 18:23:28 +0000 Subject: [PATCH 29/46] More tests --- danode/multipart.d | 59 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/danode/multipart.d b/danode/multipart.d index de1aa98..0604f6c 100644 --- a/danode/multipart.d +++ b/danode/multipart.d @@ -42,7 +42,9 @@ struct MultipartParser { case MPState.INIT: // Find opening boundary ptrdiff_t i = indexOf(data, boundary ~ "\r\n"); if (i < 0) { return(saveTail(data)); } - data = data[i + boundary.length + 2 .. $]; + ptrdiff_t next = i + boundary.length + 2; + if (next > data.length) { return(saveTail(data[i .. $])); } + data = data[next .. $]; state = MPState.HEADER; break; case MPState.HEADER: // Accumulate until \r\n\r\n @@ -217,4 +219,59 @@ unittest { assert(written == binaryContent, "binary content must be preserved exactly"); if (r.postinfo["bin"].value.exists) remove(r.postinfo["bin"].value); } + { // Test 6: header split across chunks + Request r; + r.id = md5UUID("test6"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string mp = buildMultipart(boundary, [["field", "value"]], []); + // Find the \r\n\r\n and split there + ptrdiff_t split = mp.indexOf("\r\n\r\n") + 2; // split mid-header-terminator + bool done = parser.feed(r, mp[0..split]); + if (!done) done = parser.feed(r, mp[split..$]); + assert(done, "header split across chunks must complete"); + assert(r.postinfo["field"].value == "value", "header split value must be correct"); + } + { // Test 7: empty file upload + Request r; + r.id = md5UUID("test7"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string mp = buildMultipart(boundary, [], [["file", "empty.txt", ""]]); + assert(parser.feed(r, mp), "empty file must complete"); + assert(r.postinfo["file"].size == 0, "empty file size must be 0"); + if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); + } + { // Test 8: multiple text fields + Request r; + r.id = md5UUID("test8"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string mp = buildMultipart(boundary, [["a", "1"], ["b", "2"], ["c", "3"]], []); + assert(parser.feed(r, mp), "multiple text fields must complete"); + assert(r.postinfo["a"].value == "1", "field a must be 1"); + assert(r.postinfo["b"].value == "2", "field b must be 2"); + assert(r.postinfo["c"].value == "3", "field c must be 3"); + } + { // Test 9: value containing boundary-like content + Request r; + r.id = md5UUID("test9"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string mp = buildMultipart(boundary, [["field", "--notaboundary--"]], []); + assert(parser.feed(r, mp), "boundary-like value must complete"); + assert(r.postinfo["field"].value == "--notaboundary--", "boundary-like value must be preserved"); + } + { // Test 10: large file in chunks + Request r; + r.id = md5UUID("test10"); + auto parser = MultipartParser("--" ~ boundary, uploadDir); + string fileContent = "x".replicate(1024 * 100); // 100KB + string mp = buildMultipart(boundary, [], [["file", "large.bin", fileContent]]); + // Feed in 4KB chunks + bool done = false; + for (size_t pos = 0; pos < mp.length && !done; pos += 4096) { + size_t end = min(pos + 4096, mp.length); + done = parser.feed(r, mp[pos..end]); + } + assert(done, "large file must complete"); + assert(r.postinfo["file"].size == fileContent.length, "large file size must match"); + if (r.postinfo["file"].value.exists) remove(r.postinfo["file"].value); + } } From b9eca4458684e68042b6824203d58ab59c89808e Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 18:26:25 +0000 Subject: [PATCH 30/46] Minor code inconsistency --- danode/post.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/danode/post.d b/danode/post.d index cc4ef72..7aa2039 100644 --- a/danode/post.d +++ b/danode/post.d @@ -96,7 +96,7 @@ final void parseXform(ref Request request, const string content) { This data is picked-up by the different CGI APIs, and presented to the client in the regular way */ final void serverAPI(in FileSystem filesystem, in WebConfig config, in Request request, in Response response) { Appender!(string) content; - content.put(format("S=REDIRECT_STATUS=%d\n", (response.payload)? response.payload.statuscode.code : 200)); + content.put(format("S=REDIRECT_STATUS=%d\n", (response.payload !is null)? response.payload.statuscode.code : 200)); // Give HTTP_COOKIES to CGI foreach (c; request.cookies.split("; ")) { content.put(format("C=%s\n", chomp(c)) ); } From 5a4b177ac71fbc9c7a3daf31f77d5d22e1ae32ce Mon Sep 17 00:00:00 2001 From: DannyArends Date: Sat, 21 Mar 2026 18:49:35 +0000 Subject: [PATCH 31/46] Moving file system functions into filesystem --- danode/files.d | 6 +-- danode/filesystem.d | 88 ++++++++++++++++++++++++++++----- danode/functions.d | 116 ++------------------------------------------ danode/process.d | 5 +- danode/request.d | 4 +- danode/response.d | 50 ++++++++++--------- danode/router.d | 3 +- 7 files changed, 118 insertions(+), 154 deletions(-) diff --git a/danode/files.d b/danode/files.d index 7107e76..71475b0 100644 --- a/danode/files.d +++ b/danode/files.d @@ -4,13 +4,13 @@ module danode.files; import danode.imports; import danode.statuscode : StatusCode; -import danode.mimetypes : mime; +import danode.mimetypes : mime, UNSUPPORTED_FILE; import danode.payload : Payload, PayloadType, Message; import danode.log : log, tag, error, Level; -import danode.functions : isCGI, htmltime; +import danode.functions : htmltime; import danode.request : Request; import danode.response : Response, notModified; -import danode.filesystem : FileSystem; +import danode.filesystem : FileSystem, isCGI; /* Per-client streaming wrapper around a shared FilePayload. Keeps its own File handle open for the duration of streaming, diff --git a/danode/filesystem.d b/danode/filesystem.d index efe4269..84adf45 100644 --- a/danode/filesystem.d +++ b/danode/filesystem.d @@ -6,8 +6,9 @@ import danode.imports; import danode.statuscode : StatusCode; import danode.payload : PayloadType; +import danode.mimetypes : mime, CGI_FILE, UNSUPPORTED_FILE; import danode.files : FilePayload, FileStream; -import danode.functions : has, isFILE, isDIR; +import danode.functions : has; import danode.log : log, tag, error, Level; /* Domain name structure containing files in that domain @@ -99,23 +100,88 @@ class FileSystem { } } } -/* Basic unit-tests should be extended */ +pure bool isAllowed(in string path) { return(mime(path) != UNSUPPORTED_FILE); } +bool isFILE(in string path) { try { return(isFile(path)); } catch(Exception e) { error("isFILE: I/O exception '%s'", e.msg); } return false; } +bool isDIR(in string path) { try { return(isDir(path)); } catch(Exception e) { error("isDIR: I/O exception '%s'", e.msg); } return false; } +bool isCGI(in string path) { + try { return(isFile(path) && mime(path).indexOf(CGI_FILE) >= 0); + }catch(Exception e) { error("isCGI: I/O exception '%s'", e.msg); } + return false; +} + +// Which interpreter (if any) should be used for the path ? +string interpreter(in string path) { + if (!isCGI(path)) return []; + string[] parts = mime(path).split("/"); + if(parts.length > 1) return(parts[1]); + return []; +} + +pure string resolve(string path) { return(buildNormalizedPath(absolutePath(path)).replace("\\", "/")); } + +string resolveFolder(string path) { + path = path.resolve(); + path = (path.endsWith("/"))? path : path ~ "/"; + if (!exists(path)) mkdirRecurse(path); + return(path); +} + +// Returns null if path escapes root +string safePath(in string root, in string path) { + if (path.canFind("..")) return null; + if (path.canFind("\0")) return null; + string full = root ~ (path.startsWith("/") ? path : "/" ~ path); + try { + string absroot = root.resolve(); + if (!absroot.endsWith("/")) absroot ~= "/"; + if (exists(full)) { + string resolved = full.resolve(); + if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null; + } else { + string parent = dirName(full).resolve(); + if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null; + } + } catch (Exception e) { return null; } + return full; +} + unittest { tag(Level.Always, "FILE", "%s", __FILE__); FileSystem fs = new FileSystem("./www/"); + // Local root assert(fs.localroot("localhost").length > 0, "localroot must resolve"); - + // Domains Domain localdomain = fs.scan("www/localhost"); assert(localdomain.buffersize() > 0, "buffersize must be positive"); - assert(localdomain.size() > 0, "size must be positive"); - + assert(localdomain.size() > 0, "size must be positive"); + // Files auto fp = fs.file(fs.localroot("localhost"), "/dmd.d"); auto stream = new FileStream(fp); - assert(stream.bytes(0, 6).length == 6, "FileStream must read 6 bytes"); - assert(fp.statuscode() == StatusCode.Ok, "file statuscode must be Ok"); - assert(fp.mimetype().length > 0, "file must have mimetype"); - assert(fp.type() == PayloadType.File, "type must be File"); - assert(fp.ready() > 0, "file must be ready"); + assert(stream.bytes(0, 6).length == 6, "FileStream must read 6 bytes"); + assert(fp.statuscode() == StatusCode.Ok, "file statuscode must be Ok"); + assert(fp.mimetype().length > 0, "file must have mimetype"); + assert(fp.type() == PayloadType.File, "type must be File"); + assert(fp.ready() > 0, "file must be ready"); + // isFILE / isDIR / isCGI + assert(isFILE("danode/functions.d"), "functions.d must be a file"); + assert(!isFILE("danode"), "directory must not be a file"); + assert(isDIR("danode"), "danode must be a directory"); + assert(!isDIR("danode/functions.d"), "file must not be a directory"); + assert(isCGI("www/localhost/dmd.d"), "dmd.d must be CGI"); + assert(!isCGI("www/localhost/test.txt"),"txt must not be CGI"); + // interpreter + assert(interpreter("www/localhost/dmd.d").length > 0, "dmd.d must have interpreter"); + assert(interpreter("www/localhost/php.php").length > 0, "php must have interpreter"); + assert(interpreter("www/localhost/test.txt").length == 0,"txt must have no interpreter"); + // safePath - security critical + assert(safePath("www/localhost", "/../etc/passwd") is null, "path traversal .. must be blocked"); + assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked"); + assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed"); + assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed"); + assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed"); + // isAllowed / isAllowedFile + assert(isAllowed("test.html"), "html must be allowed"); + assert(isAllowed("test.txt"), "txt must be allowed"); + assert(!isAllowed("test.ill"), "unknown extension must be blocked"); } - diff --git a/danode/functions.d b/danode/functions.d index 6fef4aa..1036a47 100644 --- a/danode/functions.d +++ b/danode/functions.d @@ -5,7 +5,6 @@ module danode.functions; import danode.imports; import danode.log : log, tag, error, Level; -import danode.mimetypes : CGI_FILE, mime, UNSUPPORTED_FILE; immutable string[int] months; shared static this(){ @@ -34,37 +33,6 @@ pure string htmlEscape(string s) nothrow { return(s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'")); } -pure string resolve(string path) { return(buildNormalizedPath(absolutePath(path)).replace("\\", "/")); } - -string resolveFolder(string path) { - path = path.resolve(); - path = (path.endsWith("/"))? path : path ~ "/"; - if (!exists(path)) mkdirRecurse(path); - return(path); -} - -void safeClose(ref File f) nothrow { try { if (f.isOpen()) { f.close(); } } catch(Exception e) {} } -void safeRemove(string path) nothrow { try { if (exists(path)) { remove(path); } } catch(Exception e) {} } - -// Returns null if path escapes root -string safePath(in string root, in string path) { - if (path.canFind("..")) return null; - if (path.canFind("\0")) return null; - string full = root ~ (path.startsWith("/") ? path : "/" ~ path); - try { - string absroot = root.resolve(); - if (!absroot.endsWith("/")) absroot ~= "/"; - if (exists(full)) { - string resolved = full.resolve(); - if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null; - } else { - string parent = dirName(full).resolve(); - if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null; - } - } catch (Exception e) { return null; } - return full; -} - // Month to index of the year @nogc pure int monthToIndex(in string m) nothrow { for (int x = 1; x <= 12; ++x) { if(icmp(m, months[x]) == 0) return x; } @@ -114,23 +82,6 @@ string htmltime(in SysTime d = Clock.currTime()) { return format("%s %s %s %02d:%02d:%02d GMT", utc.day(), months[utc.month()], utc.year(), utc.hour(), utc.minute(), utc.second()); } -bool isFILE(in string path) { - try { return(isFile(path)); } catch(Exception e) { error("isFILE: I/O exception '%s'", e.msg); } return false; -} - -bool isDIR(in string path) { - try { return(isDir(path)); } catch(Exception e) { error("isDIR: I/O exception '%s'", e.msg); } - return false; -} - -bool isCGI(in string path) { - try { return(isFile(path) && mime(path).indexOf(CGI_FILE) >= 0); } - catch(Exception e) { error("isCGI: I/O exception '%s'", e.msg); } - return false; -} - -pure bool isAllowed(in string path) { return(mime(path) != UNSUPPORTED_FILE); } - // Where does the HTTP request header end ? @nogc pure ptrdiff_t endofheader(T)(const(T) buffer) nothrow { ptrdiff_t len = buffer.length; @@ -155,27 +106,6 @@ pure string fullheader(T)(const(T) buffer) { return []; } -// Which interpreter (if any) should be used for the path ? -string interpreter(in string path) { - if (!isCGI(path)) return []; - string[] parts = mime(path).split("/"); - if(parts.length > 1) return(parts[1]); - return []; -} - -// Browse the content of a directory, generate a rudimentairy HTML file -string browseDir(in string root, in string localpath) { - Appender!(string) content; - content.put(format("Content of: %s
\n", htmlEscape(localpath))); - foreach (DirEntry d; dirEntries(localpath, SpanMode.shallow)) { - string name = d.name[root.length .. $].replace("\\", "/"); - if (name.endsWith(".in") || name.endsWith(".up")) continue; - string escaped = htmlEscape(name); - content.put(format("%s
", escaped, escaped)); - } - return(format("200 - Allowed directory%s", content.data)); -} - // Reset the socketset and add a server socket to the set int sISelect(SocketSet set, Socket socket, bool write = false, int timeout = 25) { set.reset(); @@ -191,72 +121,34 @@ unittest { assert(monthToIndex("Jan") == 1, "Jan must be month 1"); assert(monthToIndex("Dec") == 12, "Dec must be month 12"); assert(monthToIndex("xyz") == -1, "invalid month must return -1"); - // htmltime assert(htmltime().length > 0, "htmltime must return non-empty string"); - - // isFILE / isDIR / isCGI - assert(isFILE("danode/functions.d"), "functions.d must be a file"); - assert(!isFILE("danode"), "directory must not be a file"); - assert(isDIR("danode"), "danode must be a directory"); - assert(!isDIR("danode/functions.d"), "file must not be a directory"); - assert(isCGI("www/localhost/dmd.d"), "dmd.d must be CGI"); - assert(!isCGI("www/localhost/test.txt"),"txt must not be CGI"); - - // interpreter - assert(interpreter("www/localhost/dmd.d").length > 0, "dmd.d must have interpreter"); - assert(interpreter("www/localhost/php.php").length > 0, "php must have interpreter"); - assert(interpreter("www/localhost/test.txt").length == 0,"txt must have no interpreter"); - - // safePath - security critical - assert(safePath("www/localhost", "/../etc/passwd") is null, "path traversal .. must be blocked"); - assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked"); - assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed"); - assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed"); - assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed"); - // htmlEscape - XSS critical assert(htmlEscape("