Development Updates

README.md

@@ -10,13 +10,13 @@ DaNode is a web server written in the [D programming language](https://dlang.org
 ### Main features
 
 - Host websites in **ANY** language that writes to *stdout*
-- HTTPS via [OpenSSL](https://www.openssl.org/): SNI, Modern TLS (1.2+), [ACME](https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment) based auto-renewing certificates
-- Minimal footprint: Code, CPU, and RAM
+- HTTPS: SNI, TLS 1.2+, ACME auto-renewing certificates via [OpenSSL](https://www.openssl.org/)
+- Minimal footprint — Code, CPU, and RAM
+- Static file serving: ETag, gzip, range requests, SSE, keep-alive, conditional GET
+- Streaming multipart uploads — large files written directly to disk
+- Per-domain configuration: CGI control, redirects, directory access
+- Per-IP rate limiting and configurable request/upload size limits
 - Native APIs for PHP, Python, D, R — or [add your own](api/)
-- [Range request](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Range_requests) support for video/audio streaming
-- [Example sites](www/localhost/) in PHP, Perl, D, R, Ada, brainfuck
-- Per-domain configuration via `web.config` (CGI, redirects, directory access control)
-- HTTP keep-alive, conditional GET (`If-Modified-Since`), and per-IP rate limiting
 
 ### Get DaNode
 
@@ -95,20 +95,34 @@ connections to port 80 (and 443, when using the ssl version), then start the web
 The content of the [./sh/run](sh/run) shell script:
 
 ```
-nohup authbind danode/server -k -b 100 -v 2 > server.log 2>&1 &
+nohup authbind danode/server -b 100 -v 0 > server.log 2>&1 &
 ```
 
-This starts the server, does not allow for keyboard command (-k) has a backlog (-b) 
-of 100 simultaneous connection (per port), and produces more log output (-v 2).
+This starts the server with a backlog (-b) of 100 simultaneous connection (per port), and produces 
+less log output (-v 0).
 
 ```
---port      -p       HTTP port to listen on (integer)
---backlog   -b       Backlog of clients supported simultaneously per port (integer)
---keyoff    -k       Keyboard input via STDIN (boolean)
---certDir            Location of folder with SSL certificates (string)
---keyFile            Server private key location (string)
---wwwRoot            Server www root folder holding website domains (string)
---verbose   -v       Verbose level, logs on STDOUT (integer)
+--port      -p      # HTTP port to listen on (integer)
+--backlog   -b      # Backlog of clients supported simultaneously per port (integer)
+--certDir           # Location of folder with SSL certificates (string)
+--keyFile           # Server private key location (string)
+--wwwRoot           # Server www root folder holding website domains (string)
+--verbose   -v      # Verbose level, logs on STDOUT (integer)
+```
+
+### server.config
+
+Place a `server.config` file in your `wwwRoot` folder to tune server behaviour:
+
+```
+max_header_size   = 32768       # Max HTTP header size in bytes (default 32KB)
+max_request_size  = 2097152     # Max POST body size in bytes (default 2MB)
+max_upload_size   = 104857600   # Max multipart upload size in bytes (default 100MB)
+max_cgi_output    = 10485760    # Max CGI output size in bytes (default 10MB)
+cgi_timeout       = 4500        # CGI script timeout in ms (default 4500ms)
+max_sse_time      = 60000       # Max SSE connection lifetime in ms (default 60s)
+pool_size         = 200         # Worker thread pool size (default 200)
+serverinfo        = DaNode/1.0  # Server header string
 ```
 
 ### Example websites

danode/acme.d

@@ -8,7 +8,7 @@ version(SSL) {
 
   import danode.log : log, error, Level;
   import danode.ssl : loadSSL, generateKey;
-  import danode.functions : writeFile, isFILE;
+  import danode.filesystem : isFILE, writeFile;
   import danode.webconfig : serverConfig;
 
   immutable string ACME_DIR_PROD = "https://acme-v02.api.letsencrypt.org/directory";

danode/cgi.d

@@ -5,7 +5,7 @@ module danode.cgi;
 import danode.imports;
 
 import danode.functions : bodystart, endofheader, fullheader;
-import danode.log : log, tag, error, Level;
+import danode.log : tag, error, Level;
 import danode.process : Process;
 import danode.statuscode : StatusCode;
 import danode.payload : HeaderType, Payload, PayloadType;
@@ -28,7 +28,7 @@ class CGI : Payload {
     final @property PayloadType type() const { return(PayloadType.Script); }
 
     // Ready to start sending ?
-    final @property long ready() { 
+    final @property bool ready() { 
       if (external.finished) return true;
       if (mimetype == "text/event-stream") return (endOfHeader > 0);
       return false;
@@ -60,16 +60,15 @@ class CGI : Payload {
     final T getHeader(T)(string key, T def = T.init) const {
       if (endOfHeader > 0) {
         foreach (line; fullHeader().split("\n")) {
-          string[] elems = line.split(": ");
-          if (elems.length == 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(elems[1]))); }
+          string[] elems = line.split(":");
+          if (elems.length >= 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(join(elems[1 .. $], ":")))); }
         }
       }
       return(def);
     }
 
-    private const(char)[] rawOutput() const {
-      return cast(const(char)[]) external.output(0);
-    }
+    private const(char)[] rawOutput() const { return cast(const(char)[]) external.output(0); }
+    void joinThread() { external.join(); }
 
     // Type of header returned by the script: FastCGI, HTTP10, HTTP11
     @property final HeaderType headerType() const {
@@ -78,7 +77,6 @@ class CGI : Payload {
       if (values.length >= 3 && values[0] == "HTTP/1.0") return HeaderType.HTTP10;
       if (values.length >= 3 && values[0] == "HTTP/1.1") return HeaderType.HTTP11;
       if (getHeader("Status", "") != "") return HeaderType.FastCGI;
-      //if (getHeader("Content-Type", "") != "") return HeaderType.FastCGI;
       return HeaderType.None;
     }
 
@@ -140,7 +138,9 @@ class CGI : Payload {
 unittest {
   import danode.router : Router, runRequest;
   import danode.interfaces : StringDriver;
+  import danode.signals : registerExitHandler;
 
+  registerExitHandler();
   tag(Level.Always, "FILE", "%s", __FILE__);
 
   auto router = new Router("./www/", Address.init);

danode/client.d

@@ -12,6 +12,7 @@ import danode.response : Response;
 import danode.request : Request;
 import danode.log : log, tag, Level;
 import danode.webconfig : serverConfig;
+import danode.signals : shutdownSignal;
 
 class Client {
   private:
@@ -28,26 +29,24 @@ class Client {
     }
 
    final void run() {
+      Request request;
+      Response response;
       log(Level.Trace, "New connection established %s:%d", ip(), port() );
+      scope (exit) {
+        if (driver.socketReady()) driver.closeConnection();   // Close connection
+        request.clearUploadFiles();                           // Clean uploaded files
+        response.kill();                                      // Kill any running CGI process
+      }
       try {
         if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; }
-        Request request;
-        Response response;
-        scope (exit) {
-          if (driver.socketReady()) driver.closeConnection();   // Close connection
-          request.clearUploadFiles();                           // Clean uploaded files
-          response.kill();                                      // kill any running CGI process
-        }
-        size_t headerLimit  = serverConfig.get("max_header_size", 32 * 1024);
-        size_t uploadLimit  = serverConfig.get("max_upload_size",  100 * 1024 * 1024);
-        size_t requestLimit = serverConfig.get("max_request_size", 2   * 1024 * 1024);
+        size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024);
         while (running) {
-          if (driver.receive(driver.socket) > 0) { // We've received new data
+          if (driver.receive() > 0) { // We've received new data
             if (!driver.hasHeader()) {
               if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; }
             } else {
               if (driver.endOfHeader > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; }
-              size_t limit = (driver.header.indexOf("multipart/") >= 0)? uploadLimit: requestLimit;
+              size_t limit = (driver.header.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize;
               if (driver.inbuffer.data.length > limit) { driver.sendPayloadTooLarge(response); stop(); continue; }
             }
             // Parse the data and try to create a response (Could fail multiple times)
@@ -84,17 +83,18 @@ class Client {
     }
 
     void logConnection(in Request rq, in Response rs) {
+      if(rq.starttime == SysTime.init) return;
       string uri;
       try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; }
-      long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0);
-      int code = cast(int)(rs.payload ? rs.statuscode.code : 0);
+      long bytes = (rs.payload !is null && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0);
+      int code = cast(int)((rs.payload !is null)? rs.statuscode.code : 0);
       long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime);
       tag(Level.Always, format("%d", code),
           "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), ip, port, rq.shorthost, uri.replace("%", "%%"), requests, bytes/1024f, ms);
     }
 
     // Is the client still running, if the socket was gone it's not otherwise check the terminated flag
-    final @property bool running() const { return(!atomicLoad(terminated) && driver.socketReady()); }
+    final @property bool running() const { return(!atomicLoad(terminated) && !atomicLoad(shutdownSignal) && driver.socketReady()); }
 
     // Stop the client by setting the terminated flag
     final void stop() {