Development

danode/acme.d

@@ -8,9 +8,10 @@ version(SSL) {
 
   import danode.log : log, error, Level;
   import danode.ssl : loadSSL, generateKey;
-  import danode.functions : writeFile;
+  import danode.functions : writeFile, isFILE;
+  import danode.webconfig : serverConfig;
 
-  immutable string ACME_DIR_PROD    = "https://acme-v02.api.letsencrypt.org/directory";
+  immutable string ACME_DIR_PROD = "https://acme-v02.api.letsencrypt.org/directory";
   immutable string ACME_DIR_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory";
 
   __gshared string[string] acmeChallenges; // Shared challenge store: token -> keyAuthorization
@@ -79,7 +80,7 @@ version(SSL) {
 
     // Check cert expiry and renew if < 30 days remaining
   void checkAndRenew(string certDir = ".ssl/", string keyFile = ".ssl/server.key", string accountKey = ".ssl/account.key", bool staging = false) {
-    if (!exists(accountKey) || !isFile(accountKey)) { accountKey.generateKey(); }
+    if (!isFILE(accountKey)) { accountKey.generateKey(); }
     new Thread({
       try {
         log(Level.Always, "checkAndRenew called on '%s' with key '%s'", certDir, accountKey);
@@ -89,7 +90,7 @@ version(SSL) {
           string chainPath = certDir ~ domain ~ ".chain";
 
           if (!exists(chainPath)) { log(Level.Always, "ACME: no chain found for %s, bootstrapping", domain);
-            if (renewCert(domain, "Danny.Arends@gmail.com", d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); }
+            if (renewCert(domain, serverConfig.get("user_email", ""), d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); }
             continue;
           }
 
@@ -105,7 +106,7 @@ version(SSL) {
 
           log(Level.Verbose, "ACME: chain %s expires in %d days", domain, days);
           if (days < 30) { log(Level.Verbose, "ACME: renewing chain for %s", domain);
-            if (renewCert(domain, "Danny.Arends@gmail.com", d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); }
+            if (renewCert(domain, serverConfig.get("user_email", ""), d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); }
           }
         }
       }

danode/cgi.d

@@ -18,9 +18,9 @@ class CGI : Payload {
   public:
     string command;
 
-    this(string[] command, string path, string[string] environ, bool removeInput = true, long maxtime = 4500){
+    this(string[] command, string path, string[string] environ, bool removeInput = true){
       this.command = command.join(" ");
-      external = new Process(command, path, environ, removeInput, maxtime); 
+      external = new Process(command, path, environ, removeInput);
       external.start();
     }
 

danode/client.d

@@ -1,26 +1,19 @@
-/** danode/client.d - Per-connection thread: request/response lifecycle, keep-alive, timeouts
+/** danode/client.d - Per-connection handler: request/response lifecycle, keep-alive, timeouts
   * License: GPLv3 (https://github.com/DannyArends/DaNode) - Danny Arends **/
 module danode.client;
 
 import danode.imports;
 
-import danode.cgi : CGI;
 import danode.statuscode : StatusCode;
 import danode.functions: htmltime, Msecs;
-import danode.interfaces : DriverInterface, ClientInterface, StringDriver;
+import danode.interfaces : DriverInterface, StringDriver, sendHeaderTooLarge, sendPayloadTooLarge, sendTimedOut;
 import danode.router : Router, runRequest;
-import danode.response : Response, setPayload;
+import danode.response : Response;
 import danode.request : Request;
-import danode.payload : PayloadType;
 import danode.log : log, tag, Level;
+import danode.webconfig : serverConfig;
 
-immutable size_t MAX_HEADER_SIZE  = 1024 * 32;          //  32KB Header
-immutable size_t MAX_REQUEST_SIZE = 1024 * 1024 * 2;    //   2MB Body
-immutable size_t MAX_UPLOAD_SIZE  = 1024 * 1024 * 100;  // 100MB Multipart uploads
-immutable size_t MAX_SSE_TIME = 60_000;                 // 60 seconds max SSE lifetime
-
-
-class Client : Thread, ClientInterface {
+class Client {
   private:
     Router              router;              /// Router class from server
     DriverInterface     driver;              /// Driver
@@ -29,46 +22,47 @@ class Client : Thread, ClientInterface {
 
   public:
     this(Router router, DriverInterface driver, long maxtime = 5000) {
-      log(Level.Trace, "client constructor");
       this.router = router;
       this.driver = driver;
       this.maxtime = maxtime;
-      super(&run); // initialize the thread
     }
 
    final void run() {
       log(Level.Trace, "New connection established %s:%d", ip(), port() );
       try {
-        if (driver.openConnection() == false) { log(Level.Verbose, "WARN: Unable to open connection"); stop(); }
+        if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; }
         Request request;
         Response response;
         scope (exit) {
           if (driver.socketReady()) driver.closeConnection();   // Close connection
           request.clearUploadFiles();                           // Clean uploaded files
           response.kill();                                      // kill any running CGI process
         }
+        size_t headerLimit  = serverConfig.get("max_header_size", 32 * 1024);
+        size_t uploadLimit  = serverConfig.get("max_upload_size",  100 * 1024 * 1024);
+        size_t requestLimit = serverConfig.get("max_request_size", 2   * 1024 * 1024);
         while (running) {
           if (driver.receive(driver.socket) > 0) { // We've received new data
             if (!driver.hasHeader()) {
-              if (driver.inbuffer.data.length > MAX_HEADER_SIZE) { driver.setHeaderTooLarge(response); stop(); continue; }
+              if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; }
             } else {
-              if (driver.endOfHeader > MAX_HEADER_SIZE) { driver.setHeaderTooLarge(response); stop(); continue; }
-              size_t limit = (driver.header.indexOf("multipart/") >= 0) ? MAX_UPLOAD_SIZE : MAX_REQUEST_SIZE;
-              if (driver.inbuffer.data.length > limit) { driver.setPayloadTooLarge(response); stop(); continue; }
+              if (driver.endOfHeader > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; }
+              size_t limit = (driver.header.indexOf("multipart/") >= 0)? uploadLimit: requestLimit;
+              if (driver.inbuffer.data.length > limit) { driver.sendPayloadTooLarge(response); stop(); continue; }
             }
             // Parse the data and try to create a response (Could fail multiple times)
-            if (!response.ready) { router.route(driver, request, response, maxtime); }
+            if (!response.ready) { router.route(driver, request, response); }
           }
           if (response.ready && !response.completed) {      // We know what to respond, but haven't send all of it yet
             driver.send(response, driver.socket);           // Send the response, hit multiple times, send what you can and return
             if (response.isSSE) {
               if (response.scriptCompleted) { response.completed = true; stop(); continue; }
-              if (starttime >= MAX_SSE_TIME) { log(Level.Verbose, "SSE max lifetime reached"); stop(); continue; }
+              if (starttime >= serverConfig.get("max_sse_time", 60_000)) { log(Level.Verbose, "SSE max lifetime reached"); stop(); continue; }
             }
           }
           if (response.ready && response.completed) {       // We've completed the request, response cycle
             driver.requests++;
-            if(response.keepalive) { this.log(request, response); }
+            if(response.keepalive) { logConnection(request, response); }
             request.clearUploadFiles();                     // Clean uploaded files
             driver.inbuffer.clear();                        // Clear the input buffer
             if(!response.keepalive){ stop(); continue; }    // No keep alive, then stop this client
@@ -77,69 +71,44 @@ class Client : Thread, ClientInterface {
           }
           if (lastmodified >= maxtime) { // Client are not allowed to be silent for more than maxtime
             log(Level.Trace, "inactivity: %s > %s", lastmodified, maxtime);
-            driver.setTimedOut(response);
+            driver.sendTimedOut(response);
             stop(); continue;
           }
           log(Level.Trace, "Connection %s:%s (%s msecs) %s", ip, port, starttime, to!string(driver.inbuffer.data));
         }
-        this.log(request, response);
+        logConnection(request, response);
       } catch(Exception e) { log(Level.Verbose, "Unknown Client Exception: %s", e); stop();
       } catch(Error e) { log(Level.Verbose, "Unknown Client Error: %s", e); stop(); }
-
       log(Level.Verbose, "Connection %s:%s (%s) closed. %d requests %s (%s msecs)", ip, port, (driver.isSecure() ? "SSL" : "HTTP"), 
                                                                                       driver.requests, driver.senddata, starttime);
     }
 
+    void logConnection(in Request rq, in Response rs) {
+      string uri;
+      try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; }
+      long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0);
+      int code = cast(int)(rs.payload ? rs.statuscode.code : 0);
+      long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime);
+      tag(Level.Always, format("%d", code),
+          "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), ip, port, rq.shorthost, uri.replace("%", "%%"), requests, bytes/1024f, ms);
+    }
+
     // Is the client still running, if the socket was gone it's not otherwise check the terminated flag
     final @property bool running() const { return(!atomicLoad(terminated) && driver.socketReady()); }
 
     // Stop the client by setting the terminated flag
-    final @property void stop() {
-      log(Level.Trace, "connection %s:%s stop called", ip, port);
+    final void stop() {
+      log(Level.Trace, "Connection %s:%s stop called", ip, port);
       atomicStore(terminated, true);
     }
 
-    // Number of requests served
     final @property long requests() const { return(driver ? driver.requests : 0); }
-    // Start time of the client in mseconds (stored in the connection driver)
     final @property long starttime() const { return(driver.starttime); }
-    // When was the client last modified in mseconds (stored in the connection driver)
     final @property long lastmodified() const { return(driver.lastmodified); }
-    // Port of the client
     final @property long port() const { return(driver.port()); } 
-    // ip address of the client
     final @property string ip() const { return(driver.ip()); } 
 }
 
-void log(in ClientInterface cl, in Request rq, in Response rs) {
-  string uri;
-  try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; }
-  long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0);
-  int code = cast(int)(rs.payload ? rs.statuscode.code : 0);
-  long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime);
-  tag(Level.Always, format("%d", code),
-      "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), cl.ip, cl.port, rq.shorthost, uri.replace("%", "%%"), cl.requests, bytes/1024f, ms);
-}
-
-// serve a 408 connection timed out page
-void setTimedOut(ref DriverInterface driver, ref Response response) {
-  if(response.payload && response.payload.type == PayloadType.Script){ to!CGI(response.payload).notifyovertime(); }
-  response.setPayload(StatusCode.TimedOut, "408 - Connection Timed Out\n", "text/plain");
-  driver.send(response, driver.socket);
-}
-
-// serve a 431 request header fields too large page
-void setHeaderTooLarge(ref DriverInterface driver, ref Response response) {
-  response.setPayload(StatusCode.HeaderFieldsTooLarge, "431 - Request Header Fields Too Large\n", "text/plain");
-  driver.send(response, driver.socket);
-}
-
-// serve a 413 payload too large page
-void setPayloadTooLarge(ref DriverInterface driver, ref Response response) {
-  response.setPayload(StatusCode.PayloadTooLarge, "413 - Payload Too Large\n", "text/plain");
-  driver.send(response, driver.socket);
-}
-
 unittest {
   tag(Level.Always, "FILE", "%s", __FILE__);
 

danode/files.d

@@ -106,40 +106,6 @@ class FilePayload : Payload {
       return(buffered = true);
     } }
 
-    /* Whole file content served via the bytes function */
-    final @property string content(){ return( to!string(bytes(0, length)) ); }
-    /* Is the file a real file (i.e. does it exist on disk) */
-    final @property bool realfile() const { return(path.exists()); }
-    /* Do we have a gzip encoded version */
-    final @property bool hasEncodedVersion() const { return(encbuf !is null); }
-    /* Is the file defined as static in mimetypes.d ? */
-    final @property bool isStaticFile() { return(!path.isCGI()); }
-    /* Time the file was last modified ? */
-    final @property SysTime mtime() const { if(!realfile){ return btime; } return path.timeLastModified(); }
-    /* Files are always assumed ready to be handled (unlike Common Gate Way threads)  */
-    final @property long ready() { return(true); }
-    /* Payload type delivered to the client  */
-    final @property PayloadType type() const { return(PayloadType.File); }
-    /* Size of the file, -1 if it does not exist  */
-    final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); }
-    /* Length of the buffer  */
-    final @property long buffersize() const { return cast(long)(buf.length); }
-    /* Mimetype of the file  */
-    final @property string mimetype() const { return mime(path); }
-    /* Buffer status of the file  */
-    final @property bool isBuffered() const { return buffered; }
-    /* Path of the file  */
-    final @property string filePath() const { return path; }
-    /* Status code for file is StatusCode.Ok ? */
-    final @property StatusCode statuscode() const { 
-      return realfile ? StatusCode.Ok : StatusCode.NotFound; 
-    }
-    /* Get the number of bytes that the client response has, based on encoding */
-    final @property ptrdiff_t length() const {
-      if(hasEncodedVersion && gzip) return(encbuf.length);
-      return(fileSize());
-    }
-
     /* Get bytes in a lockfree manner from the correct underlying buffer */
     final const(char)[] bytes(ptrdiff_t from, ptrdiff_t maxsize = 4096, bool isRange = false, long start = 0, long end = -1) { synchronized {
       if (!realfile) { return []; }
@@ -154,6 +120,21 @@ class FilePayload : Payload {
       log(Level.Verbose, "FilePayload.bytes() called on unbuffered file '%s', this should not happen", path);
       return([]);
     } }
+
+    final @property string content(){ return( to!string(bytes(0, length)) ); }
+    final @property bool realfile() const { return(path.exists()); }
+    final @property bool hasEncodedVersion() const { return(encbuf !is null); }
+    final @property bool isStaticFile() { return(!path.isCGI()); }
+    final @property SysTime mtime() const { try { return path.timeLastModified(); }catch (Exception e) { return btime; } }
+    final @property long ready() { return(true); }
+    final @property PayloadType type() const { return(PayloadType.File); }
+    final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); }
+    final @property long buffersize() const { return cast(long)(buf.length); }
+    final @property string mimetype() const { return mime(path); }
+    final @property bool isBuffered() const { return buffered; }
+    final @property string filePath() const { return path; }
+    final @property StatusCode statuscode() const { return realfile ? StatusCode.Ok : StatusCode.NotFound; }
+    final @property ptrdiff_t length() const { if(hasEncodedVersion && gzip) { return(encbuf.length); } return(fileSize()); }
 }
 
 // Compute the Range

danode/filesystem.d

@@ -7,7 +7,7 @@ import danode.imports;
 import danode.statuscode : StatusCode;
 import danode.payload : PayloadType;
 import danode.files : FilePayload, FileStream;
-import danode.functions : has;
+import danode.functions : has, isFILE, isDIR;
 import danode.log : log, tag, error, Level;
 
 /* Domain name structure containing files in that domain
@@ -40,7 +40,7 @@ class FileSystem {
 
     /* Scan the whole filesystem for changes */
     final void scan(){ synchronized {
-      foreach (DirEntry d; dirEntries(root, SpanMode.shallow)){ if(d.isDir()){
+      foreach (DirEntry d; dirEntries(root, SpanMode.shallow)){ if(d.name.isDIR()){
         domains[d.name] = scan(d.name);
       } }
       // Remove domains that no longer exist on disk
@@ -50,18 +50,20 @@ class FileSystem {
     /* Scan a single folder */
     final Domain scan(string dname){ synchronized {
       Domain domain;
-      foreach (DirEntry f; dirEntries(dname, SpanMode.depth)) {
-        if (f.isFile()) {
-          string shortname = replace(f.name[dname.length .. $], "\\", "/");
-          if (shortname.endsWith(".in") || shortname.endsWith(".up")) continue;
-          log(Level.Trace, "File: '%s' as '%s'", f.name, shortname);
-          if (!domain.files.has(shortname)) {
-            domain.files[shortname] = new FilePayload(f.name, maxsize);
-            domain.entries++;
-            if(domain.files[shortname].buffer()) { domain.buffered++; }
+      try {
+        foreach (DirEntry f; dirEntries(dname, SpanMode.depth)) {
+          if (f.isFILE()) {
+            string shortname = replace(f.name[dname.length .. $], "\\", "/");
+            if (shortname.endsWith(".in") || shortname.endsWith(".up")) continue;
+            log(Level.Trace, "File: '%s' as '%s'", f.name, shortname);
+            if (!domain.files.has(shortname)) {
+              domain.files[shortname] = new FilePayload(f.name, maxsize);
+              domain.entries++;
+              if(domain.files[shortname].buffer()) { domain.buffered++; }
+            }
           }
         }
-      }
+      } catch (Exception e) { log(Level.Trace, "scan: directory iteration interrupted: %s", e.msg); }
       // Remove files that no longer exist on disk
       foreach (k; domain.files.keys) { if (!exists(dname ~ k)) { domain.files.remove(k); } }
 

danode/functions.d

@@ -49,11 +49,14 @@ string safePath(in string root, in string path) {
   if (path.canFind("\0")) return null;
   string full = root ~ (path.startsWith("/") ? path : "/" ~ path);
   try {
+    string absroot = root.resolve();
+    if (!absroot.endsWith("/")) absroot ~= "/";
     if (exists(full)) {
       string resolved = full.resolve();
-      string absroot = root.resolve();
-      if (!absroot.endsWith("/")) absroot ~= "/";
       if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null;
+    } else {
+      string parent = dirName(full).resolve();
+      if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null;
     }
   } catch (Exception e) { return null; }
   return full;
@@ -207,6 +210,7 @@ unittest {
   assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked");
   assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed");
   assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed");
+  assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed");
 
   // htmlEscape - XSS critical
   assert(htmlEscape("<script>") == "&lt;script&gt;", "< and > must be escaped");

danode/https.d

@@ -10,9 +10,8 @@ version(SSL) {
   import danode.response : Response;
   import danode.log : tag, log, error, Level;
   import danode.interfaces : DriverInterface;
-  import danode.ssl;
-
-  immutable long HANDSHAKE_TIMEOUT = 5000;  // 5 seconds
+  import danode.ssl : checkForError, contexts;
+  import danode.webconfig : serverConfig;
 
   class HTTPS : DriverInterface {
     private:
@@ -26,7 +25,7 @@ version(SSL) {
       bool performHandshake() {
         log(Level.Trace, "Performing handshake");
         int rA, rE;
-        while (starttime < HANDSHAKE_TIMEOUT) {
+        while (starttime < serverConfig.get("handshake_timeout", 5000L)) {
           rA = SSL_accept(ssl);
           if (rA == 1) return(true);    // Success
           if (rA == 0) return(false);   // Controlled failure: Not retryable