diff --git a/danode/acme.d b/danode/acme.d index e6f7b2d..7c4ae00 100644 --- a/danode/acme.d +++ b/danode/acme.d @@ -8,9 +8,10 @@ version(SSL) { import danode.log : log, error, Level; import danode.ssl : loadSSL, generateKey; - import danode.functions : writeFile; + import danode.functions : writeFile, isFILE; + import danode.webconfig : serverConfig; - immutable string ACME_DIR_PROD = "https://acme-v02.api.letsencrypt.org/directory"; + immutable string ACME_DIR_PROD = "https://acme-v02.api.letsencrypt.org/directory"; immutable string ACME_DIR_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory"; __gshared string[string] acmeChallenges; // Shared challenge store: token -> keyAuthorization @@ -79,7 +80,7 @@ version(SSL) { // Check cert expiry and renew if < 30 days remaining void checkAndRenew(string certDir = ".ssl/", string keyFile = ".ssl/server.key", string accountKey = ".ssl/account.key", bool staging = false) { - if (!exists(accountKey) || !isFile(accountKey)) { accountKey.generateKey(); } + if (!isFILE(accountKey)) { accountKey.generateKey(); } new Thread({ try { log(Level.Always, "checkAndRenew called on '%s' with key '%s'", certDir, accountKey); @@ -89,7 +90,7 @@ version(SSL) { string chainPath = certDir ~ domain ~ ".chain"; if (!exists(chainPath)) { log(Level.Always, "ACME: no chain found for %s, bootstrapping", domain); - if (renewCert(domain, "Danny.Arends@gmail.com", d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); } + if (renewCert(domain, serverConfig.get("user_email", ""), d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); } continue; } @@ -105,7 +106,7 @@ version(SSL) { log(Level.Verbose, "ACME: chain %s expires in %d days", domain, days); if (days < 30) { log(Level.Verbose, "ACME: renewing chain for %s", domain); - if (renewCert(domain, "Danny.Arends@gmail.com", d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); } + if (renewCert(domain, serverConfig.get("user_email", ""), d.name, chainPath, accountKey, staging)) { loadSSL(certDir, keyFile); } } } } diff --git a/danode/cgi.d b/danode/cgi.d index 36063cf..6e20194 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -18,9 +18,9 @@ class CGI : Payload { public: string command; - this(string[] command, string path, string[string] environ, bool removeInput = true, long maxtime = 4500){ + this(string[] command, string path, string[string] environ, bool removeInput = true){ this.command = command.join(" "); - external = new Process(command, path, environ, removeInput, maxtime); + external = new Process(command, path, environ, removeInput); external.start(); } diff --git a/danode/client.d b/danode/client.d index 99b803b..2dc0b9d 100644 --- a/danode/client.d +++ b/danode/client.d @@ -1,26 +1,19 @@ -/** danode/client.d - Per-connection thread: request/response lifecycle, keep-alive, timeouts +/** danode/client.d - Per-connection handler: request/response lifecycle, keep-alive, timeouts * License: GPLv3 (https://github.com/DannyArends/DaNode) - Danny Arends **/ module danode.client; import danode.imports; -import danode.cgi : CGI; import danode.statuscode : StatusCode; import danode.functions: htmltime, Msecs; -import danode.interfaces : DriverInterface, ClientInterface, StringDriver; +import danode.interfaces : DriverInterface, StringDriver, sendHeaderTooLarge, sendPayloadTooLarge, sendTimedOut; import danode.router : Router, runRequest; -import danode.response : Response, setPayload; +import danode.response : Response; import danode.request : Request; -import danode.payload : PayloadType; import danode.log : log, tag, Level; +import danode.webconfig : serverConfig; -immutable size_t MAX_HEADER_SIZE = 1024 * 32; // 32KB Header -immutable size_t MAX_REQUEST_SIZE = 1024 * 1024 * 2; // 2MB Body -immutable size_t MAX_UPLOAD_SIZE = 1024 * 1024 * 100; // 100MB Multipart uploads -immutable size_t MAX_SSE_TIME = 60_000; // 60 seconds max SSE lifetime - - -class Client : Thread, ClientInterface { +class Client { private: Router router; /// Router class from server DriverInterface driver; /// Driver @@ -29,17 +22,15 @@ class Client : Thread, ClientInterface { public: this(Router router, DriverInterface driver, long maxtime = 5000) { - log(Level.Trace, "client constructor"); this.router = router; this.driver = driver; this.maxtime = maxtime; - super(&run); // initialize the thread } final void run() { log(Level.Trace, "New connection established %s:%d", ip(), port() ); try { - if (driver.openConnection() == false) { log(Level.Verbose, "WARN: Unable to open connection"); stop(); } + if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; } Request request; Response response; scope (exit) { @@ -47,28 +38,31 @@ class Client : Thread, ClientInterface { request.clearUploadFiles(); // Clean uploaded files response.kill(); // kill any running CGI process } + size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); + size_t uploadLimit = serverConfig.get("max_upload_size", 100 * 1024 * 1024); + size_t requestLimit = serverConfig.get("max_request_size", 2 * 1024 * 1024); while (running) { if (driver.receive(driver.socket) > 0) { // We've received new data if (!driver.hasHeader()) { - if (driver.inbuffer.data.length > MAX_HEADER_SIZE) { driver.setHeaderTooLarge(response); stop(); continue; } + if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } } else { - if (driver.endOfHeader > MAX_HEADER_SIZE) { driver.setHeaderTooLarge(response); stop(); continue; } - size_t limit = (driver.header.indexOf("multipart/") >= 0) ? MAX_UPLOAD_SIZE : MAX_REQUEST_SIZE; - if (driver.inbuffer.data.length > limit) { driver.setPayloadTooLarge(response); stop(); continue; } + if (driver.endOfHeader > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } + size_t limit = (driver.header.indexOf("multipart/") >= 0)? uploadLimit: requestLimit; + if (driver.inbuffer.data.length > limit) { driver.sendPayloadTooLarge(response); stop(); continue; } } // Parse the data and try to create a response (Could fail multiple times) - if (!response.ready) { router.route(driver, request, response, maxtime); } + if (!response.ready) { router.route(driver, request, response); } } if (response.ready && !response.completed) { // We know what to respond, but haven't send all of it yet driver.send(response, driver.socket); // Send the response, hit multiple times, send what you can and return if (response.isSSE) { if (response.scriptCompleted) { response.completed = true; stop(); continue; } - if (starttime >= MAX_SSE_TIME) { log(Level.Verbose, "SSE max lifetime reached"); stop(); continue; } + if (starttime >= serverConfig.get("max_sse_time", 60_000)) { log(Level.Verbose, "SSE max lifetime reached"); stop(); continue; } } } if (response.ready && response.completed) { // We've completed the request, response cycle driver.requests++; - if(response.keepalive) { this.log(request, response); } + if(response.keepalive) { logConnection(request, response); } request.clearUploadFiles(); // Clean uploaded files driver.inbuffer.clear(); // Clear the input buffer if(!response.keepalive){ stop(); continue; } // No keep alive, then stop this client @@ -77,69 +71,44 @@ class Client : Thread, ClientInterface { } if (lastmodified >= maxtime) { // Client are not allowed to be silent for more than maxtime log(Level.Trace, "inactivity: %s > %s", lastmodified, maxtime); - driver.setTimedOut(response); + driver.sendTimedOut(response); stop(); continue; } log(Level.Trace, "Connection %s:%s (%s msecs) %s", ip, port, starttime, to!string(driver.inbuffer.data)); } - this.log(request, response); + logConnection(request, response); } catch(Exception e) { log(Level.Verbose, "Unknown Client Exception: %s", e); stop(); } catch(Error e) { log(Level.Verbose, "Unknown Client Error: %s", e); stop(); } - log(Level.Verbose, "Connection %s:%s (%s) closed. %d requests %s (%s msecs)", ip, port, (driver.isSecure() ? "SSL" : "HTTP"), driver.requests, driver.senddata, starttime); } + void logConnection(in Request rq, in Response rs) { + string uri; + try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } + long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); + int code = cast(int)(rs.payload ? rs.statuscode.code : 0); + long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime); + tag(Level.Always, format("%d", code), + "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), ip, port, rq.shorthost, uri.replace("%", "%%"), requests, bytes/1024f, ms); + } + // Is the client still running, if the socket was gone it's not otherwise check the terminated flag final @property bool running() const { return(!atomicLoad(terminated) && driver.socketReady()); } // Stop the client by setting the terminated flag - final @property void stop() { - log(Level.Trace, "connection %s:%s stop called", ip, port); + final void stop() { + log(Level.Trace, "Connection %s:%s stop called", ip, port); atomicStore(terminated, true); } - // Number of requests served final @property long requests() const { return(driver ? driver.requests : 0); } - // Start time of the client in mseconds (stored in the connection driver) final @property long starttime() const { return(driver.starttime); } - // When was the client last modified in mseconds (stored in the connection driver) final @property long lastmodified() const { return(driver.lastmodified); } - // Port of the client final @property long port() const { return(driver.port()); } - // ip address of the client final @property string ip() const { return(driver.ip()); } } -void log(in ClientInterface cl, in Request rq, in Response rs) { - string uri; - try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } - long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); - int code = cast(int)(rs.payload ? rs.statuscode.code : 0); - long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime); - tag(Level.Always, format("%d", code), - "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), cl.ip, cl.port, rq.shorthost, uri.replace("%", "%%"), cl.requests, bytes/1024f, ms); -} - -// serve a 408 connection timed out page -void setTimedOut(ref DriverInterface driver, ref Response response) { - if(response.payload && response.payload.type == PayloadType.Script){ to!CGI(response.payload).notifyovertime(); } - response.setPayload(StatusCode.TimedOut, "408 - Connection Timed Out\n", "text/plain"); - driver.send(response, driver.socket); -} - -// serve a 431 request header fields too large page -void setHeaderTooLarge(ref DriverInterface driver, ref Response response) { - response.setPayload(StatusCode.HeaderFieldsTooLarge, "431 - Request Header Fields Too Large\n", "text/plain"); - driver.send(response, driver.socket); -} - -// serve a 413 payload too large page -void setPayloadTooLarge(ref DriverInterface driver, ref Response response) { - response.setPayload(StatusCode.PayloadTooLarge, "413 - Payload Too Large\n", "text/plain"); - driver.send(response, driver.socket); -} - unittest { tag(Level.Always, "FILE", "%s", __FILE__); diff --git a/danode/files.d b/danode/files.d index 6c44af6..b166bea 100644 --- a/danode/files.d +++ b/danode/files.d @@ -106,40 +106,6 @@ class FilePayload : Payload { return(buffered = true); } } - /* Whole file content served via the bytes function */ - final @property string content(){ return( to!string(bytes(0, length)) ); } - /* Is the file a real file (i.e. does it exist on disk) */ - final @property bool realfile() const { return(path.exists()); } - /* Do we have a gzip encoded version */ - final @property bool hasEncodedVersion() const { return(encbuf !is null); } - /* Is the file defined as static in mimetypes.d ? */ - final @property bool isStaticFile() { return(!path.isCGI()); } - /* Time the file was last modified ? */ - final @property SysTime mtime() const { if(!realfile){ return btime; } return path.timeLastModified(); } - /* Files are always assumed ready to be handled (unlike Common Gate Way threads) */ - final @property long ready() { return(true); } - /* Payload type delivered to the client */ - final @property PayloadType type() const { return(PayloadType.File); } - /* Size of the file, -1 if it does not exist */ - final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); } - /* Length of the buffer */ - final @property long buffersize() const { return cast(long)(buf.length); } - /* Mimetype of the file */ - final @property string mimetype() const { return mime(path); } - /* Buffer status of the file */ - final @property bool isBuffered() const { return buffered; } - /* Path of the file */ - final @property string filePath() const { return path; } - /* Status code for file is StatusCode.Ok ? */ - final @property StatusCode statuscode() const { - return realfile ? StatusCode.Ok : StatusCode.NotFound; - } - /* Get the number of bytes that the client response has, based on encoding */ - final @property ptrdiff_t length() const { - if(hasEncodedVersion && gzip) return(encbuf.length); - return(fileSize()); - } - /* Get bytes in a lockfree manner from the correct underlying buffer */ final const(char)[] bytes(ptrdiff_t from, ptrdiff_t maxsize = 4096, bool isRange = false, long start = 0, long end = -1) { synchronized { if (!realfile) { return []; } @@ -154,6 +120,21 @@ class FilePayload : Payload { log(Level.Verbose, "FilePayload.bytes() called on unbuffered file '%s', this should not happen", path); return([]); } } + + final @property string content(){ return( to!string(bytes(0, length)) ); } + final @property bool realfile() const { return(path.exists()); } + final @property bool hasEncodedVersion() const { return(encbuf !is null); } + final @property bool isStaticFile() { return(!path.isCGI()); } + final @property SysTime mtime() const { try { return path.timeLastModified(); }catch (Exception e) { return btime; } } + final @property long ready() { return(true); } + final @property PayloadType type() const { return(PayloadType.File); } + final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); } + final @property long buffersize() const { return cast(long)(buf.length); } + final @property string mimetype() const { return mime(path); } + final @property bool isBuffered() const { return buffered; } + final @property string filePath() const { return path; } + final @property StatusCode statuscode() const { return realfile ? StatusCode.Ok : StatusCode.NotFound; } + final @property ptrdiff_t length() const { if(hasEncodedVersion && gzip) { return(encbuf.length); } return(fileSize()); } } // Compute the Range diff --git a/danode/filesystem.d b/danode/filesystem.d index b80a1a1..004c82c 100644 --- a/danode/filesystem.d +++ b/danode/filesystem.d @@ -7,7 +7,7 @@ import danode.imports; import danode.statuscode : StatusCode; import danode.payload : PayloadType; import danode.files : FilePayload, FileStream; -import danode.functions : has; +import danode.functions : has, isFILE, isDIR; import danode.log : log, tag, error, Level; /* Domain name structure containing files in that domain @@ -40,7 +40,7 @@ class FileSystem { /* Scan the whole filesystem for changes */ final void scan(){ synchronized { - foreach (DirEntry d; dirEntries(root, SpanMode.shallow)){ if(d.isDir()){ + foreach (DirEntry d; dirEntries(root, SpanMode.shallow)){ if(d.name.isDIR()){ domains[d.name] = scan(d.name); } } // Remove domains that no longer exist on disk @@ -50,18 +50,20 @@ class FileSystem { /* Scan a single folder */ final Domain scan(string dname){ synchronized { Domain domain; - foreach (DirEntry f; dirEntries(dname, SpanMode.depth)) { - if (f.isFile()) { - string shortname = replace(f.name[dname.length .. $], "\\", "/"); - if (shortname.endsWith(".in") || shortname.endsWith(".up")) continue; - log(Level.Trace, "File: '%s' as '%s'", f.name, shortname); - if (!domain.files.has(shortname)) { - domain.files[shortname] = new FilePayload(f.name, maxsize); - domain.entries++; - if(domain.files[shortname].buffer()) { domain.buffered++; } + try { + foreach (DirEntry f; dirEntries(dname, SpanMode.depth)) { + if (f.isFILE()) { + string shortname = replace(f.name[dname.length .. $], "\\", "/"); + if (shortname.endsWith(".in") || shortname.endsWith(".up")) continue; + log(Level.Trace, "File: '%s' as '%s'", f.name, shortname); + if (!domain.files.has(shortname)) { + domain.files[shortname] = new FilePayload(f.name, maxsize); + domain.entries++; + if(domain.files[shortname].buffer()) { domain.buffered++; } + } } } - } + } catch (Exception e) { log(Level.Trace, "scan: directory iteration interrupted: %s", e.msg); } // Remove files that no longer exist on disk foreach (k; domain.files.keys) { if (!exists(dname ~ k)) { domain.files.remove(k); } } diff --git a/danode/functions.d b/danode/functions.d index 5d2320e..07076b6 100644 --- a/danode/functions.d +++ b/danode/functions.d @@ -49,11 +49,14 @@ string safePath(in string root, in string path) { if (path.canFind("\0")) return null; string full = root ~ (path.startsWith("/") ? path : "/" ~ path); try { + string absroot = root.resolve(); + if (!absroot.endsWith("/")) absroot ~= "/"; if (exists(full)) { string resolved = full.resolve(); - string absroot = root.resolve(); - if (!absroot.endsWith("/")) absroot ~= "/"; if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null; + } else { + string parent = dirName(full).resolve(); + if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null; } } catch (Exception e) { return null; } return full; @@ -207,6 +210,7 @@ unittest { assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked"); assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed"); assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed"); + assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed"); // htmlEscape - XSS critical assert(htmlEscape("