diff --git a/README.md b/README.md index 9b68927..fc203c7 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ DaNode is a web server written in the [D programming language](https://dlang.org ### Main features - Host websites in **ANY** language that writes to *stdout* -- HTTPS via [OpenSSL](https://www.openssl.org/): SNI, Modern TLS (1.2+), [ACME](https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment) based auto-renewing certificates -- Minimal footprint: Code, CPU, and RAM +- HTTPS: SNI, TLS 1.2+, ACME auto-renewing certificates via [OpenSSL](https://www.openssl.org/) +- Minimal footprint — Code, CPU, and RAM +- Static file serving: ETag, gzip, range requests, SSE, keep-alive, conditional GET +- Streaming multipart uploads — large files written directly to disk +- Per-domain configuration: CGI control, redirects, directory access +- Per-IP rate limiting and configurable request/upload size limits - Native APIs for PHP, Python, D, R — or [add your own](api/) -- [Range request](https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/Range_requests) support for video/audio streaming -- [Example sites](www/localhost/) in PHP, Perl, D, R, Ada, brainfuck -- Per-domain configuration via `web.config` (CGI, redirects, directory access control) -- HTTP keep-alive, conditional GET (`If-Modified-Since`), and per-IP rate limiting ### Get DaNode @@ -95,20 +95,34 @@ connections to port 80 (and 443, when using the ssl version), then start the web The content of the [./sh/run](sh/run) shell script: ``` -nohup authbind danode/server -k -b 100 -v 2 > server.log 2>&1 & +nohup authbind danode/server -b 100 -v 0 > server.log 2>&1 & ``` -This starts the server, does not allow for keyboard command (-k) has a backlog (-b) -of 100 simultaneous connection (per port), and produces more log output (-v 2). +This starts the server with a backlog (-b) of 100 simultaneous connection (per port), and produces +less log output (-v 0). ``` ---port -p HTTP port to listen on (integer) ---backlog -b Backlog of clients supported simultaneously per port (integer) ---keyoff -k Keyboard input via STDIN (boolean) ---certDir Location of folder with SSL certificates (string) ---keyFile Server private key location (string) ---wwwRoot Server www root folder holding website domains (string) ---verbose -v Verbose level, logs on STDOUT (integer) +--port -p # HTTP port to listen on (integer) +--backlog -b # Backlog of clients supported simultaneously per port (integer) +--certDir # Location of folder with SSL certificates (string) +--keyFile # Server private key location (string) +--wwwRoot # Server www root folder holding website domains (string) +--verbose -v # Verbose level, logs on STDOUT (integer) +``` + +### server.config + +Place a `server.config` file in your `wwwRoot` folder to tune server behaviour: + +``` +max_header_size = 32768 # Max HTTP header size in bytes (default 32KB) +max_request_size = 2097152 # Max POST body size in bytes (default 2MB) +max_upload_size = 104857600 # Max multipart upload size in bytes (default 100MB) +max_cgi_output = 10485760 # Max CGI output size in bytes (default 10MB) +cgi_timeout = 4500 # CGI script timeout in ms (default 4500ms) +max_sse_time = 60000 # Max SSE connection lifetime in ms (default 60s) +pool_size = 200 # Worker thread pool size (default 200) +serverinfo = DaNode/1.0 # Server header string ``` ### Example websites diff --git a/danode/acme.d b/danode/acme.d index 7c4ae00..581b1a2 100644 --- a/danode/acme.d +++ b/danode/acme.d @@ -8,7 +8,7 @@ version(SSL) { import danode.log : log, error, Level; import danode.ssl : loadSSL, generateKey; - import danode.functions : writeFile, isFILE; + import danode.filesystem : isFILE, writeFile; import danode.webconfig : serverConfig; immutable string ACME_DIR_PROD = "https://acme-v02.api.letsencrypt.org/directory"; diff --git a/danode/cgi.d b/danode/cgi.d index 6e20194..a4a5d16 100644 --- a/danode/cgi.d +++ b/danode/cgi.d @@ -5,7 +5,7 @@ module danode.cgi; import danode.imports; import danode.functions : bodystart, endofheader, fullheader; -import danode.log : log, tag, error, Level; +import danode.log : tag, error, Level; import danode.process : Process; import danode.statuscode : StatusCode; import danode.payload : HeaderType, Payload, PayloadType; @@ -28,7 +28,7 @@ class CGI : Payload { final @property PayloadType type() const { return(PayloadType.Script); } // Ready to start sending ? - final @property long ready() { + final @property bool ready() { if (external.finished) return true; if (mimetype == "text/event-stream") return (endOfHeader > 0); return false; @@ -60,16 +60,15 @@ class CGI : Payload { final T getHeader(T)(string key, T def = T.init) const { if (endOfHeader > 0) { foreach (line; fullHeader().split("\n")) { - string[] elems = line.split(": "); - if (elems.length == 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(elems[1]))); } + string[] elems = line.split(":"); + if (elems.length >= 2) { if (icmp(elems[0], key) == 0) return(to!T(strip(join(elems[1 .. $], ":")))); } } } return(def); } - private const(char)[] rawOutput() const { - return cast(const(char)[]) external.output(0); - } + private const(char)[] rawOutput() const { return cast(const(char)[]) external.output(0); } + void joinThread() { external.join(); } // Type of header returned by the script: FastCGI, HTTP10, HTTP11 @property final HeaderType headerType() const { @@ -78,7 +77,6 @@ class CGI : Payload { if (values.length >= 3 && values[0] == "HTTP/1.0") return HeaderType.HTTP10; if (values.length >= 3 && values[0] == "HTTP/1.1") return HeaderType.HTTP11; if (getHeader("Status", "") != "") return HeaderType.FastCGI; - //if (getHeader("Content-Type", "") != "") return HeaderType.FastCGI; return HeaderType.None; } @@ -140,7 +138,9 @@ class CGI : Payload { unittest { import danode.router : Router, runRequest; import danode.interfaces : StringDriver; + import danode.signals : registerExitHandler; + registerExitHandler(); tag(Level.Always, "FILE", "%s", __FILE__); auto router = new Router("./www/", Address.init); diff --git a/danode/client.d b/danode/client.d index 2dc0b9d..3644d82 100644 --- a/danode/client.d +++ b/danode/client.d @@ -12,6 +12,7 @@ import danode.response : Response; import danode.request : Request; import danode.log : log, tag, Level; import danode.webconfig : serverConfig; +import danode.signals : shutdownSignal; class Client { private: @@ -28,26 +29,24 @@ class Client { } final void run() { + Request request; + Response response; log(Level.Trace, "New connection established %s:%d", ip(), port() ); + scope (exit) { + if (driver.socketReady()) driver.closeConnection(); // Close connection + request.clearUploadFiles(); // Clean uploaded files + response.kill(); // Kill any running CGI process + } try { if (!driver.openConnection()) { log(Level.Verbose, "WARN: Unable to open connection"); return; } - Request request; - Response response; - scope (exit) { - if (driver.socketReady()) driver.closeConnection(); // Close connection - request.clearUploadFiles(); // Clean uploaded files - response.kill(); // kill any running CGI process - } - size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); - size_t uploadLimit = serverConfig.get("max_upload_size", 100 * 1024 * 1024); - size_t requestLimit = serverConfig.get("max_request_size", 2 * 1024 * 1024); + size_t headerLimit = serverConfig.get("max_header_size", 32 * 1024); while (running) { - if (driver.receive(driver.socket) > 0) { // We've received new data + if (driver.receive() > 0) { // We've received new data if (!driver.hasHeader()) { if (driver.inbuffer.data.length > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } } else { if (driver.endOfHeader > headerLimit) { driver.sendHeaderTooLarge(response); stop(); continue; } - size_t limit = (driver.header.indexOf("multipart/") >= 0)? uploadLimit: requestLimit; + size_t limit = (driver.header.indexOf("multipart/") >= 0)? serverConfig.maxUploadSize : serverConfig.maxRequestSize; if (driver.inbuffer.data.length > limit) { driver.sendPayloadTooLarge(response); stop(); continue; } } // Parse the data and try to create a response (Could fail multiple times) @@ -84,17 +83,18 @@ class Client { } void logConnection(in Request rq, in Response rs) { + if(rq.starttime == SysTime.init) return; string uri; try { uri = decodeComponent(rq.uri); } catch (Exception e) { uri = rq.uri; } - long bytes = (rs.payload && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); - int code = cast(int)(rs.payload ? rs.statuscode.code : 0); + long bytes = (rs.payload !is null && rs.isRange) ? (rs.rangeEnd - rs.rangeStart + 1) : (rs.payload ? rs.payload.length : 0); + int code = cast(int)((rs.payload !is null)? rs.statuscode.code : 0); long ms = rq.starttime == SysTime.init ? -1 : Msecs(rq.starttime); tag(Level.Always, format("%d", code), "%s %s:%s %s%s [%d] %.1fkb in %s ms ", htmltime(), ip, port, rq.shorthost, uri.replace("%", "%%"), requests, bytes/1024f, ms); } // Is the client still running, if the socket was gone it's not otherwise check the terminated flag - final @property bool running() const { return(!atomicLoad(terminated) && driver.socketReady()); } + final @property bool running() const { return(!atomicLoad(terminated) && !atomicLoad(shutdownSignal) && driver.socketReady()); } // Stop the client by setting the terminated flag final void stop() { diff --git a/danode/files.d b/danode/files.d index b166bea..d5c6d25 100644 --- a/danode/files.d +++ b/danode/files.d @@ -4,156 +4,34 @@ module danode.files; import danode.imports; import danode.statuscode : StatusCode; -import danode.mimetypes : mime; -import danode.payload : Payload, PayloadType, Message; -import danode.log : log, tag, error, Level; -import danode.functions : isCGI, htmltime; +import danode.mimetypes : isCompressible; +import danode.payload : Message, FilePayload, FileStream; +import danode.log : log, tag, Level; +import danode.functions : htmltime; import danode.request : Request; -import danode.response : Response, notModified; -import danode.filesystem : FileSystem; +import danode.response : Response; +import danode.router : notModified; -/* Per-client streaming wrapper around a shared FilePayload. - Keeps its own File handle open for the duration of streaming, - avoiding repeated open/seek/close per chunk. */ -class FileStream : Payload { - private: - FilePayload payload; - File handle; +/* Set a filestream to nonblocking mode, if not Posix, use winbase.h */ +bool nonblocking(ref File file) { + version(Posix) { + import core.sys.posix.fcntl : fcntl, F_SETFL, O_NONBLOCK; - public: - this(FilePayload payload) { - this.payload = payload; - if (payload.realfile) { - try { handle = File(payload.filePath(), "rb"); } - catch (Exception e) { log(Level.Verbose, "FileStream: failed to open '%s': %s", payload.filePath(), e.msg); } - } - } + return(fcntl(fileno(file.getFP()), F_SETFL, O_NONBLOCK) != -1); + }else{ + import core.sys.windows.winbase; - final @property PayloadType type() const { return PayloadType.File; } - final @property long ready() { return payload.ready(); } - final @property ptrdiff_t length() const { return payload.length(); } - final @property SysTime mtime() { return payload.mtime(); } - final @property string mimetype() const { return payload.mimetype(); } - final @property StatusCode statuscode() const { return payload.statuscode(); } - - final const(char)[] bytes(ptrdiff_t from, ptrdiff_t maxsize = 4096, bool isRange = false, long start = 0, long end = -1) { - // If buffered, delegate to the shared in-memory buffer — no file handle needed - if (payload.isBuffered()) { return payload.bytes(from, maxsize, isRange, start, end); } - if (!handle.isOpen()) return []; - auto r = rangeCalc(from, maxsize, isRange, start, end); - if (r[0] >= payload.fileSize()) return []; - try { - char[] tmpbuf = new char[](r[1]); - handle.seek(r[0]); - return handle.rawRead!char(tmpbuf).dup; - } catch (Exception e) { log(Level.Verbose, "FileStream.bytes exception '%s': %s", payload.filePath(), e.msg); return []; } - } -} - - -/* Implementation of the Payload interface, by using an underlying file (static / deflate / cgi) */ -class FilePayload : Payload { - public: - bool gzip = false; // Is a gzip version of the file available ? - - private: - string path; // Path of the file - SysTime btime; // Time buffered - bool buffered = false; // Is buffered ? - size_t buffermaxsize; // Maximum size of the buffer - char[] buf = null; // Byte buffer of the file - char[] encbuf = null; // Encoded buffer for the file - - public: - this(string path, size_t buffermaxsize) { - this.path = path; - this.buffermaxsize = buffermaxsize; - } - - /* Does the file require to be updated before sending ? */ - final bool needsupdate() { - if (!isStaticFile()) return false; // CGI files are never buffered, since they are executed - ptrdiff_t sz = fileSize(); - if (sz > 0 && sz < buffermaxsize) { // - if (!buffered) { log(Level.Trace, "File: '%s' needs buffering", path); return true; } - if (mtime > btime) { log(Level.Trace, "File: '%s' stale record", path); return true; } - }else{ - log(Level.Verbose, "File: '%s' exceeds buffer (%.1fkb > %.1fkb)", path, sz / 1024f, buffermaxsize / 1024f); - } - return false; - } - - /* Reads the file into the internal buffer, and compress the buffer to the enc buffer - Updates the buffer time and status. - */ - final bool buffer() { synchronized { - if (!needsupdate()) return(false); // re-check under lock - ptrdiff_t sz = fileSize(); - if(buf is null) buf = new char[](sz); - buf.length = sz; - try { - auto f = File(path, "rb"); - f.rawRead(buf); - f.close(); - } catch (Exception e) { error("Exception during buffering '%s': %s", path, e.msg); return(false); } - try { - auto c = new Compress(6, HeaderFormat.gzip); - encbuf = cast(char[])(c.compress(buf)); - encbuf ~= cast(char[])(c.flush()); - } catch (Exception e) { error("Exception during compressing '%s': %s", path, e.msg); return(false); } - btime = Clock.currTime(); - log(Level.Trace, "File: '%s' buffered %.1fkb|%.1fkb", path, sz / 1024f, encbuf.length / 1024f); - return(buffered = true); - } } - - /* Get bytes in a lockfree manner from the correct underlying buffer */ - final const(char)[] bytes(ptrdiff_t from, ptrdiff_t maxsize = 4096, bool isRange = false, long start = 0, long end = -1) { synchronized { - if (!realfile) { return []; } - if (needsupdate) { buffer(); if (!buffered) { log(Level.Verbose, "FilePayload.bytes() failed to buffer '%s'", path); return([]); } } - auto r = rangeCalc(from, maxsize, isRange, start, end); - log(Level.Trace, "bytes: isRange=%s start=%d end=%d from=%d offset=%d sz=%d", isRange, start, end, from, r[0], r[1]); - if(hasEncodedVersion && gzip) { - if(r[0] < encbuf.length) return( encbuf[r[0] .. to!ptrdiff_t(min(r[0]+r[1], $))] ); - } else { - if(r[0] < buf.length) return( buf[r[0] .. to!ptrdiff_t(min(r[0]+r[1], $))] ); - } - log(Level.Verbose, "FilePayload.bytes() called on unbuffered file '%s', this should not happen", path); - return([]); - } } - - final @property string content(){ return( to!string(bytes(0, length)) ); } - final @property bool realfile() const { return(path.exists()); } - final @property bool hasEncodedVersion() const { return(encbuf !is null); } - final @property bool isStaticFile() { return(!path.isCGI()); } - final @property SysTime mtime() const { try { return path.timeLastModified(); }catch (Exception e) { return btime; } } - final @property long ready() { return(true); } - final @property PayloadType type() const { return(PayloadType.File); } - final @property ptrdiff_t fileSize() const { if(!realfile){ return -1; } return to!ptrdiff_t(path.getSize()); } - final @property long buffersize() const { return cast(long)(buf.length); } - final @property string mimetype() const { return mime(path); } - final @property bool isBuffered() const { return buffered; } - final @property string filePath() const { return path; } - final @property StatusCode statuscode() const { return realfile ? StatusCode.Ok : StatusCode.NotFound; } - final @property ptrdiff_t length() const { if(hasEncodedVersion && gzip) { return(encbuf.length); } return(fileSize()); } + auto x = PIPE_NOWAIT; + return(SetNamedPipeHandleState(file.windowsHandle(), &x, null, null) != 0); + } } -// Compute the Range -@nogc pure ptrdiff_t[2] rangeCalc(ptrdiff_t from, ptrdiff_t maxsize, bool isRange, long start, long end) nothrow { - ptrdiff_t offset = isRange ? to!ptrdiff_t(start) + from : from; - ptrdiff_t limit = isRange ? to!ptrdiff_t(end - start + 1) : -1; - ptrdiff_t sz = (limit > 0) ? to!ptrdiff_t(min(maxsize, max(0, limit - from))) : maxsize; - return [offset, sz]; -} - -// Should the file be compressed ? -bool isCompressible(string mime) { - return mime.startsWith("text/") || mime == "application/json" || mime == "application/javascript" || mime == "image/svg+xml"; -} +void safeClose(ref File f) nothrow { try { if (f.isOpen()) { f.close(); } } catch(Exception e) {} } +void safeRemove(string path) nothrow { try { if (exists(path)) { remove(path); } } catch(Exception e) {} } // Serve a static file from the disc, send encrypted when requested and available -void serveStaticFile(ref Response response, in Request request, FileSystem fs) { +void serveStaticFile(ref Response response, in Request request, FilePayload reqFile) { log(Level.Trace, "serving a static file"); - FilePayload reqFile = fs.file(fs.localroot(request.shorthost()), request.path); string etag = format(`"%s"`, md5UUID(reqFile.filePath ~ reqFile.mtime.toISOString())); // Send not modified if ETag matches (this adds the etag in the response) @@ -204,26 +82,21 @@ unittest { auto router = new Router("./www/", Address.init); StringDriver res; - // Route 1: 304 Not Modified + // 304 Not Modified res = router.runRequest("GET /index.html HTTP/1.1\nHost: localhost\nIf-Modified-Since: " ~ htmltime(Clock.currTime + 1.hours) ~ "\n\n"); assert(res.lastStatus == StatusCode.NotModified, format("Expected 304, got %d", res.lastStatus.code)); - - // Route 2: Range request + // Range request res = router.runRequest("GET /test.pdf HTTP/1.1\nHost: localhost\nRange: bytes=0-1023\n\n"); assert(res.lastStatus == StatusCode.PartialContent, format("Expected 206, got %d", res.lastStatus.code)); - - // Route 3: Gzip compression + // Gzip compression res = router.runRequest("GET /index.html HTTP/1.1\nHost: localhost\nAccept-Encoding: gzip\n\n"); assert(res.lastStatus == StatusCode.Ok, format("Expected 200, got %d", res.lastStatus.code)); assert(res.lastHeaders.get("Content-Encoding", "") == "gzip", "Expected gzip Content-Encoding header"); assert(res.lastBody.length >= 2 && res.lastBody[0] == 0x1f && res.lastBody[1] == 0x8b, "Expected gzip magic bytes 1f 8b"); - - // Route 4: ETag - not modified + // ETag - not modified res = router.runRequest("GET /index.html HTTP/1.1\nHost: localhost\n\n"); string etag = res.lastHeaders.get("ETag", ""); assert(etag.length > 0, "Expected ETag header in response"); - res = router.runRequest("GET /index.html HTTP/1.1\nHost: localhost\nIf-None-Match: " ~ etag ~ "\n\n"); assert(res.lastStatus == StatusCode.NotModified, format("Expected 304, got %d", res.lastStatus.code)); } - diff --git a/danode/filesystem.d b/danode/filesystem.d index 004c82c..8436fe5 100644 --- a/danode/filesystem.d +++ b/danode/filesystem.d @@ -5,9 +5,9 @@ module danode.filesystem; import danode.imports; import danode.statuscode : StatusCode; -import danode.payload : PayloadType; -import danode.files : FilePayload, FileStream; -import danode.functions : has, isFILE, isDIR; +import danode.payload : PayloadType, FilePayload, FileStream; +import danode.mimetypes : mime, isCGI, UNSUPPORTED_FILE; +import danode.functions : has; import danode.log : log, tag, error, Level; /* Domain name structure containing files in that domain @@ -16,9 +16,9 @@ import danode.log : log, tag, error, Level; Note 2: ./www/localhost existing is required for unit testing */ struct Domain { FilePayload[string] files; - long entries; - long buffered; + @property long entries() const { return files.length; } + @property long buffered() const { long n = 0; foreach(ref f; files.byValue) { if(f.isBuffered) n++; } return n; } @property long buffersize() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].buffersize(); } return sum; } @property long size() const { long sum = 0; foreach(ref f; files.byKey){ sum += files[f].length(); } return sum; } } @@ -58,8 +58,6 @@ class FileSystem { log(Level.Trace, "File: '%s' as '%s'", f.name, shortname); if (!domain.files.has(shortname)) { domain.files[shortname] = new FilePayload(f.name, maxsize); - domain.entries++; - if(domain.files[shortname].buffer()) { domain.buffered++; } } } } @@ -101,23 +99,93 @@ class FileSystem { } } } -/* Basic unit-tests should be extended */ +pure bool isAllowed(in string path) { return(mime(path) != UNSUPPORTED_FILE); } +bool isFILE(in string path) { try { return(isFile(path)); } catch(Exception e) { error("isFILE: I/O exception '%s'", e.msg); } return false; } +bool isDIR(in string path) { try { return(isDir(path)); } catch(Exception e) { error("isDIR: I/O exception '%s'", e.msg); } return false; } + +// Write content into a file to disk +void writeFile(in string localpath, in string content) { + try { + auto fp = File(localpath, "wb"); + fp.rawWrite(content); + fp.close(); + log(Level.Trace, "writeFile: %d bytes to: %s", content.length, localpath); + } catch(Exception e) { error("writeFile: I/O exception '%s'", e.msg); } +} + +// Which interpreter (if any) should be used for the path ? +string interpreter(in string path) { + if (!isCGI(path)) return []; + string[] parts = mime(path).split("/"); + if(parts.length > 1) return(parts[1]); + return []; +} + +pure string resolve(string path) { return(buildNormalizedPath(absolutePath(path)).replace("\\", "/")); } + +string resolveFolder(string path) { + path = path.resolve(); + path = (path.endsWith("/"))? path : path ~ "/"; + if (!exists(path)) mkdirRecurse(path); + return(path); +} + +// Returns null if path escapes root +string safePath(in string root, in string path) { + if (path.canFind("..")) return null; + if (path.canFind("\0")) return null; + string full = root ~ (path.startsWith("/") ? path : "/" ~ path); + try { + string absroot = root.resolve(); + if (!absroot.endsWith("/")) absroot ~= "/"; + if (exists(full)) { + string resolved = full.resolve(); + if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null; + } else { + string parent = dirName(full).resolve(); + if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null; + } + } catch (Exception e) { return null; } + return full; +} + unittest { tag(Level.Always, "FILE", "%s", __FILE__); FileSystem fs = new FileSystem("./www/"); + // Local root assert(fs.localroot("localhost").length > 0, "localroot must resolve"); - + // Domains Domain localdomain = fs.scan("www/localhost"); assert(localdomain.buffersize() > 0, "buffersize must be positive"); - assert(localdomain.size() > 0, "size must be positive"); - + assert(localdomain.size() > 0, "size must be positive"); + // Files auto fp = fs.file(fs.localroot("localhost"), "/dmd.d"); auto stream = new FileStream(fp); - assert(stream.bytes(0, 6).length == 6, "FileStream must read 6 bytes"); - assert(fp.statuscode() == StatusCode.Ok, "file statuscode must be Ok"); - assert(fp.mimetype().length > 0, "file must have mimetype"); - assert(fp.type() == PayloadType.File, "type must be File"); - assert(fp.ready() > 0, "file must be ready"); + assert(stream.bytes(0, 6).length == 6, "FileStream must read 6 bytes"); + assert(fp.statuscode() == StatusCode.Ok, "file statuscode must be Ok"); + assert(fp.mimetype().length > 0, "file must have mimetype"); + assert(fp.type() == PayloadType.File, "type must be File"); + assert(fp.ready() > 0, "file must be ready"); + // isFILE / isDIR / isCGI + assert(isFILE("danode/functions.d"), "functions.d must be a file"); + assert(!isFILE("danode"), "directory must not be a file"); + assert(isDIR("danode"), "danode must be a directory"); + assert(!isDIR("danode/functions.d"), "file must not be a directory"); + assert(isCGI("www/localhost/dmd.d"), "dmd.d must be CGI"); + assert(!isCGI("www/localhost/test.txt"),"txt must not be CGI"); + // interpreter + assert(interpreter("www/localhost/dmd.d").length > 0, "dmd.d must have interpreter"); + assert(interpreter("www/localhost/php.php").length > 0, "php must have interpreter"); + assert(interpreter("www/localhost/test.txt").length == 0,"txt must have no interpreter"); + // safePath - security critical + assert(safePath("www/localhost", "/../etc/passwd") is null, "path traversal .. must be blocked"); + assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked"); + assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed"); + assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed"); + assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed"); + // isAllowed / isAllowedFile + assert(isAllowed("test.html"), "html must be allowed"); + assert(isAllowed("test.txt"), "txt must be allowed"); + assert(!isAllowed("test.ill"), "unknown extension must be blocked"); } - diff --git a/danode/functions.d b/danode/functions.d index 07076b6..fd91132 100644 --- a/danode/functions.d +++ b/danode/functions.d @@ -4,8 +4,7 @@ module danode.functions; import danode.imports; -import danode.log : log, tag, error, Level; -import danode.mimetypes : CGI_FILE, mime, UNSUPPORTED_FILE; +import danode.log : tag, error, Level; immutable string[int] months; shared static this(){ @@ -16,8 +15,7 @@ shared static this(){ immutable auto htmlDateRegex = ctRegex!(r"([0-9]{1,2}) ([a-z]{1,3}) ([0-9]{4}) ([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2}) [a-z]{3}", "g"); -// Try to convert a HTML date in a string into a SysTime -// Structure that we expect: "21 Apr 2014 20:20:13 GMT" +// Try to convert a HTML date in a string into a SysTime, expected: "21 Apr 2014 20:20:13 GMT" SysTime parseHtmlDate(const string datestr) { SysTime ts = SysTime(DateTime(-7, 1, 1, 1, 0, 0)); auto m = match(datestr.toLower(), htmlDateRegex); @@ -34,34 +32,6 @@ pure string htmlEscape(string s) nothrow { return(s.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """).replace("'", "'")); } -pure string resolve(string path) { return(buildNormalizedPath(absolutePath(path)).replace("\\", "/")); } - -string resolveFolder(string path) { - path = path.resolve(); - path = (path.endsWith("/"))? path : path ~ "/"; - if (!exists(path)) mkdirRecurse(path); - return(path); -} - -// Returns null if path escapes root -string safePath(in string root, in string path) { - if (path.canFind("..")) return null; - if (path.canFind("\0")) return null; - string full = root ~ (path.startsWith("/") ? path : "/" ~ path); - try { - string absroot = root.resolve(); - if (!absroot.endsWith("/")) absroot ~= "/"; - if (exists(full)) { - string resolved = full.resolve(); - if (resolved != absroot[0..$-1] && !resolved.startsWith(absroot)) return null; - } else { - string parent = dirName(full).resolve(); - if (parent != absroot[0..$-1] && !parent.startsWith(absroot)) return null; - } - } catch (Exception e) { return null; } - return full; -} - // Month to index of the year @nogc pure int monthToIndex(in string m) nothrow { for (int x = 1; x <= 12; ++x) { if(icmp(m, months[x]) == 0) return x; } @@ -97,13 +67,10 @@ string[string] parseQueryString(const string query) { return(*p); } -void writeFile(in string localpath, in string content) { - try { - auto fp = File(localpath, "wb"); - fp.rawWrite(content); - fp.close(); - log(Level.Trace, "writeFile: %d bytes to: %s", content.length, localpath); - } catch(Exception e) { error("writeFile: I/O exception '%s'", e.msg); } +int sISelect(SocketSet set, Socket socket, bool write = false, int timeout = 25) { + set.reset(); + set.add(socket); + return(write ? Socket.select(null, set, null, dur!"msecs"(timeout)) : Socket.select(set, null, null, dur!"msecs"(timeout))); } string htmltime(in SysTime d = Clock.currTime()) { @@ -111,23 +78,13 @@ string htmltime(in SysTime d = Clock.currTime()) { return format("%s %s %s %02d:%02d:%02d GMT", utc.day(), months[utc.month()], utc.year(), utc.hour(), utc.minute(), utc.second()); } -bool isFILE(in string path) { - try { return(isFile(path)); } catch(Exception e) { error("isFILE: I/O exception '%s'", e.msg); } return false; -} - -bool isDIR(in string path) { - try { return(isDir(path)); } catch(Exception e) { error("isDIR: I/O exception '%s'", e.msg); } - return false; -} - -bool isCGI(in string path) { - try { return(isFile(path) && mime(path).indexOf(CGI_FILE) >= 0); } - catch(Exception e) { error("isCGI: I/O exception '%s'", e.msg); } - return false; +// Get the HTTP header contained in the buffer (including the \r\n\r\n) +pure string fullheader(T)(const(T) buffer) { + auto i = bodystart(buffer); + if (i > 0 && i <= buffer.length) { return(to!string(buffer[0 .. i])); } + return []; } -pure bool isAllowed(in string path) { return(mime(path) != UNSUPPORTED_FILE); } - // Where does the HTTP request header end ? @nogc pure ptrdiff_t endofheader(T)(const(T) buffer) nothrow { ptrdiff_t len = buffer.length; @@ -145,41 +102,6 @@ pure bool isAllowed(in string path) { return(mime(path) != UNSUPPORTED_FILE); } return((i + 3 < buffer.length && buffer[i] == '\r' && buffer[i+1] == '\n') ? i + 4 : i + 2); } -// get the HTTP header contained in the buffer (including the \r\n\r\n) -pure string fullheader(T)(const(T) buffer) { - auto i = bodystart(buffer); - if (i > 0 && i <= buffer.length) { return(to!string(buffer[0 .. i])); } - return []; -} - -// Which interpreter (if any) should be used for the path ? -string interpreter(in string path) { - if (!isCGI(path)) return []; - string[] parts = mime(path).split("/"); - if(parts.length > 1) return(parts[1]); - return []; -} - -// Browse the content of a directory, generate a rudimentairy HTML file -string browseDir(in string root, in string localpath) { - Appender!(string) content; - content.put(format("Content of: %s
\n", htmlEscape(localpath))); - foreach (DirEntry d; dirEntries(localpath, SpanMode.shallow)) { - string name = d.name[root.length .. $].replace("\\", "/"); - if (name.endsWith(".in") || name.endsWith(".up")) continue; - string escaped = htmlEscape(name); - content.put(format("%s
", escaped, escaped)); - } - return(format("200 - Allowed directory%s", content.data)); -} - -// Reset the socketset and add a server socket to the set -int sISelect(SocketSet set, Socket socket, bool write = false, int timeout = 25) { - set.reset(); - set.add(socket); - return(write ? Socket.select(null, set, null, dur!"msecs"(timeout)) : Socket.select(set, null, null, dur!"msecs"(timeout))); -} - unittest { tag(Level.Always, "FILE", "%s", __FILE__); @@ -188,72 +110,21 @@ unittest { assert(monthToIndex("Jan") == 1, "Jan must be month 1"); assert(monthToIndex("Dec") == 12, "Dec must be month 12"); assert(monthToIndex("xyz") == -1, "invalid month must return -1"); - // htmltime assert(htmltime().length > 0, "htmltime must return non-empty string"); - - // isFILE / isDIR / isCGI - assert(isFILE("danode/functions.d"), "functions.d must be a file"); - assert(!isFILE("danode"), "directory must not be a file"); - assert(isDIR("danode"), "danode must be a directory"); - assert(!isDIR("danode/functions.d"), "file must not be a directory"); - assert(isCGI("www/localhost/dmd.d"), "dmd.d must be CGI"); - assert(!isCGI("www/localhost/test.txt"),"txt must not be CGI"); - - // interpreter - assert(interpreter("www/localhost/dmd.d").length > 0, "dmd.d must have interpreter"); - assert(interpreter("www/localhost/php.php").length > 0, "php must have interpreter"); - assert(interpreter("www/localhost/test.txt").length == 0,"txt must have no interpreter"); - - // safePath - security critical - assert(safePath("www/localhost", "/../etc/passwd") is null, "path traversal .. must be blocked"); - assert(safePath("www/localhost", "/\0etc/passwd") is null, "null byte must be blocked"); - assert(safePath("www/localhost", "/test.txt") !is null, "valid path must be allowed"); - assert(safePath("www/localhost", "/test/1.txt") !is null, "valid subpath must be allowed"); - assert(safePath("www/localhost", "/nonexistent.txt") !is null, "non-existent valid path must be allowed"); - // htmlEscape - XSS critical assert(htmlEscape("