From ec3db8b6df417094fb3e4b6f9f3b47299397fb8f Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 16:18:54 +0000 Subject: [PATCH 01/17] Refactor HTTP client: unified support for HTTP/HTTPS and Unix sockets - Created reusable http_client module with object-oriented API - Added support for Unix sockets (unix:/path/to/socket) - Implemented connection pooling and keep-alive via resty.http - Unified live and stream modules to use single API_CLIENT instance - Added mTLS support with parsed PEM objects (with file path fallback) - Moved APPSEC_CLIENT from runtime.conf to runtime object - Added request_base() method for AppSec (uses base path only) - Enhanced path building to merge query strings from URL and request - Removed unused get_remediation_http_request helper functions - Fixed unix socket connection format (scheme=nil, host=socket_path) --- lib/crowdsec.lua | 121 +++--- lib/plugins/crowdsec/http_client.lua | 580 +++++++++++++++++++++++++++ lib/plugins/crowdsec/live.lua | 108 +++-- lib/plugins/crowdsec/stream.lua | 129 +++--- lib/plugins/crowdsec/utils.lua | 33 -- 5 files changed, 768 insertions(+), 203 deletions(-) create mode 100644 lib/plugins/crowdsec/http_client.lua diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 71fa6c3..8c93158 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -2,13 +2,12 @@ package.path = package.path .. ";./?.lua" local config = require "plugins.crowdsec.config" local iputils = require "plugins.crowdsec.iputils" -local http = require "resty.http" +local http_client = require "plugins.crowdsec.http_client" local cjson = require "cjson" local captcha = require "plugins.crowdsec.captcha" local flag = require "plugins.crowdsec.flag" local utils = require "plugins.crowdsec.utils" local ban = require "plugins.crowdsec.ban" -local url = require "plugins.crowdsec.url" local metrics = require "plugins.crowdsec.metrics" local live = require "plugins.crowdsec.live" local stream = require "plugins.crowdsec.stream" @@ -218,15 +217,31 @@ function csmod.init(configFile, userAgent) end runtime.conf["APPSEC_ENABLED"] = false + runtime.APPSEC_CLIENT = nil if runtime.conf["APPSEC_URL"] ~= "" then - local u = url.parse(runtime.conf["APPSEC_URL"]) - runtime.conf["APPSEC_ENABLED"] = true - runtime.conf["APPSEC_HOST"] = u.host - if u.port ~= nil then - runtime.conf["APPSEC_HOST"] = runtime.conf["APPSEC_HOST"] .. ":" .. u.port + -- Create HTTP client object once (URL parsed once) + local client, err = http_client.new(runtime.conf["APPSEC_URL"], { + timeouts = { + connect = runtime.conf["APPSEC_CONNECT_TIMEOUT"], + send = runtime.conf["APPSEC_SEND_TIMEOUT"], + read = runtime.conf["APPSEC_PROCESS_TIMEOUT"] + }, + ssl_verify = runtime.conf["SSL_VERIFY"] + }) + + if not client then + ngx.log(ngx.ERR, "Failed to create APPSEC HTTP client: " .. (err or "unknown")) + else + runtime.conf["APPSEC_ENABLED"] = true + runtime.APPSEC_CLIENT = client + local url_params, _ = http_client.parse_url(runtime.conf["APPSEC_URL"]) + if url_params then + local display_url = url_params.is_unix and ("unix:" .. url_params.socket_path) or + (url_params.scheme .. "://" .. url_params.host .. ":" .. url_params.port) + ngx.log(ngx.INFO, "APPSEC is enabled on '" .. display_url .. "'") + end end - ngx.log(ngx.ERR, "APPSEC is enabled on '" .. runtime.conf["APPSEC_HOST"] .. "'") end @@ -263,12 +278,14 @@ function csmod.init(configFile, userAgent) runtime.conf["API_URL"] = tmp end + -- Note: HTTP clients are created inside live:new() and stream:new() + if runtime.conf["MODE"] == "live" then ngx.log(ngx.INFO, "lua nginx bouncer enabled with live mode") - live:new() + runtime.live = live:new(runtime.conf, runtime.userAgent, REMEDIATION_API_KEY_HEADER) else ngx.log(ngx.INFO, "lua nginx bouncer enabled with stream mode") - stream:new() + runtime.stream = stream:new(runtime.conf, runtime.userAgent, REMEDIATION_API_KEY_HEADER) end return true, nil end @@ -382,28 +399,9 @@ function csmod.SetupStream() end local refreshing = stream.cache:get("refreshing") if not refreshing then - local err - if runtime.conf["USE_TLS_AUTH"] then - err = stream:stream_query_tls( - runtime.conf["API_URL"], - runtime.conf["REQUEST_TIMEOUT"], - runtime.userAgent, - runtime.conf["SSL_VERIFY"], - runtime.conf["TLS_CLIENT_CERT_PARSED"], - runtime.conf["TLS_CLIENT_KEY_PARSED"], - runtime.conf["BOUNCING_ON_TYPE"] - ) - else - err = stream:stream_query_api( - runtime.conf["API_URL"], - runtime.conf["REQUEST_TIMEOUT"], - REMEDIATION_API_KEY_HEADER, - runtime.conf["API_KEY"], - runtime.userAgent, - runtime.conf["SSL_VERIFY"], - runtime.conf["BOUNCING_ON_TYPE"] - ) - end + local err = runtime.stream:stream_query( + runtime.conf["BOUNCING_ON_TYPE"] + ) if err ~=nil then ngx.log(ngx.ERR, "Failed to query the stream: " .. err) end @@ -529,32 +527,11 @@ function csmod.allowIp(ip) -- if live mode, query lapi if runtime.conf["MODE"] == "live" then ngx.log(ngx.DEBUG, "live mode") - local ok, remediation, origin, err - if runtime.conf["USE_TLS_AUTH"] then - ok, remediation, origin, err = live:live_query_tls( - ip, - runtime.conf["API_URL"], - runtime.conf["REQUEST_TIMEOUT"], - runtime.conf["CACHE_EXPIRATION"], - runtime.userAgent, - runtime.conf["SSL_VERIFY"], - runtime.conf["TLS_CLIENT_CERT_PARSED"], - runtime.conf["TLS_CLIENT_KEY_PARSED"], - runtime.conf["BOUNCING_ON_TYPE"] - ) - else - ok, remediation, origin, err = live:live_query_api( - ip, - runtime.conf["API_URL"], - runtime.conf["REQUEST_TIMEOUT"], - runtime.conf["CACHE_EXPIRATION"], - REMEDIATION_API_KEY_HEADER, - runtime.conf['API_KEY'], - runtime.userAgent, - runtime.conf["SSL_VERIFY"], - runtime.conf["BOUNCING_ON_TYPE"] - ) - end + local ok, remediation, origin, err = runtime.live:live_query( + ip, + runtime.conf["CACHE_EXPIRATION"], + runtime.conf["BOUNCING_ON_TYPE"] + ) -- debug: wip ngx.log(ngx.DEBUG, "live_query: " .. ip .. " | " .. (ok and "not banned with" or "banned with") .. " | " .. tostring(remediation) .. " | " .. tostring(origin) .. " | " .. tostring(err)) local _, is_ipv4 = iputils.parseIPAddress(ip) @@ -573,9 +550,6 @@ function csmod.allowIp(ip) end function csmod.AppSecCheck(ip) - local httpc = http.new() - httpc:set_timeouts(runtime.conf["APPSEC_CONNECT_TIMEOUT"], runtime.conf["APPSEC_SEND_TIMEOUT"], runtime.conf["APPSEC_PROCESS_TIMEOUT"]) - local uri = ngx.var.request_uri local headers = ngx.req.get_headers() @@ -587,9 +561,6 @@ function csmod.AppSecCheck(ip) headers[APPSEC_USER_AGENT_HEADER] = ngx.var.http_user_agent headers[APPSEC_API_KEY_HEADER] = runtime.conf["API_KEY"] - -- set CrowdSec APPSEC Host - headers["host"] = runtime.conf["APPSEC_HOST"] - local ok, remediation, status_code = true, "allow", 200 if runtime.conf["APPSEC_FAILURE_ACTION"] == DENY then ok = false @@ -610,16 +581,22 @@ function csmod.AppSecCheck(ip) headers["content-length"] = nil end - local res, err = httpc:request_uri(runtime.conf["APPSEC_URL"], { + -- Use pre-created HTTP client object (URL already parsed, connection pooling handled) + if not runtime.APPSEC_CLIENT then + ngx.log(ngx.ERR, "APPSEC client not initialized") + return ok, remediation, status_code, "APPSEC client not initialized" + end + + -- AppSec expects requests at the base path from APPSEC_URL + -- The incoming request URI is already communicated via APPSEC_URI_HEADER + local res, err = runtime.APPSEC_CLIENT:request_base({ method = method, headers = headers, - body = body, - ssl_verify = runtime.conf["SSL_VERIFY"], + body = body }) - httpc:close() - if err ~= nil then - ngx.log(ngx.ERR, "Fallback because of err: " .. err) + if err ~= nil or not res then + ngx.log(ngx.ERR, "Fallback because of err: " .. (err or "unknown")) return ok, remediation, status_code, err end @@ -628,7 +605,7 @@ function csmod.AppSecCheck(ip) remediation = "allow" elseif res.status == 403 then ok = false - ngx.log(ngx.DEBUG, "Appsec body response: " .. res.body) + ngx.log(ngx.DEBUG, "Appsec body response: " .. (res.body or "")) local response = cjson.decode(res.body) remediation = response.action if response.http_status ~= nil then @@ -640,7 +617,7 @@ function csmod.AppSecCheck(ip) elseif res.status == 401 then ngx.log(ngx.ERR, "Unauthenticated request to APPSEC") else - ngx.log(ngx.ERR, "Bad request to APPSEC (" .. res.status .. "): " .. res.body) + ngx.log(ngx.ERR, "Bad request to APPSEC (" .. res.status .. "): " .. (res.body or "")) end return ok, remediation, status_code, err diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua new file mode 100644 index 0000000..524b7f5 --- /dev/null +++ b/lib/plugins/crowdsec/http_client.lua @@ -0,0 +1,580 @@ +-- HTTP Client Module +-- Provides unified HTTP client with support for: +-- - HTTP/HTTPS URLs (http://host:port/path) +-- - Unix sockets (unix:/path/to/socket) +-- - Connection pooling and keep-alive +-- - Both API key and TLS authentication +-- +-- Usage: +-- local client = http_client.new("http://example.com:8081", { +-- timeouts = {connect=1000, send=1000, read=1000}, +-- ssl_verify = true, +-- ssl_client_cert = "/path/to/cert", +-- ssl_client_priv_key = "/path/to/key" +-- }) +-- local res, err = client:request_uri("/v1/decisions?ip=1.1.1.1", {headers={...}}) + +local http = require "resty.http" +local url = require "plugins.crowdsec.url" + +local M = {} + +-- Default keep-alive settings +local DEFAULT_KEEPALIVE_TIMEOUT = 60000 -- 60 seconds +local DEFAULT_KEEPALIVE_POOL_SIZE = 10 -- 10 connections per endpoint + +-- Note: Connection pooling is handled by resty.http via set_keepalive() +-- Each Client object owns its own httpc instance + +-- Client class +local Client = {} +Client.__index = Client + +--- Parse URL and extract connection parameters +-- Supports: +-- unix:/path/to/socket +-- http://host:port/path +-- https://host:port/path +-- @param url_str string: The URL to parse +-- @return table: Parsed URL parameters with fields: +-- scheme: "unix", "http", or "https" +-- host: hostname or nil for unix +-- port: port number or nil for unix +-- path: path portion of URL +-- socket_path: socket path for unix scheme +-- is_unix: boolean indicating if this is a unix socket +-- connection_key: unique key for connection pooling +-- full_url: original URL string +function M.parse_url(url_str) + if not url_str or url_str == "" then + return nil, "URL cannot be empty" + end + + local parsed = { + full_url = url_str, + is_unix = false, + scheme = nil, + host = nil, + port = nil, + path = nil, + full_path = nil, + query = nil, + socket_path = nil, + connection_key = nil + } + + -- Check for unix socket format: unix:/path/to/socket or unix:/path/to/socket/http/path + if url_str:match("^unix:") then + parsed.scheme = "unix" + parsed.is_unix = true + -- Extract everything after "unix:" + local after_scheme = url_str:match("^unix:(.+)$") + if not after_scheme then + return nil, "Invalid unix socket URL format: " .. url_str + end + + -- For unix sockets, we need to separate socket path from HTTP path + -- Common patterns: + -- unix:/var/run/crowdsec.sock -> socket: /var/run/crowdsec.sock, path: / + -- unix:/var/run/crowdsec.sock/v1/appsec -> socket: /var/run/crowdsec.sock, path: /v1/appsec + -- unix:/tmp/sock -> socket: /tmp/sock, path: / + + -- Try to find where socket path ends and HTTP path begins + -- Heuristic: Look for patterns like /v1/, /api/, or paths that start with common HTTP path prefixes + -- If no clear separator, assume everything is the socket path + + -- Check if there's a path component that looks like an HTTP path + -- Common HTTP path patterns: /v1/, /api/, /v2/, etc. + local http_path_match = after_scheme:match("(/v%d+/.*)$") or + after_scheme:match("(/api/.*)$") or + after_scheme:match("(/[^/]+/[^/]+/.*)$") -- At least 2 path segments + + if http_path_match then + -- Extract socket path (everything before the HTTP path) + parsed.socket_path = after_scheme:sub(1, #after_scheme - #http_path_match) + parsed.path = http_path_match + else + -- No clear HTTP path, treat everything as socket path + parsed.socket_path = after_scheme + parsed.path = "/" + end + + -- Ensure socket_path starts with / if it doesn't already + if parsed.socket_path and not parsed.socket_path:match("^/") then + parsed.socket_path = "/" .. parsed.socket_path + end + + parsed.full_path = parsed.path + parsed.connection_key = "unix:" .. parsed.socket_path + return parsed + end + + -- Parse HTTP/HTTPS URL + local u = url.parse(url_str) + + if not u.scheme then + return nil, "URL must have a scheme (http://, https://, or unix:)" + end + + parsed.scheme = u.scheme:lower() + + if parsed.scheme ~= "http" and parsed.scheme ~= "https" then + return nil, "Unsupported URL scheme: " .. parsed.scheme .. " (supported: http, https, unix)" + end + + parsed.host = u.host + parsed.port = u.port + + -- Set default ports if not specified + if not parsed.port then + if parsed.scheme == "https" then + parsed.port = 443 + else + parsed.port = 80 + end + end + + parsed.path = u.path or "/" + parsed.query = u.query + parsed.full_path = parsed.path + if parsed.query and parsed.query ~= "" then + parsed.full_path = parsed.full_path .. "?" .. parsed.query + end + + -- Build connection key for pooling + parsed.connection_key = parsed.scheme .. "://" .. parsed.host .. ":" .. parsed.port + + return parsed +end + +--- Make simple HTTP request (like request_uri, for LAPI - backward compatibility) +-- This function creates a temporary client object internally +-- @param url_str string: Full URL (http://host:port/path or unix:/socket/path) +-- @param options table: Request options: +-- timeout: number (single timeout for all operations) +-- connect_timeout: number (optional, separate connect timeout) +-- send_timeout: number (optional, separate send timeout) +-- read_timeout: number (optional, separate read timeout) +-- method: string (default: "GET") +-- headers: table (HTTP headers) +-- body: string (request body) +-- ssl_verify: boolean (whether to verify SSL) +-- ssl_client_cert: string (optional, for TLS auth) +-- ssl_client_priv_key: string (optional, for TLS auth) +-- @return res: HTTP response object with .status and .body, or nil on error +-- @return err: Error message if failed +function M.request_uri(url_str, options) + options = options or {} + + -- Parse URL once to split base and path/query + local parsed, perr = M.parse_url(url_str) + if not parsed then + return nil, perr + end + + local base_url + local path_with_query = parsed.full_path or parsed.path or "/" + + if parsed.is_unix then + base_url = "unix:" .. parsed.socket_path + else + base_url = parsed.scheme .. "://" .. parsed.host .. ":" .. parsed.port + end + + -- Setup timeouts for client creation + local timeouts = {} + if options.timeout then + timeouts.connect = options.timeout + timeouts.send = options.timeout + timeouts.read = options.timeout + else + timeouts.connect = options.connect_timeout or 1000 + timeouts.send = options.send_timeout or 1000 + timeouts.read = options.read_timeout or 1000 + end + + -- Create temporary client object (reuses connection pooling internally) + local client, err = M.new(base_url, { + timeouts = timeouts, + ssl_verify = options.ssl_verify, + ssl_client_cert = options.ssl_client_cert, + ssl_client_priv_key = options.ssl_client_priv_key + }) + + if not client then + return nil, err + end + + -- Use client's request_uri method + return client:request_uri(path_with_query ~= "" and path_with_query or "/", { + method = options.method, + headers = options.headers, + body = options.body + }) +end + +--- Create a new HTTP client object +-- Parse URL once and create a reusable client object +-- @param url_str string: URL (http://host:port, https://host:port, or unix:/path) +-- @param options table: Client options: +-- timeouts: table {connect, send, read} - timeout values in ms +-- ssl_verify: boolean - whether to verify SSL certificates +-- ssl_client_cert: string - (optional) path to client certificate for mTLS +-- ssl_client_priv_key: string - (optional) path to client private key for mTLS +-- keepalive_timeout: number - (optional) keep-alive timeout in ms +-- keepalive_pool_size: number - (optional) pool size +-- @return client: HTTP client object, or nil on error +-- @return err: Error message if failed +function M.new(url_str, options) + options = options or {} + + -- Parse URL once + local url_params, err = M.parse_url(url_str) + if not url_params then + return nil, err + end + + -- Build connection key with mTLS info if applicable + local connection_key = url_params.connection_key + if options.ssl_client_cert and options.ssl_client_priv_key then + connection_key = connection_key .. "|mtls" + end + + -- Create client object with its own httpc instance + local client = setmetatable({ + url_params = url_params, + connection_key = connection_key, + timeouts = options.timeouts or {connect=1000, send=1000, read=1000}, + ssl_verify = options.ssl_verify ~= false, -- default true + ssl_client_cert = options.ssl_client_cert, + ssl_client_priv_key = options.ssl_client_priv_key, + keepalive_timeout = options.keepalive_timeout or DEFAULT_KEEPALIVE_TIMEOUT, + keepalive_pool_size = options.keepalive_pool_size or DEFAULT_KEEPALIVE_POOL_SIZE, + httpc = nil, -- HTTP client instance (created on first use) + }, Client) + + return client, nil +end + +--- Get or create HTTP client instance (internal method) +-- Each client object owns its httpc instance +-- @return httpc: HTTP client object, or nil on error +-- @return err: Error message if failed +function Client:_get_httpc() + -- Reuse existing connection if available + if self.httpc then + local connect_opts = {} + + if self.url_params.is_unix then + connect_opts.host = "localhost" + connect_opts.path = self.url_params.socket_path + connect_opts.scheme = nil + connect_opts.port = nil + else + connect_opts.scheme = self.url_params.scheme + connect_opts.host = self.url_params.host + connect_opts.port = self.url_params.port + end + + connect_opts.ssl_verify = self.ssl_verify + + if self.ssl_client_cert and self.ssl_client_priv_key then + connect_opts.ssl_client_cert = self.ssl_client_cert + connect_opts.ssl_client_priv_key = self.ssl_client_priv_key + end + + -- Try to reconnect (resty.http handles keep-alive automatically) + local ok, err = self.httpc:connect(connect_opts) + if ok then + return self.httpc, nil + end + + -- Connection failed, create new one + ngx.log(ngx.DEBUG, "Connection reuse failed, creating new: " .. (err or "unknown")) + self.httpc = nil + end + + -- Create new HTTP client instance + self.httpc = http.new() + self.httpc:set_timeouts(self.timeouts.connect, self.timeouts.send, self.timeouts.read) + + -- Connect + local connect_opts = {} + + if self.url_params.is_unix then + connect_opts.host = "localhost" + connect_opts.path = self.url_params.socket_path + connect_opts.scheme = nil + connect_opts.port = nil + else + connect_opts.scheme = self.url_params.scheme + connect_opts.host = self.url_params.host + connect_opts.port = self.url_params.port + end + + connect_opts.ssl_verify = self.ssl_verify + + if self.ssl_client_cert and self.ssl_client_priv_key then + connect_opts.ssl_client_cert = self.ssl_client_cert + connect_opts.ssl_client_priv_key = self.ssl_client_priv_key + end + + local ok, err = self.httpc:connect(connect_opts) + if not ok then + self.httpc = nil + return nil, "Failed to connect: " .. (err or "unknown") + end + + return self.httpc, nil +end + +--- Release HTTP client back to keep-alive pool (internal method) +-- Uses resty.http's built-in connection pooling via set_keepalive +-- @return ok: boolean indicating success +-- @return err: Error message if failed +function Client:_release_httpc() + if not self.httpc then + return true, nil + end + + local ok, err = self.httpc:set_keepalive(self.keepalive_timeout, self.keepalive_pool_size) + if not ok then + -- If keepalive fails, close the connection + self.httpc:close() + self.httpc = nil + return false, "Failed to set keepalive: " .. (err or "unknown") + end + + -- Note: We keep self.httpc set - resty.http will reuse it from its pool on next connect() + return true, nil +end + +--- Build request path by joining base path from URL and provided path +-- @param path string: Request path (may be absolute or relative, may include query string) +-- @return string: Normalized path including base path and merged query strings +function Client:_build_path(path) + local function normalize(p) + if not p or p == "" then + return "/" + end + -- Remove query string and fragment for path normalization + p = p:gsub("%?.*$", ""):gsub("#.*$", "") + if p:sub(1,1) ~= "/" then + return "/" .. p + end + return p + end + + local base_path = normalize(self.url_params.path or "/") + + -- Extract query strings + local path_query = "" + local url_query = self.url_params.query or "" + + if path then + local query_match = path:match("%?([^#]+)") + if query_match then + path_query = query_match + end + end + + -- If caller passes no path (or just "/"), use configured base path + query + if not path or path == "" or path == "/" then + if url_query ~= "" then + return base_path .. "?" .. url_query + end + return base_path + end + + local normalized_extra = normalize(path) + + -- If caller already provided a path with the base prefix, keep it as-is + if base_path ~= "/" and normalized_extra:sub(1, #base_path) == base_path then + -- Merge query strings + if path_query ~= "" and url_query ~= "" then + return normalized_extra .. "?" .. path_query .. "&" .. url_query + elseif path_query ~= "" then + return normalized_extra .. "?" .. path_query + elseif url_query ~= "" then + return normalized_extra .. "?" .. url_query + end + return normalized_extra + end + + -- Build final path + local final_path + if base_path == "/" then + final_path = normalized_extra + elseif base_path:sub(-1) == "/" then + final_path = base_path .. normalized_extra:sub(2) + else + final_path = base_path .. normalized_extra + end + + -- Merge query strings + if path_query ~= "" and url_query ~= "" then + return final_path .. "?" .. path_query .. "&" .. url_query + elseif path_query ~= "" then + return final_path .. "?" .. path_query + elseif url_query ~= "" then + return final_path .. "?" .. url_query + end + + return final_path +end + +--- Make HTTP request with full control +-- @param client: Client object (self) +-- @param method string: HTTP method (GET, POST, etc.) +-- @param path string: Request path +-- @param headers table: HTTP headers +-- @param body string: (optional) Request body +-- @return res: HTTP response object, or nil on error +-- @return err: Error message if failed +function Client:request(method, path, headers, body) + local httpc, err = self:_get_httpc() + if not httpc then + return nil, err + end + + -- Set Host header appropriately + if not headers then + headers = {} + end + + if self.url_params.is_unix then + if not headers["Host"] and not headers["host"] then + headers["Host"] = "localhost" + end + else + if not headers["Host"] and not headers["host"] then + local host_header = self.url_params.host + if self.url_params.port and + ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or + (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then + host_header = host_header .. ":" .. self.url_params.port + end + headers["Host"] = host_header + end + end + + local full_path = self:_build_path(path) + + -- Make request + local res, err = httpc:request({ + method = method, + path = full_path, + headers = headers, + body = body + }) + + if not res then + -- Request failed, clear httpc so we create a new one next time + self.httpc:close() + self.httpc = nil + return nil, err or "Request failed" + end + + -- Read response body + local body_str, err = res:read_body() + if err then + ngx.log(ngx.WARN, "Failed to read response body: " .. err) + else + res.body = body_str + end + + -- Return connection to keep-alive pool (resty.http handles pooling) + self:_release_httpc() + + return res, nil +end + +--- Make HTTP request using only the base path from URL (no additional path needed) +-- Useful for services like AppSec that always use the configured base path +-- @param client: Client object (self) +-- @param options table: Request options: +-- method: string (default: "GET") +-- headers: table (HTTP headers) +-- body: string (request body) +-- @return res: HTTP response object with .status and .body, or nil on error +-- @return err: Error message if failed +function Client:request_base(options) + return self:request_uri("", options) +end + +--- Make simple HTTP request (like request_uri) +-- @param client: Client object (self) +-- @param path string: Request path (can include query string) +-- @param options table: Request options: +-- method: string (default: "GET") +-- headers: table (HTTP headers) +-- body: string (request body) +-- @return res: HTTP response object with .status and .body, or nil on error +-- @return err: Error message if failed +function Client:request_uri(path, options) + options = options or {} + + -- Build full path (include base path and query if configured) + local full_path = self:_build_path(path) + + -- Get or create client + local httpc, err = self:_get_httpc() + if not httpc then + return nil, err + end + + -- Prepare headers + local headers = options.headers or {} + + -- Set Host header appropriately + if self.url_params.is_unix then + if not headers["Host"] and not headers["host"] then + headers["Host"] = "localhost" + end + else + if not headers["Host"] and not headers["host"] then + local host_header = self.url_params.host + if self.url_params.port and + ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or + (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then + host_header = host_header .. ":" .. self.url_params.port + end + headers["Host"] = host_header + end + end + + -- Remove Connection: close header if present (we want keep-alive) + headers["Connection"] = nil + headers["connection"] = nil + + -- Make request + local res, err = httpc:request({ + method = options.method or "GET", + path = full_path, + headers = headers, + body = options.body + }) + + if not res then + -- Request failed, clear httpc so we create a new one next time + self.httpc:close() + self.httpc = nil + return nil, err or "Request failed" + end + + -- Read response body + local body_str, err = res:read_body() + if err then + ngx.log(ngx.WARN, "Failed to read response body: " .. err) + res.body = "" + else + res.body = body_str + end + + -- Return connection to keep-alive pool (resty.http handles pooling) + self:_release_httpc() + + return res, nil +end + +return M diff --git a/lib/plugins/crowdsec/live.lua b/lib/plugins/crowdsec/live.lua index ec92460..9be59ee 100644 --- a/lib/plugins/crowdsec/live.lua +++ b/lib/plugins/crowdsec/live.lua @@ -1,5 +1,6 @@ local cjson = require "cjson" local utils = require "plugins.crowdsec.utils" +local http_client = require "plugins.crowdsec.http_client" local live = {} live.__index = live @@ -8,61 +9,88 @@ live.cache = ngx.shared.crowdsec_cache --- Create a new live object -- Create a new live object to query the live API --- @return live: the live object - -function live:new() - return self -end - ---- Live query the API to get the decision for the IP using API key authentication --- Query the live API to get the decision for the IP in real time --- @param ip string: the IP to query --- @param api_url string: the URL of the LAPI --- @param timeout number: the timeout of the request to lapi --- @param cache_expiration number: the expiration time of the cache +-- @param conf table: Runtime configuration table +-- @param user_agent string: User agent string -- @param api_key_header string: the authorization header to use for the lapi request --- @param api_key string: the API key to use for the lapi request --- @param user_agent string: the user agent to use for the lapi request --- @param ssl_verify boolean: whether to verify the SSL certificate or not --- @param bouncing_on_type string: the type of decision to bounce on --- @return boolean: true if the IP is allowed, false if the IP is blocked --- @return string: the type of the decision --- @return string: the origin of the decision --- @return string: the error message if any -function live:live_query_api(ip, api_url, timeout, cache_expiration, api_key_header, api_key, user_agent, ssl_verify, bouncing_on_type) - local link = api_url .. "/v1/decisions?ip=" .. ip - local res, err = utils.get_remediation_http_request(link, timeout, api_key_header, api_key, user_agent, ssl_verify) +-- @return live: the live object - if not res then - ngx.log(ngx.ERR, "failed to query LAPI " .. link .. ": ".. err) - return true, nil, nil, "request failed: ".. err +function live:new(conf, user_agent, api_key_header) + local instance = setmetatable({}, self) + instance.api_url = conf["API_URL"] + instance.api_key_header = api_key_header + instance.api_key = conf["API_KEY"] + instance.user_agent = user_agent + instance.use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil + + -- Create single HTTP client (handles mTLS if configured) + instance.API_CLIENT = nil + + if conf["API_URL"] ~= "" then + local client_options = { + timeouts = { + connect = conf["REQUEST_TIMEOUT"] or 1000, + send = conf["REQUEST_TIMEOUT"] or 1000, + read = conf["REQUEST_TIMEOUT"] or 1000 + }, + ssl_verify = conf["SSL_VERIFY"] + } + + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) + if instance.use_tls_auth then + client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] + client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] + end + + local client, err = http_client.new(conf["API_URL"], client_options) + + if client then + instance.API_CLIENT = client + else + ngx.log(ngx.WARN, "Failed to create API HTTP client: " .. (err or "unknown")) + end end - - return self:live_query_process(res, ip, cache_expiration, bouncing_on_type, link) + + return instance end ---- Live query the API to get the decision for the IP using mTLS authentication +--- Live query the API to get the decision for the IP -- Query the live API to get the decision for the IP in real time +-- Uses API key authentication if mTLS is not configured, otherwise uses mTLS -- @param ip string: the IP to query --- @param api_url string: the URL of the LAPI --- @param timeout number: the timeout of the request to lapi -- @param cache_expiration number: the expiration time of the cache --- @param user_agent string: the user agent to use for the lapi request --- @param ssl_verify boolean: whether to verify the SSL certificate or not --- @param ssl_client_cert string: path to the client certificate file --- @param ssl_client_priv_key string: path to the client private key file -- @param bouncing_on_type string: the type of decision to bounce on -- @return boolean: true if the IP is allowed, false if the IP is blocked -- @return string: the type of the decision -- @return string: the origin of the decision -- @return string: the error message if any -function live:live_query_tls(ip, api_url, timeout, cache_expiration, user_agent, ssl_verify, ssl_client_cert, ssl_client_priv_key, bouncing_on_type) - local link = api_url .. "/v1/decisions?ip=" .. ip - local res, err = utils.get_remediation_http_request_tls(link, timeout, user_agent, ssl_verify, ssl_client_cert, ssl_client_priv_key) +function live:live_query(ip, cache_expiration, bouncing_on_type) + if not self.API_CLIENT then + return true, nil, nil, "HTTP client not available" + end + + -- Build path (base path from API_URL will be prepended by request_uri) + local path = "/v1/decisions?ip=" .. ip + local link = self.api_url .. path + + -- Build headers: always include User-Agent, include API key only if not using mTLS + local headers = { + ['User-Agent'] = self.user_agent + } + + if not self.use_tls_auth then + headers[self.api_key_header] = self.api_key + end + + local res, err = self.API_CLIENT:request_uri(path, { + method = "GET", + headers = headers + }) if not res then - ngx.log(ngx.ERR, "failed to query LAPI " .. link .. ": ".. err) - return true, nil, nil, "request failed: ".. err + ngx.log(ngx.ERR, "failed to query LAPI " .. link .. ": ".. (err or "unknown")) + return true, nil, nil, "request failed: ".. (err or "unknown") end return self:live_query_process(res, ip, cache_expiration, bouncing_on_type, link) diff --git a/lib/plugins/crowdsec/stream.lua b/lib/plugins/crowdsec/stream.lua index 52f8581..9b972f1 100644 --- a/lib/plugins/crowdsec/stream.lua +++ b/lib/plugins/crowdsec/stream.lua @@ -1,6 +1,7 @@ local utils = require "plugins.crowdsec.utils" local cjson = require "cjson" local metrics = require "plugins.crowdsec.metrics" +local http_client = require "plugins.crowdsec.http_client" local stream = {} stream.__index = stream @@ -124,62 +125,65 @@ function stream:get(key) return stream.cache:get("decision_cache/" .. key) end -function stream:new() - return self -end - ---- Query the local API to get the decisions using API key authentication --- @param api_url string: the URL of the local API --- @param timeout number: the timeout for the request +--- Create a new stream object +-- Create a new stream object to query the stream API +-- @param conf table: Runtime configuration table +-- @param user_agent string: User agent string -- @param api_key_header string: the header to use for the API key --- @param api_key string: the API key to use for the request --- @param user_agent string: the user agent to use for the request --- @param ssl_verify boolean: whether to verify the SSL certificate or not --- @param bouncing_on_type string: the type of decision to bounce on -function stream:stream_query_api(api_url, timeout, api_key_header, api_key, user_agent, ssl_verify, bouncing_on_type) - -- As this function is running inside coroutine (with ngx.timer.at), - -- we need to raise error instead of returning them - - if api_url == "" then - return "No API URL defined" - end - - set_refreshing(true) - - local is_startup = stream.cache:get("startup") - ngx.log(ngx.DEBUG, "startup: " .. tostring(is_startup)) - ngx.log(ngx.DEBUG, "Stream Query API from worker : " .. tostring(ngx.worker.id()) .. " with startup "..tostring(is_startup)) - local link = api_url .. "/v1/decisions/stream?startup=" .. tostring(is_startup) - - local res, err = utils.get_remediation_http_request(link, - timeout, - api_key_header, - api_key, - user_agent, - ssl_verify) - - if not res then - set_refreshing(false) - ngx.log(ngx.ERR, "request to crowdsec lapi " .. link .. " failed: " .. err) - return "request to crowdsec lapi " .. link .. " failed: " .. err +-- @return stream: the stream object +function stream:new(conf, user_agent, api_key_header) + local instance = setmetatable({}, self) + instance.api_url = conf["API_URL"] + instance.api_key_header = api_key_header + instance.api_key = conf["API_KEY"] + instance.user_agent = user_agent + instance.use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil + + -- Create single HTTP client (handles mTLS if configured) + instance.API_CLIENT = nil + + if conf["API_URL"] ~= "" then + local client_options = { + timeouts = { + connect = conf["REQUEST_TIMEOUT"] or 1000, + send = conf["REQUEST_TIMEOUT"] or 1000, + read = conf["REQUEST_TIMEOUT"] or 1000 + }, + ssl_verify = conf["SSL_VERIFY"] + } + + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) + if instance.use_tls_auth then + client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] + client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] + end + + local client, err = http_client.new(conf["API_URL"], client_options) + + if client then + instance.API_CLIENT = client + else + ngx.log(ngx.WARN, "Failed to create API HTTP client: " .. (err or "unknown")) + end end - - return self:stream_query_process(res, bouncing_on_type) + + return instance end ---- Query the local API to get the decisions using mTLS authentication --- @param api_url string: the URL of the local API --- @param timeout number: the timeout for the request --- @param user_agent string: the user agent to use for the request --- @param ssl_verify boolean: whether to verify the SSL certificate or not --- @param ssl_client_cert string: path to the client certificate file --- @param ssl_client_priv_key string: path to the client private key file +--- Query the local API to get the decisions +-- Uses API key authentication if mTLS is not configured, otherwise uses mTLS -- @param bouncing_on_type string: the type of decision to bounce on -function stream:stream_query_tls(api_url, timeout, user_agent, ssl_verify, ssl_client_cert, ssl_client_priv_key, bouncing_on_type) +function stream:stream_query(bouncing_on_type) -- As this function is running inside coroutine (with ngx.timer.at), -- we need to raise error instead of returning them - if api_url == "" then + if not self.API_CLIENT then + return "HTTP client not available" + end + + if self.api_url == "" then return "No API URL defined" end @@ -187,20 +191,29 @@ function stream:stream_query_tls(api_url, timeout, user_agent, ssl_verify, ssl_c local is_startup = stream.cache:get("startup") ngx.log(ngx.DEBUG, "startup: " .. tostring(is_startup)) - ngx.log(ngx.DEBUG, "Stream Query TLS from worker : " .. tostring(ngx.worker.id()) .. " with startup "..tostring(is_startup)) - local link = api_url .. "/v1/decisions/stream?startup=" .. tostring(is_startup) + ngx.log(ngx.DEBUG, "Stream Query from worker : " .. tostring(ngx.worker.id()) .. " with startup "..tostring(is_startup)) + -- Build path (base path from API_URL will be prepended by request_uri) + local path = "/v1/decisions/stream?startup=" .. tostring(is_startup) + local link = self.api_url .. path + + -- Build headers: always include User-Agent, include API key only if not using mTLS + local headers = { + ['User-Agent'] = self.user_agent + } + + if not self.use_tls_auth then + headers[self.api_key_header] = self.api_key + end - local res, err = utils.get_remediation_http_request_tls(link, - timeout, - user_agent, - ssl_verify, - ssl_client_cert, - ssl_client_priv_key) + local res, err = self.API_CLIENT:request_uri(path, { + method = "GET", + headers = headers + }) if not res then set_refreshing(false) - ngx.log(ngx.ERR, "request to crowdsec lapi " .. link .. " failed: " .. err) - return "request to crowdsec lapi " .. link .. " failed: " .. err + ngx.log(ngx.ERR, "request to crowdsec lapi " .. link .. " failed: " .. (err or "unknown")) + return "request to crowdsec lapi " .. link .. " failed: " .. (err or "unknown") end return self:stream_query_process(res, bouncing_on_type) diff --git a/lib/plugins/crowdsec/utils.lua b/lib/plugins/crowdsec/utils.lua index ca6c8d9..12d75db 100644 --- a/lib/plugins/crowdsec/utils.lua +++ b/lib/plugins/crowdsec/utils.lua @@ -1,5 +1,4 @@ local iputils = require "plugins.crowdsec.iputils" -local http = require "resty.http" local M = {} @@ -94,38 +93,6 @@ function M.item_to_string(item, scope) end -function M.get_remediation_http_request_tls(link,timeout, user_agent,ssl_verify, ssl_client_cert, ssl_client_priv_key) - local httpc = http.new() - httpc:set_timeout(timeout) - local res, err = httpc:request_uri(link, { - method = "GET", - headers = { - ['Connection'] = 'close', - ['User-Agent'] = user_agent - }, - ssl_verify = ssl_verify, - ssl_client_cert = ssl_client_cert, - ssl_client_priv_key = ssl_client_priv_key - }) - httpc:close() - return res, err -end - -function M.get_remediation_http_request(link,timeout, api_key_header, api_key, user_agent,ssl_verify) - local httpc = http.new() - httpc:set_timeout(timeout) - local res, err = httpc:request_uri(link, { - method = "GET", - headers = { - ['Connection'] = 'close', - [api_key_header] = api_key, - ['User-Agent'] = user_agent - }, - ssl_verify = ssl_verify - }) - httpc:close() - return res, err -end function M.split_on_delimiter(str, delimiter) if str == nil then From f05517a800464ab82ec18adc7697d86489efa753 Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 16:29:31 +0000 Subject: [PATCH 02/17] Fix query string handling: convert query table to string - Use tostring() to convert query table from url.parse() to string - Fixes error: attempt to concatenate field 'query' (a table value) - Query table has __tostring metatable that calls buildQuery() --- lib/plugins/crowdsec/http_client.lua | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 524b7f5..43099ea 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -137,8 +137,11 @@ function M.parse_url(url_str) parsed.path = u.path or "/" parsed.query = u.query parsed.full_path = parsed.path - if parsed.query and parsed.query ~= "" then - parsed.full_path = parsed.full_path .. "?" .. parsed.query + if parsed.query then + local query_str = tostring(parsed.query) + if query_str and query_str ~= "" then + parsed.full_path = parsed.full_path .. "?" .. query_str + end end -- Build connection key for pooling @@ -369,7 +372,10 @@ function Client:_build_path(path) -- Extract query strings local path_query = "" - local url_query = self.url_params.query or "" + local url_query = "" + if self.url_params.query then + url_query = tostring(self.url_params.query) + end if path then local query_match = path:match("%?([^#]+)") From 79a533498c9a8be458010ea0f93c5f7062bc79eb Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 16:47:13 +0000 Subject: [PATCH 03/17] Make keep-alive timeout configurable with 60s default, 5s for tests --- lib/crowdsec.lua | 4 +++- lib/plugins/crowdsec/config.lua | 4 +++- lib/plugins/crowdsec/http_client.lua | 2 +- lib/plugins/crowdsec/live.lua | 4 +++- lib/plugins/crowdsec/stream.lua | 4 +++- t/conf_t/01_conf_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/02_live_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/03_live_and_ban_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/05_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/07_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/08_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/09_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/10_live_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/11_live_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/12_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/13_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/14_stream_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/16_conf_crowdsec_nginx_bouncer.conf | 1 + t/conf_t/17_live_and_tls_crowdsec_nginx_bouncer.conf | 1 + 19 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 8c93158..0fe53cc 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -227,7 +227,9 @@ function csmod.init(configFile, userAgent) send = runtime.conf["APPSEC_SEND_TIMEOUT"], read = runtime.conf["APPSEC_PROCESS_TIMEOUT"] }, - ssl_verify = runtime.conf["SSL_VERIFY"] + ssl_verify = runtime.conf["SSL_VERIFY"], + keepalive_timeout = runtime.conf["KEEPALIVE_TIMEOUT"], + keepalive_pool_size = runtime.conf["KEEPALIVE_POOL_SIZE"] }) if not client then diff --git a/lib/plugins/crowdsec/config.lua b/lib/plugins/crowdsec/config.lua index 4fdfa5d..fa8b424 100644 --- a/lib/plugins/crowdsec/config.lua +++ b/lib/plugins/crowdsec/config.lua @@ -1,7 +1,7 @@ local config = {} local valid_params = {'ENABLED', 'ENABLE_INTERNAL', 'API_URL', 'API_KEY', 'BOUNCING_ON_TYPE', 'MODE', 'SECRET_KEY', 'SITE_KEY', 'BAN_TEMPLATE_PATH' ,'CAPTCHA_TEMPLATE_PATH', 'REDIRECT_LOCATION', 'RET_CODE', 'CAPTCHA_RET_CODE', 'EXCLUDE_LOCATION', 'FALLBACK_REMEDIATION', 'CAPTCHA_PROVIDER', 'APPSEC_URL', 'APPSEC_FAILURE_ACTION', 'ALWAYS_SEND_TO_APPSEC', 'SSL_VERIFY', 'USE_TLS_AUTH', 'TLS_CLIENT_CERT', 'TLS_CLIENT_KEY'} -local valid_int_params = {'CACHE_EXPIRATION', 'CACHE_SIZE', 'REQUEST_TIMEOUT', 'UPDATE_FREQUENCY', 'CAPTCHA_EXPIRATION', 'APPSEC_CONNECT_TIMEOUT', 'APPSEC_SEND_TIMEOUT', 'APPSEC_PROCESS_TIMEOUT', 'STREAM_REQUEST_TIMEOUT'} +local valid_int_params = {'CACHE_EXPIRATION', 'CACHE_SIZE', 'REQUEST_TIMEOUT', 'UPDATE_FREQUENCY', 'CAPTCHA_EXPIRATION', 'APPSEC_CONNECT_TIMEOUT', 'APPSEC_SEND_TIMEOUT', 'APPSEC_PROCESS_TIMEOUT', 'STREAM_REQUEST_TIMEOUT', 'KEEPALIVE_TIMEOUT', 'KEEPALIVE_POOL_SIZE'} -- CACHE_SIZE is not used in the code, but as is was valid parameter for the configuration file, not removing it now local valid_bouncing_on_type_values = {'ban', 'captcha', 'all'} local valid_truefalse_values = {'false', 'true'} @@ -31,6 +31,8 @@ local default_values = { ['USE_TLS_AUTH'] = "false", ['TLS_CLIENT_CERT'] = "", ['TLS_CLIENT_KEY'] = "", + ['KEEPALIVE_TIMEOUT'] = 60000, -- 60 seconds (can be overridden, e.g., 5000 for tests) + ['KEEPALIVE_POOL_SIZE'] = 10, } diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 43099ea..d9af081 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -20,7 +20,7 @@ local url = require "plugins.crowdsec.url" local M = {} -- Default keep-alive settings -local DEFAULT_KEEPALIVE_TIMEOUT = 60000 -- 60 seconds +local DEFAULT_KEEPALIVE_TIMEOUT = 60000 -- 60 seconds (can be overridden via config) local DEFAULT_KEEPALIVE_POOL_SIZE = 10 -- 10 connections per endpoint -- Note: Connection pooling is handled by resty.http via set_keepalive() diff --git a/lib/plugins/crowdsec/live.lua b/lib/plugins/crowdsec/live.lua index 9be59ee..437c77b 100644 --- a/lib/plugins/crowdsec/live.lua +++ b/lib/plugins/crowdsec/live.lua @@ -34,7 +34,9 @@ function live:new(conf, user_agent, api_key_header) send = conf["REQUEST_TIMEOUT"] or 1000, read = conf["REQUEST_TIMEOUT"] or 1000 }, - ssl_verify = conf["SSL_VERIFY"] + ssl_verify = conf["SSL_VERIFY"], + keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] } -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) diff --git a/lib/plugins/crowdsec/stream.lua b/lib/plugins/crowdsec/stream.lua index 9b972f1..c8db678 100644 --- a/lib/plugins/crowdsec/stream.lua +++ b/lib/plugins/crowdsec/stream.lua @@ -151,7 +151,9 @@ function stream:new(conf, user_agent, api_key_header) send = conf["REQUEST_TIMEOUT"] or 1000, read = conf["REQUEST_TIMEOUT"] or 1000 }, - ssl_verify = conf["SSL_VERIFY"] + ssl_verify = conf["SSL_VERIFY"], + keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] } -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) diff --git a/t/conf_t/01_conf_crowdsec_nginx_bouncer.conf b/t/conf_t/01_conf_crowdsec_nginx_bouncer.conf index 84478dd..a9d8890 100644 --- a/t/conf_t/01_conf_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/01_conf_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/02_live_crowdsec_nginx_bouncer.conf b/t/conf_t/02_live_crowdsec_nginx_bouncer.conf index 84478dd..a9d8890 100644 --- a/t/conf_t/02_live_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/02_live_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/03_live_and_ban_crowdsec_nginx_bouncer.conf b/t/conf_t/03_live_and_ban_crowdsec_nginx_bouncer.conf index c5221a9..5fdf045 100644 --- a/t/conf_t/03_live_and_ban_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/03_live_and_ban_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/05_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/05_stream_crowdsec_nginx_bouncer.conf index b0aa53a..8cedd7f 100644 --- a/t/conf_t/05_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/05_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=stream diff --git a/t/conf_t/07_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/07_stream_crowdsec_nginx_bouncer.conf index 2c3f6ad..c120c31 100644 --- a/t/conf_t/07_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/07_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=stream diff --git a/t/conf_t/08_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/08_stream_crowdsec_nginx_bouncer.conf index 2c3f6ad..c120c31 100644 --- a/t/conf_t/08_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/08_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=stream diff --git a/t/conf_t/09_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/09_stream_crowdsec_nginx_bouncer.conf index 2c3f6ad..c120c31 100644 --- a/t/conf_t/09_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/09_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=stream diff --git a/t/conf_t/10_live_crowdsec_nginx_bouncer.conf b/t/conf_t/10_live_crowdsec_nginx_bouncer.conf index 84478dd..a9d8890 100644 --- a/t/conf_t/10_live_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/10_live_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/11_live_crowdsec_nginx_bouncer.conf b/t/conf_t/11_live_crowdsec_nginx_bouncer.conf index 418de86..becd2bc 100644 --- a/t/conf_t/11_live_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/11_live_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/12_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/12_stream_crowdsec_nginx_bouncer.conf index c2fe49c..46efe81 100644 --- a/t/conf_t/12_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/12_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=5 # live or stream MODE=stream diff --git a/t/conf_t/13_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/13_stream_crowdsec_nginx_bouncer.conf index 862d5f5..403e0d2 100644 --- a/t/conf_t/13_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/13_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=5 # live or stream MODE=stream diff --git a/t/conf_t/14_stream_crowdsec_nginx_bouncer.conf b/t/conf_t/14_stream_crowdsec_nginx_bouncer.conf index 862d5f5..403e0d2 100644 --- a/t/conf_t/14_stream_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/14_stream_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=5 # live or stream MODE=stream diff --git a/t/conf_t/16_conf_crowdsec_nginx_bouncer.conf b/t/conf_t/16_conf_crowdsec_nginx_bouncer.conf index 988a563..df0fed6 100644 --- a/t/conf_t/16_conf_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/16_conf_crowdsec_nginx_bouncer.conf @@ -7,6 +7,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live diff --git a/t/conf_t/17_live_and_tls_crowdsec_nginx_bouncer.conf b/t/conf_t/17_live_and_tls_crowdsec_nginx_bouncer.conf index 872c360..e59b38a 100644 --- a/t/conf_t/17_live_and_tls_crowdsec_nginx_bouncer.conf +++ b/t/conf_t/17_live_and_tls_crowdsec_nginx_bouncer.conf @@ -10,6 +10,7 @@ CACHE_EXPIRATION=1 BOUNCING_ON_TYPE=all FALLBACK_REMEDIATION=ban REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 UPDATE_FREQUENCY=10 # live or stream MODE=live From 8da209278d96810d8d6b4dc47db780b479ceeadd Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 17:09:04 +0000 Subject: [PATCH 04/17] Update metrics to use new HTTP client with mTLS support and fix connection reuse --- lib/crowdsec.lua | 24 +++++---- lib/plugins/crowdsec/http_client.lua | 12 ++--- lib/plugins/crowdsec/metrics.lua | 80 ++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 0fe53cc..2be6a11 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -317,22 +317,26 @@ function csmod.SetupMetrics() local first_run = runtime.cache:get("metrics_first_run") if first_run then ngx.log(ngx.DEBUG, "First run for setup metrics ") - metrics:new(runtime.userAgent) + metrics:new(runtime.userAgent, runtime.conf, REMEDIATION_API_KEY_HEADER) runtime.cache:set("metrics_first_run",false) Setup_metrics_timer() return end local started = runtime.cache:get("metrics_startup_time") if ngx.time() - started >= METRICS_PERIOD then - if runtime.conf["MODE"] == "stream" then - stream:refresh_metrics() - end - metrics:sendMetrics( - runtime.conf["API_URL"], - {['User-Agent']=runtime.userAgent,[REMEDIATION_API_KEY_HEADER]=runtime.conf["API_KEY"],["Content-Type"]="application/json"}, - runtime.conf["SSL_VERIFY"], - METRICS_PERIOD - ) + -- Don't send metrics if worker is exiting (prevents shutdown hangs) + if not ngx.worker.exiting() then + if runtime.conf["MODE"] == "stream" then + stream:refresh_metrics() + end + -- Headers are now handled internally by metrics (API key added conditionally based on mTLS) + metrics:sendMetrics( + runtime.conf["API_URL"], + {['User-Agent']=runtime.userAgent,["Content-Type"]="application/json"}, + runtime.conf["SSL_VERIFY"], + METRICS_PERIOD + ) + end local succ, err, forcible = runtime.cache:set("metrics_startup_time", ngx.time()) -- to make sure we have only one thread sending metrics if not succ then ngx.log(ngx.ERR, "failed to add metrics_startup_time key in cache: "..err) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index d9af081..8db36f5 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -19,10 +19,6 @@ local url = require "plugins.crowdsec.url" local M = {} --- Default keep-alive settings -local DEFAULT_KEEPALIVE_TIMEOUT = 60000 -- 60 seconds (can be overridden via config) -local DEFAULT_KEEPALIVE_POOL_SIZE = 10 -- 10 connections per endpoint - -- Note: Connection pooling is handled by resty.http via set_keepalive() -- Each Client object owns its own httpc instance @@ -251,8 +247,8 @@ function M.new(url_str, options) ssl_verify = options.ssl_verify ~= false, -- default true ssl_client_cert = options.ssl_client_cert, ssl_client_priv_key = options.ssl_client_priv_key, - keepalive_timeout = options.keepalive_timeout or DEFAULT_KEEPALIVE_TIMEOUT, - keepalive_pool_size = options.keepalive_pool_size or DEFAULT_KEEPALIVE_POOL_SIZE, + keepalive_timeout = options.keepalive_timeout, + keepalive_pool_size = options.keepalive_pool_size, httpc = nil, -- HTTP client instance (created on first use) }, Client) @@ -348,7 +344,9 @@ function Client:_release_httpc() return false, "Failed to set keepalive: " .. (err or "unknown") end - -- Note: We keep self.httpc set - resty.http will reuse it from its pool on next connect() + -- After set_keepalive(), the httpc object is in a "closed" state but the connection + -- is in resty.http's pool. We keep the reference - on next request, connect() will + -- reuse the connection from the pool automatically. return true, nil end diff --git a/lib/plugins/crowdsec/metrics.lua b/lib/plugins/crowdsec/metrics.lua index e7782d9..7460cb4 100644 --- a/lib/plugins/crowdsec/metrics.lua +++ b/lib/plugins/crowdsec/metrics.lua @@ -1,5 +1,5 @@ local cjson = require "cjson" -local http = require "resty.http" +local http_client = require "plugins.crowdsec.http_client" local utils = require "plugins.crowdsec.utils" local osinfo = require "plugins.crowdsec.osinfo" local metrics = {} @@ -9,7 +9,7 @@ metrics.cache = ngx.shared.crowdsec_cache -- Constructor for the store -function metrics:new(userAgent) +function metrics:new(userAgent, conf, api_key_header) local info = osinfo.get_os_info() self.cache:set("metrics_data", cjson.encode({ version = userAgent, @@ -21,6 +21,49 @@ function metrics:new(userAgent) name="nginx bouncer", utc_startup_timestamp = ngx.time(), })) + + -- Create HTTP client for metrics (with mTLS support if configured) + self.metrics_client = nil + self.use_tls_auth = false + self.api_key_header = nil + self.api_key = nil + + if conf and conf["API_URL"] and conf["API_URL"] ~= "" then + -- Check if mTLS is enabled + self.use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil + + -- Store API key info for non-mTLS requests + if not self.use_tls_auth then + self.api_key_header = api_key_header or "X-Api-Key" + self.api_key = conf["API_KEY"] + end + + local client_options = { + timeouts = { + connect = conf["REQUEST_TIMEOUT"] or 1000, + send = conf["REQUEST_TIMEOUT"] or 1000, + read = conf["REQUEST_TIMEOUT"] or 1000 + }, + ssl_verify = conf["SSL_VERIFY"], + keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] + } + + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) + if self.use_tls_auth then + client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] + client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] + end + + local client, err = http_client.new(conf["API_URL"], client_options) + if client then + self.metrics_client = client + else + ngx.log(ngx.WARN, "Failed to create metrics HTTP client: " .. (err or "unknown")) + end + end end @@ -151,23 +194,40 @@ end function metrics:sendMetrics(link, headers, ssl, window) local body = self:toJson(window) .. "\n" - ngx.log(ngx.DEBUG, "Sending metrics to " .. link .. "/v1/usage-metrics") + ngx.log(ngx.DEBUG, "Sending metrics to /v1/usage-metrics") ngx.log(ngx.DEBUG, "metrics: " .. body) - local httpc = http.new() - local res, err = httpc:request_uri(link .. "/v1/usage-metrics", { - body = body, + + -- Use HTTP client if available (created during initialization) + if not self.metrics_client then + ngx.log(ngx.ERR, "metrics HTTP client not initialized, cannot send metrics") + return + end + + -- Build headers (conditionally add API key if not using mTLS) + local request_headers = {} + if headers then + for k, v in pairs(headers) do + request_headers[k] = v + end + end + + -- Only add API key header if not using mTLS + if not self.use_tls_auth and self.api_key_header and self.api_key then + request_headers[self.api_key_header] = self.api_key + end + + local res, err = self.metrics_client:request_uri("/v1/usage-metrics", { method = "POST", - headers = headers, - ssl_verify = ssl + headers = request_headers, + body = body }) - httpc:close() + if not res then ngx.log(ngx.ERR, "failed to send metrics: ", err) else ngx.log(ngx.DEBUG, "metrics status: " .. res.status) ngx.log(ngx.DEBUG, "metrics body: " .. body) end - end -- Function to retrieve all keys that start with a given prefix From b58fb2f82a1a6f587e675d70c233ca6d242066d4 Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 17:37:28 +0000 Subject: [PATCH 05/17] Fix resty.http connect error by explicitly setting nil for Unix sockets and always creating new httpc --- lib/plugins/crowdsec/http_client.lua | 42 ++++++---------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 8db36f5..4ca1453 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -260,38 +260,10 @@ end -- @return httpc: HTTP client object, or nil on error -- @return err: Error message if failed function Client:_get_httpc() - -- Reuse existing connection if available - if self.httpc then - local connect_opts = {} - - if self.url_params.is_unix then - connect_opts.host = "localhost" - connect_opts.path = self.url_params.socket_path - connect_opts.scheme = nil - connect_opts.port = nil - else - connect_opts.scheme = self.url_params.scheme - connect_opts.host = self.url_params.host - connect_opts.port = self.url_params.port - end - - connect_opts.ssl_verify = self.ssl_verify - - if self.ssl_client_cert and self.ssl_client_priv_key then - connect_opts.ssl_client_cert = self.ssl_client_cert - connect_opts.ssl_client_priv_key = self.ssl_client_priv_key - end - - -- Try to reconnect (resty.http handles keep-alive automatically) - local ok, err = self.httpc:connect(connect_opts) - if ok then - return self.httpc, nil - end - - -- Connection failed, create new one - ngx.log(ngx.DEBUG, "Connection reuse failed, creating new: " .. (err or "unknown")) - self.httpc = nil - end + -- After set_keepalive(), the httpc is in a closed state + -- resty.http will reuse the underlying connection from its pool when we create a new httpc + -- So we always create a new httpc instance here + -- (The actual TCP connection is reused by resty.http internally) -- Create new HTTP client instance self.httpc = http.new() @@ -303,6 +275,7 @@ function Client:_get_httpc() if self.url_params.is_unix then connect_opts.host = "localhost" connect_opts.path = self.url_params.socket_path + -- Explicitly set scheme and port to nil for Unix sockets (resty.http requirement) connect_opts.scheme = nil connect_opts.port = nil else @@ -345,8 +318,9 @@ function Client:_release_httpc() end -- After set_keepalive(), the httpc object is in a "closed" state but the connection - -- is in resty.http's pool. We keep the reference - on next request, connect() will - -- reuse the connection from the pool automatically. + -- is in resty.http's pool. Clear the reference - we'll create a new httpc on next request, + -- and resty.http will automatically reuse the underlying connection from its pool. + self.httpc = nil return true, nil end From f16a5f906be4080311e8b0badd2f11702a3db2ca Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 17:46:12 +0000 Subject: [PATCH 06/17] Fix Unix socket connection by using full unix:/path format in host field --- lib/plugins/crowdsec/http_client.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 4ca1453..7dee055 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -100,6 +100,9 @@ function M.parse_url(url_str) parsed.socket_path = "/" .. parsed.socket_path end + -- Store full unix path with scheme for resty.http (host field expects "unix:/path") + parsed.socket_path_full = "unix:" .. parsed.socket_path + parsed.full_path = parsed.path parsed.connection_key = "unix:" .. parsed.socket_path return parsed @@ -273,8 +276,8 @@ function Client:_get_httpc() local connect_opts = {} if self.url_params.is_unix then - connect_opts.host = "localhost" - connect_opts.path = self.url_params.socket_path + -- For Unix sockets, resty.http expects the full "unix:/path" in the host field + connect_opts.host = self.url_params.socket_path_full or ("unix:" .. self.url_params.socket_path) -- Explicitly set scheme and port to nil for Unix sockets (resty.http requirement) connect_opts.scheme = nil connect_opts.port = nil From a110ae051d7e5bc5a283cdc267057a41719636a8 Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 17:50:18 +0000 Subject: [PATCH 07/17] Add error handling for set_keepalive and use connection_key for Unix socket host --- lib/plugins/crowdsec/http_client.lua | 35 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 7dee055..033f5c3 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -100,10 +100,8 @@ function M.parse_url(url_str) parsed.socket_path = "/" .. parsed.socket_path end - -- Store full unix path with scheme for resty.http (host field expects "unix:/path") - parsed.socket_path_full = "unix:" .. parsed.socket_path - parsed.full_path = parsed.path + -- connection_key is "unix:/path" which is what resty.http expects in the host field parsed.connection_key = "unix:" .. parsed.socket_path return parsed end @@ -277,7 +275,8 @@ function Client:_get_httpc() if self.url_params.is_unix then -- For Unix sockets, resty.http expects the full "unix:/path" in the host field - connect_opts.host = self.url_params.socket_path_full or ("unix:" .. self.url_params.socket_path) + -- Use connection_key which already contains "unix:/path" + connect_opts.host = self.url_params.connection_key -- Explicitly set scheme and port to nil for Unix sockets (resty.http requirement) connect_opts.scheme = nil connect_opts.port = nil @@ -312,12 +311,32 @@ function Client:_release_httpc() return true, nil end - local ok, err = self.httpc:set_keepalive(self.keepalive_timeout, self.keepalive_pool_size) + -- Check if keepalive_timeout and keepalive_pool_size are set + -- If not, just close the connection (no keep-alive) + if not self.keepalive_timeout or not self.keepalive_pool_size then + pcall(function() self.httpc:close() end) + self.httpc = nil + return true, nil + end + + -- Try to set keepalive - use pcall to safely handle any errors + local success, ok, err = pcall(function() + return self.httpc:set_keepalive(self.keepalive_timeout, self.keepalive_pool_size) + end) + + if not success then + -- pcall failed - set_keepalive threw an error (ok contains the error message) + pcall(function() self.httpc:close() end) + self.httpc = nil + return false, "Failed to set keepalive: " .. tostring(ok) + end + + -- Check if set_keepalive returned success (ok is boolean, err is error message if failed) if not ok then - -- If keepalive fails, close the connection - self.httpc:close() + -- set_keepalive returned false + pcall(function() self.httpc:close() end) self.httpc = nil - return false, "Failed to set keepalive: " .. (err or "unknown") + return false, "Failed to set keepalive: " .. (tostring(err) or "unknown") end -- After set_keepalive(), the httpc object is in a "closed" state but the connection From 656071c1849275760c284b938eead16d66cdf434 Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 18:03:53 +0000 Subject: [PATCH 08/17] Update documentation for ssl_client_cert to note it accepts parsed PEM objects --- lib/plugins/crowdsec/http_client.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 033f5c3..9f0a7ba 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -219,8 +219,8 @@ end -- @param options table: Client options: -- timeouts: table {connect, send, read} - timeout values in ms -- ssl_verify: boolean - whether to verify SSL certificates --- ssl_client_cert: string - (optional) path to client certificate for mTLS --- ssl_client_priv_key: string - (optional) path to client private key for mTLS +-- ssl_client_cert: string or cdata - (optional) path to client certificate for mTLS, or parsed PEM object +-- ssl_client_priv_key: string or cdata - (optional) path to client private key for mTLS, or parsed PEM object -- keepalive_timeout: number - (optional) keep-alive timeout in ms -- keepalive_pool_size: number - (optional) pool size -- @return client: HTTP client object, or nil on error From 4d07533ce435a19dd155448409c3a079021219db Mon Sep 17 00:00:00 2001 From: Laurence Date: Mon, 5 Jan 2026 18:12:37 +0000 Subject: [PATCH 09/17] Remove unused M.request_uri function The module-level request_uri function was kept for backward compatibility but is not used anywhere in the codebase. All code uses the object-oriented API (http_client.new() + client:request_uri()) instead. Signed-off-by: Laurence --- lib/plugins/crowdsec/http_client.lua | 66 ---------------------------- 1 file changed, 66 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 9f0a7ba..f073c29 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -147,72 +147,6 @@ function M.parse_url(url_str) return parsed end ---- Make simple HTTP request (like request_uri, for LAPI - backward compatibility) --- This function creates a temporary client object internally --- @param url_str string: Full URL (http://host:port/path or unix:/socket/path) --- @param options table: Request options: --- timeout: number (single timeout for all operations) --- connect_timeout: number (optional, separate connect timeout) --- send_timeout: number (optional, separate send timeout) --- read_timeout: number (optional, separate read timeout) --- method: string (default: "GET") --- headers: table (HTTP headers) --- body: string (request body) --- ssl_verify: boolean (whether to verify SSL) --- ssl_client_cert: string (optional, for TLS auth) --- ssl_client_priv_key: string (optional, for TLS auth) --- @return res: HTTP response object with .status and .body, or nil on error --- @return err: Error message if failed -function M.request_uri(url_str, options) - options = options or {} - - -- Parse URL once to split base and path/query - local parsed, perr = M.parse_url(url_str) - if not parsed then - return nil, perr - end - - local base_url - local path_with_query = parsed.full_path or parsed.path or "/" - - if parsed.is_unix then - base_url = "unix:" .. parsed.socket_path - else - base_url = parsed.scheme .. "://" .. parsed.host .. ":" .. parsed.port - end - - -- Setup timeouts for client creation - local timeouts = {} - if options.timeout then - timeouts.connect = options.timeout - timeouts.send = options.timeout - timeouts.read = options.timeout - else - timeouts.connect = options.connect_timeout or 1000 - timeouts.send = options.send_timeout or 1000 - timeouts.read = options.read_timeout or 1000 - end - - -- Create temporary client object (reuses connection pooling internally) - local client, err = M.new(base_url, { - timeouts = timeouts, - ssl_verify = options.ssl_verify, - ssl_client_cert = options.ssl_client_cert, - ssl_client_priv_key = options.ssl_client_priv_key - }) - - if not client then - return nil, err - end - - -- Use client's request_uri method - return client:request_uri(path_with_query ~= "" and path_with_query or "/", { - method = options.method, - headers = options.headers, - body = options.body - }) -end - --- Create a new HTTP client object -- Parse URL once and create a reusable client object -- @param url_str string: URL (http://host:port, https://host:port, or unix:/path) From 814b84b81e40ae86236de8f308864ac5f15cff04 Mon Sep 17 00:00:00 2001 From: Laurence Date: Tue, 6 Jan 2026 08:44:06 +0000 Subject: [PATCH 10/17] Fix: Wrap httpc:close() calls in pcall to prevent runtime errors Prevent 'bad request' errors when closing HTTP client connections that may already be in a closed or bad state. This matches the pattern used elsewhere in the file for safe connection cleanup. --- lib/plugins/crowdsec/http_client.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index f073c29..c0b1474 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -404,7 +404,7 @@ function Client:request(method, path, headers, body) if not res then -- Request failed, clear httpc so we create a new one next time - self.httpc:close() + pcall(function() self.httpc:close() end) self.httpc = nil return nil, err or "Request failed" end @@ -491,7 +491,7 @@ function Client:request_uri(path, options) if not res then -- Request failed, clear httpc so we create a new one next time - self.httpc:close() + pcall(function() self.httpc:close() end) self.httpc = nil return nil, err or "Request failed" end From 3f45b7877aeccf01b10ebef62cb535362bc0d2d3 Mon Sep 17 00:00:00 2001 From: Laurence Date: Tue, 6 Jan 2026 09:11:38 +0000 Subject: [PATCH 11/17] Add tests for Unix socket LAPI support - Test 17: Fix missing newline in TLS auth test - Test 18: Live mode with Unix socket LAPI - Test 19: Stream mode with Unix socket LAPI - Test 20: Live mode with Unix socket LAPI and metrics These tests verify that the bouncer can connect to the LAPI via Unix sockets for both decisions and metrics endpoints. --- t/17live_and_tls.t | 2 +- t/18_live_unix_socket.t | 66 +++++++ t/19_stream_unix_socket.t | 141 +++++++++++++++ t/20_live_unix_socket_metrics.t | 166 ++++++++++++++++++ ...ve_unix_socket_crowdsec_nginx_bouncer.conf | 31 ++++ ...am_unix_socket_crowdsec_nginx_bouncer.conf | 31 ++++ ...socket_metrics_crowdsec_nginx_bouncer.conf | 31 ++++ 7 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 t/18_live_unix_socket.t create mode 100644 t/19_stream_unix_socket.t create mode 100644 t/20_live_unix_socket_metrics.t create mode 100644 t/conf_t/18_live_unix_socket_crowdsec_nginx_bouncer.conf create mode 100644 t/conf_t/19_stream_unix_socket_crowdsec_nginx_bouncer.conf create mode 100644 t/conf_t/20_live_unix_socket_metrics_crowdsec_nginx_bouncer.conf diff --git a/t/17live_and_tls.t b/t/17live_and_tls.t index 05f2dd1..8805b98 100644 --- a/t/17live_and_tls.t +++ b/t/17live_and_tls.t @@ -66,4 +66,4 @@ location = /t { X-Forwarded-For: 1.1.1.1 --- request GET /t ---- error_code: 403 \ No newline at end of file +--- error_code: 403 diff --git a/t/18_live_unix_socket.t b/t/18_live_unix_socket.t new file mode 100644 index 0000000..6192a42 --- /dev/null +++ b/t/18_live_unix_socket.t @@ -0,0 +1,66 @@ +use Test::Nginx::Socket 'no_plan'; + +run_tests(); + +__DATA__ + +=== TEST 18: Live mode with Unix socket LAPI + +--- main_config +load_module /usr/share/nginx/modules/ndk_http_module.so; +load_module /usr/share/nginx/modules/ngx_http_lua_module.so; + +--- http_config + +lua_package_path './lib/?.lua;;'; +lua_shared_dict crowdsec_cache 50m; +lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + +init_by_lua_block +{ + cs = require "crowdsec" + local ok, err = cs.init("./t/conf_t/18_live_unix_socket_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8") + if ok == nil then + ngx.log(ngx.ERR, "[Crowdsec] " .. err) + error() + end + ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done") +} + +access_by_lua_block { + local cs = require "crowdsec" + cs.Allow(ngx.var.remote_addr) +} + +server { + listen unix:/tmp/crowdsec-test.sock; + + location = /v1/decisions { + content_by_lua_block { + local args, err = ngx.req.get_uri_args() + if args.ip == "1.1.1.1" then + ngx.say('[{"duration":"1h00m00s","id":4091593,"origin":"CAPI","scenario":"crowdsecurity/vpatch-CVE-2024-4577","scope":"Ip","type":"ban","value":"1.1.1.1"}]') + else + ngx.say('[{}]') + end + } + } +} + +--- config + +location = /t { + set_real_ip_from 127.0.0.1; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + content_by_lua_block { + ngx.say(ngx.var.remote_addr) + } +} + +--- more_headers +X-Forwarded-For: 1.1.1.1 +--- request +GET /t +--- error_code: 403 + diff --git a/t/19_stream_unix_socket.t b/t/19_stream_unix_socket.t new file mode 100644 index 0000000..ab9eb69 --- /dev/null +++ b/t/19_stream_unix_socket.t @@ -0,0 +1,141 @@ +use Test::Nginx::Socket 'no_plan'; + +run_tests(); + +__DATA__ + +=== TEST 19: Stream mode with Unix socket LAPI + +--- init + +use LWP::UserAgent; + +my $ua = LWP::UserAgent->new; +my $url = 'http://127.0.0.1:1984/t'; + +my $req = HTTP::Request->new(GET => $url); +open my $out_fh, '>', 't/servroot/logs/perl.init.log' or die $!; +select $out_fh; +$req->header('X-Forwarded-For' => '1.1.1.2'); + +my $resp = $ua->request($req); +if ($resp->is_success) { + my $message = $resp->decoded_content; + print "Received reply: $message"; +} else { + print "Initialization failed with HTTP code " . $resp->code . " with " . $resp->message, + exit 1 +} +sleep(11) + +--- main_config +load_module /usr/share/nginx/modules/ndk_http_module.so; +load_module /usr/share/nginx/modules/ngx_http_lua_module.so; + +--- http_config + +lua_package_path './lib/?.lua;;'; +lua_shared_dict crowdsec_cache 50m; +lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + +init_by_lua_block +{ + cs = require "crowdsec" + local ok, err = cs.init("./t/conf_t/19_stream_unix_socket_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8") + if ok == nil then + ngx.log(ngx.ERR, "[Crowdsec] " .. err) + error() + end + ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done") +} + +access_by_lua_block { + local cs = require "crowdsec" + cs.Allow(ngx.var.remote_addr) +} + +init_worker_by_lua_block { + cs = require "crowdsec" + local mode = cs.get_mode() + if string.lower(mode) == "stream" then + ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id())) + cs.SetupStream() + end + + if ngx.worker.id() == 0 then + ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id())) + cs.SetupMetrics() + end +} + +server { + listen unix:/tmp/crowdsec-stream-test.sock; + + location = /v1/decisions/stream { + content_by_lua_block { + local args, err = ngx.req.get_uri_args() + if args.startup == "true" then + ngx.say('{"deleted": [], "new": [{"duration":"1h00m00s","id":4091593,"origin":"CAPI","scenario":"crowdsecurity/vpatch-CVE-2024-4577","scope":"Ip","type":"ban","value":"1.1.1.1"}]}') + else + ngx.say('{"deleted": [], "new": []}') + end + } + } +} + +--- config + +location = /t { + set_real_ip_from 127.0.0.1; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + content_by_lua_block { + ngx.print("ok") + } + log_by_lua_block { + local cache = ngx.shared.crowdsec_cache + local keys = cache:get_keys(0) + -- This dumps the crowdsec_cache to the error_log except for keys in the ignored_keys table + -- Those keys depend on timestamp, let's handle separatly in other tests if needed + ignored_keys = { "last_refresh", "metrics_startup_time", "metrics_data" } + for _, key in ipairs(keys) do + for _, ignored_key in ipairs(ignored_keys) do + if key == ignored_key then + goto continue + end + end + print("DEBUG CACHE:" .. key .. ":" .. tostring(cache:get(key))) + ::continue:: + end + } + +} + +--- more_headers +X-Forwarded-For: 1.1.1.1 +--- request +GET /t + +# 4294967295 (0xffffffff) is the netmask as an int +# 16843009 (0x01010101) is the ip as an int + +--- error_code: 403 +--- grep_error_log eval +qr/DEBUG CACHE:[^ ]*/ +--- grep_error_log_out +DEBUG CACHE:startup:true +DEBUG CACHE:first_run:true +DEBUG CACHE:metrics_first_run:false +DEBUG CACHE:metrics_processed/ip_type=ipv4&:1 +DEBUG CACHE:metrics_all:processed/ip_type=ipv4&, +DEBUG CACHE:captcha_ok:false +DEBUG CACHE:first_run:true +DEBUG CACHE:metrics_first_run:false +DEBUG CACHE:startup:false +DEBUG CACHE:refreshing:false +DEBUG CACHE:metrics_processed/ip_type=ipv4&:2 +DEBUG CACHE:decision_cache/ipv4_4294967295_16843009:ban/CAPI/ipv4 +DEBUG CACHE:metrics_dropped/ip_type=ipv4&origin=CAPI&:1 +DEBUG CACHE:metrics_all:processed/ip_type=ipv4&,dropped/ip_type=ipv4&origin=CAPI&, +DEBUG CACHE:captcha_ok:false + diff --git a/t/20_live_unix_socket_metrics.t b/t/20_live_unix_socket_metrics.t new file mode 100644 index 0000000..3469861 --- /dev/null +++ b/t/20_live_unix_socket_metrics.t @@ -0,0 +1,166 @@ +# 4294967295 (0xffffffff) is the netmask as an int +# 16843009 (0x01010101) is the ip as an int +# metrics_processed are reset after 10s in this test + +use Test::Nginx::Socket 'no_plan'; + +run_tests(); + +__DATA__ + +=== TEST 20: Live mode with Unix socket LAPI and metrics + +--- init + +use LWP::UserAgent; + +my $ua = LWP::UserAgent->new; +my $url = 'http://127.0.0.1:1984/t'; + +open my $out_fh, '>', 't/servroot/logs/perl.init.log' or die $!; +print $out_fh "Starting initialization...\n"; + +my $req = HTTP::Request->new(GET => $url); +$req->header('X-Forwarded-For' => '1.1.1.2'); + +my $resp = $ua->request($req); +if ($resp->is_success) { + my $message = $resp->decoded_content; + print $out_fh "Received reply: n\$message"; +} else { + print $out_fh "Initialization failed with HTTP code " . $resp->code . " and message: " . $resp->message . "\n"; + exit 1; +} + +sleep(1); + +$req = HTTP::Request->new(GET => $url); +$req->header('X-Forwarded-For' => '1.1.1.1'); + +$resp = $ua->request($req); +if (!$resp->is_success) { + if ($resp->code == 403) { + print $out_fh "Request forbidden with 403 as expected" . "\n"; + } else { + print $out_fh "Initialization failed with HTTP code " . $resp->code . " and message: " . $resp->message . "\n"; + exit 1 + } +} else { + my $message = $resp->decoded_content; + print $out_fh "Received reply: $message\n" . " which should not happen"; + exit 1; +} + +print $out_fh "Initialization completed successfully.\n"; +close $out_fh or warn "Could not close filehandle: $!"; +--- main_config +load_module /usr/share/nginx/modules/ndk_http_module.so; +load_module /usr/share/nginx/modules/ngx_http_lua_module.so; + +--- http_config + +lua_package_path './lib/?.lua;;'; +lua_shared_dict crowdsec_cache 50m; +lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + + +init_by_lua_block +{ + cs = require "crowdsec" + local ok, err = cs.init("./t/conf_t/20_live_unix_socket_metrics_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8") + if ok == nil then + ngx.log(ngx.ERR, "[Crowdsec] " .. err) + error() + end + ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done") + -- shortening the metrics timer + cs.debug_metrics() +} + +access_by_lua_block { + local cs = require "crowdsec" + cs.Allow(ngx.var.remote_addr) +} + +init_worker_by_lua_block { + cs = require "crowdsec" + local mode = cs.get_mode() + if string.lower(mode) == "stream" then + ngx.log(ngx.INFO, "Initilizing stream mode for worker " .. tostring(ngx.worker.id())) + cs.SetupStream() + end + + if ngx.worker.id() == 0 then + ngx.log(ngx.INFO, "Initilizing metrics for worker " .. tostring(ngx.worker.id())) + cs.SetupMetrics() + end +} + +server { + listen unix:/tmp/crowdsec-metrics-test.sock; + + location = /v1/decisions { + content_by_lua_block { + local args, err = ngx.req.get_uri_args() + if args.ip == "1.1.1.1" then + ngx.say('[{"duration":"1h00m00s","id":4091593,"origin":"CAPI","scenario":"crowdsecurity/vpatch-CVE-2024-4577","scope":"Ip","type":"ban","value":"1.1.1.1"}]') + else + ngx.print("null") + end + } + } + location = /v1/usage-metrics { + content_by_lua_block { + local cjson = require "cjson" + ngx.req.read_body() + local body = ngx.req.get_body_data() + json = cjson.decode(body) + print("EXTRACT METRICS JSON:" .. "type:" .. json["remediation_components"][1]["type"] .. " ") + print("EXTRACT METRICS JSON:" .. "name:" .. json["remediation_components"][1]["name"] .. " ") + print("EXTRACT METRICS JSON:" .. "window_size:" .. json["remediation_components"][1]["metrics"][1]["meta"]["window_size_seconds"] .. " ") + -- three loops to ensure order + for _, val in ipairs(json["remediation_components"][1]["metrics"][1]["items"]) do + if val["name"] == "processed" then + print("EXTRACT METRICS JSON:" .. val["name"] .. "/" .. val["unit"] .. "/" .. tostring(val["value"]) .. "/" .. val["labels"]["ip_type"] .. " ") + end + end + for _, val in ipairs(json["remediation_components"][1]["metrics"][1]["items"]) do + if val["name"] == "dropped" then + print("EXTRACT METRICS JSON:" .. val["name"] .. "/" .. val["unit"] .. "/" .. tostring(val["value"]) .. "/" .. val["labels"]["ip_type"] .. "/" .. val["labels"]["origin"] .. " ") + end + end + + ngx.status = 201 + ngx.say("Created") + } + } +} + + +--- config + +location = /t { + set_real_ip_from 127.0.0.1; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + content_by_lua_block { + ngx.print("ok") + } +} + +--- more_headers +X-Forwarded-For: 1.1.1.1 +--- request +GET /t + +--- error_code: 403 +--- grep_error_log eval +qr/EXTRACT METRICS JSON:[^ ]*/ +--- grep_error_log_out +EXTRACT METRICS JSON:type:lua-bouncer +EXTRACT METRICS JSON:name:nginx +EXTRACT METRICS JSON:window_size:15 +EXTRACT METRICS JSON:processed/request/3/ipv4 +EXTRACT METRICS JSON:dropped/request/2/ipv4/CAPI +--- wait: 15 + diff --git a/t/conf_t/18_live_unix_socket_crowdsec_nginx_bouncer.conf b/t/conf_t/18_live_unix_socket_crowdsec_nginx_bouncer.conf new file mode 100644 index 0000000..8ca9d2c --- /dev/null +++ b/t/conf_t/18_live_unix_socket_crowdsec_nginx_bouncer.conf @@ -0,0 +1,31 @@ +APPSEC_URL=http://127.0.0.1:7422 +ENABLED=true +API_URL=unix:/tmp/crowdsec-test.sock +API_KEY=LFrdL+aiecMTSxpGE9vLkx5sGMwdIpgVovpVMfXp3J0 +CACHE_EXPIRATION=1 +# bounce for all type of remediation that the bouncer can receive from the local API +BOUNCING_ON_TYPE=all +FALLBACK_REMEDIATION=ban +REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 +UPDATE_FREQUENCY=10 +# live or stream +MODE=live +# exclude the bouncing on those location +EXCLUDE_LOCATION=/v1/decisions +#those apply for "ban" action +# /!\ REDIRECT_LOCATION and RET_CODE can't be used together. REDIRECT_LOCATION take priority over RET_CODE +BAN_TEMPLATE_PATH=./ban +REDIRECT_LOCATION= +RET_CODE= +#those apply for "captcha" action +#valid providers are recaptcha, hcaptcha, turnstile +CAPTCHA_PROVIDER= +# Captcha Secret Key +SECRET_KEY= +# Captcha Site key +SITE_KEY= +CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html +CAPTCHA_EXPIRATION=3600 +#METRICS_PERIOD=60 + diff --git a/t/conf_t/19_stream_unix_socket_crowdsec_nginx_bouncer.conf b/t/conf_t/19_stream_unix_socket_crowdsec_nginx_bouncer.conf new file mode 100644 index 0000000..fcc1971 --- /dev/null +++ b/t/conf_t/19_stream_unix_socket_crowdsec_nginx_bouncer.conf @@ -0,0 +1,31 @@ +APPSEC_URL=http://127.0.0.1:7422 +ENABLED=true +API_URL=unix:/tmp/crowdsec-stream-test.sock +API_KEY=LFrdL+aiecMTSxpGE9vLkx5sGMwdIpgVovpVMfXp3J0 +CACHE_EXPIRATION=1 +# bounce for all type of remediation that the bouncer can receive from the local API +BOUNCING_ON_TYPE=all +FALLBACK_REMEDIATION=ban +REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 +UPDATE_FREQUENCY=10 +# live or stream +MODE=stream +# exclude the bouncing on those location +EXCLUDE_LOCATION=/v1/decisions/stream +#those apply for "ban" action +# /!\ REDIRECT_LOCATION and RET_CODE can't be used together. REDIRECT_LOCATION take priority over RET_CODE +BAN_TEMPLATE_PATH=./ban +REDIRECT_LOCATION= +RET_CODE= +#those apply for "captcha" action +#valid providers are recaptcha, hcaptcha, turnstile +CAPTCHA_PROVIDER= +# Captcha Secret Key +SECRET_KEY= +# Captcha Site key +SITE_KEY= +CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html +CAPTCHA_EXPIRATION=3600 +#METRICS_PERIOD=60 + diff --git a/t/conf_t/20_live_unix_socket_metrics_crowdsec_nginx_bouncer.conf b/t/conf_t/20_live_unix_socket_metrics_crowdsec_nginx_bouncer.conf new file mode 100644 index 0000000..c15b15b --- /dev/null +++ b/t/conf_t/20_live_unix_socket_metrics_crowdsec_nginx_bouncer.conf @@ -0,0 +1,31 @@ +APPSEC_URL=http://127.0.0.1:7422 +ENABLED=true +API_URL=unix:/tmp/crowdsec-metrics-test.sock +API_KEY=LFrdL+aiecMTSxpGE9vLkx5sGMwdIpgVovpVMfXp3J0 +CACHE_EXPIRATION=1 +# bounce for all type of remediation that the bouncer can receive from the local API +BOUNCING_ON_TYPE=all +FALLBACK_REMEDIATION=ban +REQUEST_TIMEOUT=3000 +KEEPALIVE_TIMEOUT=5000 +UPDATE_FREQUENCY=10 +# live or stream +MODE=live +# exclude the bouncing on those location +EXCLUDE_LOCATION=/v1/decisions,/v1/usage-metrics +#those apply for "ban" action +# /!\ REDIRECT_LOCATION and RET_CODE can't be used together. REDIRECT_LOCATION take priority over RET_CODE +BAN_TEMPLATE_PATH=./ban +REDIRECT_LOCATION= +RET_CODE= +#those apply for "captcha" action +#valid providers are recaptcha, hcaptcha, turnstile +CAPTCHA_PROVIDER= +# Captcha Secret Key +SECRET_KEY= +# Captcha Site key +SITE_KEY= +CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html +CAPTCHA_EXPIRATION=3600 +#METRICS_PERIOD=60 + From 46da7a9e011b325854ac2e1a252fbe65320eb328 Mon Sep 17 00:00:00 2001 From: Laurence Date: Tue, 6 Jan 2026 09:58:48 +0000 Subject: [PATCH 12/17] Refactor HTTP client to centralize authentication and configuration - Centralize API key, user agent, and TLS auth handling in http_client - Remove duplicate configuration storage from live, stream, and metrics modules - Add timeout validation and default to 3000ms if not configured - Add APPSEC client initialization with API key support (no mTLS) - Simplify metrics:sendMetrics() signature to remove unused parameters - Add debug logging for timeout configuration --- lib/crowdsec.lua | 38 +++++------ lib/plugins/crowdsec/http_client.lua | 96 ++++++++++++++++++++++++++-- lib/plugins/crowdsec/live.lua | 62 +++++++++--------- lib/plugins/crowdsec/metrics.lua | 60 +++++++++-------- lib/plugins/crowdsec/stream.lua | 63 +++++++++--------- 5 files changed, 201 insertions(+), 118 deletions(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 2be6a11..3668448 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -29,7 +29,6 @@ local APPSEC_HOST_HEADER = "x-crowdsec-appsec-host" local APPSEC_VERB_HEADER = "x-crowdsec-appsec-verb" local APPSEC_URI_HEADER = "x-crowdsec-appsec-uri" local APPSEC_USER_AGENT_HEADER = "x-crowdsec-appsec-user-agent" -local REMEDIATION_API_KEY_HEADER = 'x-api-key' local METRICS_PERIOD = 900 --- only for debug purpose @@ -210,18 +209,15 @@ function csmod.init(configFile, userAgent) ngx.log(ngx.ERR, "Lua shared dict (crowdsec cache) is full, please increase dict size in config") end - if runtime.conf["ALWAYS_SEND_TO_APPSEC"] == "false" then - runtime.conf["ALWAYS_SEND_TO_APPSEC"] = false - else - runtime.conf["ALWAYS_SEND_TO_APPSEC"] = true - end + runtime.conf["ALWAYS_SEND_TO_APPSEC"] = runtime.conf["ALWAYS_SEND_TO_APPSEC"] ~= "false" runtime.conf["APPSEC_ENABLED"] = false runtime.APPSEC_CLIENT = nil if runtime.conf["APPSEC_URL"] ~= "" then + -- APPSEC only supports API key authentication (no mTLS) -- Create HTTP client object once (URL parsed once) - local client, err = http_client.new(runtime.conf["APPSEC_URL"], { + local client_options = { timeouts = { connect = runtime.conf["APPSEC_CONNECT_TIMEOUT"], send = runtime.conf["APPSEC_SEND_TIMEOUT"], @@ -229,8 +225,15 @@ function csmod.init(configFile, userAgent) }, ssl_verify = runtime.conf["SSL_VERIFY"], keepalive_timeout = runtime.conf["KEEPALIVE_TIMEOUT"], - keepalive_pool_size = runtime.conf["KEEPALIVE_POOL_SIZE"] - }) + keepalive_pool_size = runtime.conf["KEEPALIVE_POOL_SIZE"], + user_agent = userAgent, + use_tls_auth = false, -- APPSEC does not support mTLS + api_key = runtime.conf["API_KEY"], + -- APPSEC uses a different API key header name + api_key_header = APPSEC_API_KEY_HEADER + } + + local client, err = http_client.new(runtime.conf["APPSEC_URL"], client_options) if not client then ngx.log(ngx.ERR, "Failed to create APPSEC HTTP client: " .. (err or "unknown")) @@ -284,10 +287,10 @@ function csmod.init(configFile, userAgent) if runtime.conf["MODE"] == "live" then ngx.log(ngx.INFO, "lua nginx bouncer enabled with live mode") - runtime.live = live:new(runtime.conf, runtime.userAgent, REMEDIATION_API_KEY_HEADER) + runtime.live = live:new(runtime.conf, runtime.userAgent) else ngx.log(ngx.INFO, "lua nginx bouncer enabled with stream mode") - runtime.stream = stream:new(runtime.conf, runtime.userAgent, REMEDIATION_API_KEY_HEADER) + runtime.stream = stream:new(runtime.conf, runtime.userAgent) end return true, nil end @@ -317,7 +320,7 @@ function csmod.SetupMetrics() local first_run = runtime.cache:get("metrics_first_run") if first_run then ngx.log(ngx.DEBUG, "First run for setup metrics ") - metrics:new(runtime.userAgent, runtime.conf, REMEDIATION_API_KEY_HEADER) + metrics:new(runtime.userAgent, runtime.conf) runtime.cache:set("metrics_first_run",false) Setup_metrics_timer() return @@ -329,12 +332,11 @@ function csmod.SetupMetrics() if runtime.conf["MODE"] == "stream" then stream:refresh_metrics() end - -- Headers are now handled internally by metrics (API key added conditionally based on mTLS) + -- HTTP client handles User-Agent and API key automatically + -- Only pass Content-Type header metrics:sendMetrics( - runtime.conf["API_URL"], - {['User-Agent']=runtime.userAgent,["Content-Type"]="application/json"}, - runtime.conf["SSL_VERIFY"], - METRICS_PERIOD + METRICS_PERIOD, + {["Content-Type"]="application/json"} ) end local succ, err, forcible = runtime.cache:set("metrics_startup_time", ngx.time()) -- to make sure we have only one thread sending metrics @@ -565,7 +567,7 @@ function csmod.AppSecCheck(ip) headers[APPSEC_VERB_HEADER] = ngx.var.request_method headers[APPSEC_URI_HEADER] = uri headers[APPSEC_USER_AGENT_HEADER] = ngx.var.http_user_agent - headers[APPSEC_API_KEY_HEADER] = runtime.conf["API_KEY"] + -- API key is now automatically added by http_client with the correct header name local ok, remediation, status_code = true, "allow", 200 if runtime.conf["APPSEC_FAILURE_ACTION"] == DENY then diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index c0b1474..b97060a 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -19,6 +19,9 @@ local url = require "plugins.crowdsec.url" local M = {} +-- Default API key header name +M.DEFAULT_API_KEY_HEADER = "X-Api-Key" + -- Note: Connection pooling is handled by resty.http via set_keepalive() -- Each Client object owns its own httpc instance @@ -157,6 +160,10 @@ end -- ssl_client_priv_key: string or cdata - (optional) path to client private key for mTLS, or parsed PEM object -- keepalive_timeout: number - (optional) keep-alive timeout in ms -- keepalive_pool_size: number - (optional) pool size +-- api_key: string - (optional) API key for authentication +-- api_key_header: string - (optional) Header name for API key (default: http_client.DEFAULT_API_KEY_HEADER) +-- user_agent: string - (optional) User-Agent header value +-- use_tls_auth: boolean - (optional) Whether to use TLS auth instead of API key -- @return client: HTTP client object, or nil on error -- @return err: Error message if failed function M.new(url_str, options) @@ -168,22 +175,43 @@ function M.new(url_str, options) return nil, err end + -- Determine if using TLS auth (either explicitly set or inferred from cert/key presence) + local use_tls_auth = options.use_tls_auth + if use_tls_auth == nil then + use_tls_auth = (options.ssl_client_cert ~= nil and options.ssl_client_priv_key ~= nil) + end + -- Build connection key with mTLS info if applicable local connection_key = url_params.connection_key if options.ssl_client_cert and options.ssl_client_priv_key then connection_key = connection_key .. "|mtls" end + -- Validate and set timeouts (ensure they're numbers and positive) + local timeouts = options.timeouts or {connect=3000, send=3000, read=3000} + timeouts.connect = tonumber(timeouts.connect) or 3000 + timeouts.send = tonumber(timeouts.send) or 3000 + timeouts.read = tonumber(timeouts.read) or 3000 + + -- Ensure timeouts are positive + if timeouts.connect <= 0 then timeouts.connect = 3000 end + if timeouts.send <= 0 then timeouts.send = 3000 end + if timeouts.read <= 0 then timeouts.read = 3000 end + -- Create client object with its own httpc instance local client = setmetatable({ url_params = url_params, connection_key = connection_key, - timeouts = options.timeouts or {connect=1000, send=1000, read=1000}, + timeouts = timeouts, ssl_verify = options.ssl_verify ~= false, -- default true ssl_client_cert = options.ssl_client_cert, ssl_client_priv_key = options.ssl_client_priv_key, keepalive_timeout = options.keepalive_timeout, keepalive_pool_size = options.keepalive_pool_size, + api_key = options.api_key, + api_key_header = options.api_key_header or M.DEFAULT_API_KEY_HEADER, + user_agent = options.user_agent, + use_tls_auth = use_tls_auth, httpc = nil, -- HTTP client instance (created on first use) }, Client) @@ -202,7 +230,12 @@ function Client:_get_httpc() -- Create new HTTP client instance self.httpc = http.new() - self.httpc:set_timeouts(self.timeouts.connect, self.timeouts.send, self.timeouts.read) + -- Ensure timeouts are valid numbers before setting + local connect_timeout = tonumber(self.timeouts.connect) or 3000 + local send_timeout = tonumber(self.timeouts.send) or 3000 + local read_timeout = tonumber(self.timeouts.read) or 3000 + self.httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) + ngx.log(ngx.DEBUG, "[HTTP_CLIENT] Set timeouts: connect=" .. connect_timeout .. "ms, send=" .. send_timeout .. "ms, read=" .. read_timeout .. "ms") -- Connect local connect_opts = {} @@ -357,6 +390,35 @@ function Client:_build_path(path) return final_path end +--- Build full URL for error messages +-- @param path string: Request path (may include query string) +-- @return string: Full URL including base URL and path +function Client:build_url(path) + local full_path = self:_build_path(path) + + -- For Unix sockets, the full_url is "unix:/path/to/sock" and we append the HTTP path + if self.url_params.is_unix then + -- Extract the base socket path (everything before any HTTP path) + local base_socket = self.url_params.full_url + -- If the original URL had an HTTP path component, remove it + local http_path_match = base_socket:match("(/v%d+/.*)$") or base_socket:match("(/api/.*)$") + if http_path_match then + base_socket = base_socket:sub(1, #base_socket - #http_path_match) + end + return base_socket .. full_path + end + + -- For HTTP/HTTPS, construct from scheme, host, port, and path + local base_url = self.url_params.scheme .. "://" .. self.url_params.host + if self.url_params.port and + ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or + (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then + base_url = base_url .. ":" .. self.url_params.port + end + + return base_url .. full_path +end + --- Make HTTP request with full control -- @param client: Client object (self) -- @param method string: HTTP method (GET, POST, etc.) @@ -366,9 +428,10 @@ end -- @return res: HTTP response object, or nil on error -- @return err: Error message if failed function Client:request(method, path, headers, body) + local full_url = self:build_url(path) local httpc, err = self:_get_httpc() if not httpc then - return nil, err + return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") end -- Set Host header appropriately @@ -392,6 +455,16 @@ function Client:request(method, path, headers, body) end end + -- Add User-Agent if configured + if self.user_agent and not headers["User-Agent"] and not headers["user-agent"] then + headers["User-Agent"] = self.user_agent + end + + -- Add API key header if not using TLS auth + if not self.use_tls_auth and self.api_key then + headers[self.api_key_header] = self.api_key + end + local full_path = self:_build_path(path) -- Make request @@ -406,7 +479,7 @@ function Client:request(method, path, headers, body) -- Request failed, clear httpc so we create a new one next time pcall(function() self.httpc:close() end) self.httpc = nil - return nil, err or "Request failed" + return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") end -- Read response body @@ -450,11 +523,12 @@ function Client:request_uri(path, options) -- Build full path (include base path and query if configured) local full_path = self:_build_path(path) + local full_url = self:build_url(path) -- Get or create client local httpc, err = self:_get_httpc() if not httpc then - return nil, err + return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") end -- Prepare headers @@ -477,6 +551,16 @@ function Client:request_uri(path, options) end end + -- Add User-Agent if configured + if self.user_agent and not headers["User-Agent"] and not headers["user-agent"] then + headers["User-Agent"] = self.user_agent + end + + -- Add API key header if not using TLS auth + if not self.use_tls_auth and self.api_key then + headers[self.api_key_header] = self.api_key + end + -- Remove Connection: close header if present (we want keep-alive) headers["Connection"] = nil headers["connection"] = nil @@ -493,7 +577,7 @@ function Client:request_uri(path, options) -- Request failed, clear httpc so we create a new one next time pcall(function() self.httpc:close() end) self.httpc = nil - return nil, err or "Request failed" + return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") end -- Read response body diff --git a/lib/plugins/crowdsec/live.lua b/lib/plugins/crowdsec/live.lua index 437c77b..69b5b4c 100644 --- a/lib/plugins/crowdsec/live.lua +++ b/lib/plugins/crowdsec/live.lua @@ -11,36 +11,47 @@ live.cache = ngx.shared.crowdsec_cache -- Create a new live object to query the live API -- @param conf table: Runtime configuration table -- @param user_agent string: User agent string --- @param api_key_header string: the authorization header to use for the lapi request -- @return live: the live object -function live:new(conf, user_agent, api_key_header) +function live:new(conf, user_agent) local instance = setmetatable({}, self) - instance.api_url = conf["API_URL"] - instance.api_key_header = api_key_header - instance.api_key = conf["API_KEY"] - instance.user_agent = user_agent - instance.use_tls_auth = conf["USE_TLS_AUTH"] and - conf["TLS_CLIENT_CERT_PARSED"] ~= nil and - conf["TLS_CLIENT_KEY_PARSED"] ~= nil - -- Create single HTTP client (handles mTLS if configured) + -- Create single HTTP client (handles mTLS, API key, and user agent if configured) instance.API_CLIENT = nil if conf["API_URL"] ~= "" then + local use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil + + -- Ensure REQUEST_TIMEOUT is a valid number, default to 3000ms if not set + local request_timeout = tonumber(conf["REQUEST_TIMEOUT"]) + if not request_timeout or request_timeout <= 0 then + request_timeout = 3000 -- Default to 3 seconds + ngx.log(ngx.WARN, "REQUEST_TIMEOUT not set or invalid, using default: " .. request_timeout .. "ms") + end + local client_options = { timeouts = { - connect = conf["REQUEST_TIMEOUT"] or 1000, - send = conf["REQUEST_TIMEOUT"] or 1000, - read = conf["REQUEST_TIMEOUT"] or 1000 + connect = request_timeout, + send = request_timeout, + read = request_timeout }, ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], - keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], + user_agent = user_agent, + use_tls_auth = use_tls_auth } + -- Add API key only if not using TLS auth + if not use_tls_auth then + client_options.api_key = conf["API_KEY"] + -- Use default API key header from http_client + end + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) - if instance.use_tls_auth then + if use_tls_auth then client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] end @@ -49,6 +60,7 @@ function live:new(conf, user_agent, api_key_header) if client then instance.API_CLIENT = client + ngx.log(ngx.DEBUG, "[LIVE] Created HTTP client with timeouts: connect=" .. request_timeout .. "ms, send=" .. request_timeout .. "ms, read=" .. request_timeout .. "ms") else ngx.log(ngx.WARN, "Failed to create API HTTP client: " .. (err or "unknown")) end @@ -74,28 +86,18 @@ function live:live_query(ip, cache_expiration, bouncing_on_type) -- Build path (base path from API_URL will be prepended by request_uri) local path = "/v1/decisions?ip=" .. ip - local link = self.api_url .. path - - -- Build headers: always include User-Agent, include API key only if not using mTLS - local headers = { - ['User-Agent'] = self.user_agent - } - - if not self.use_tls_auth then - headers[self.api_key_header] = self.api_key - end + local full_url = self.API_CLIENT:build_url(path) local res, err = self.API_CLIENT:request_uri(path, { - method = "GET", - headers = headers + method = "GET" }) if not res then - ngx.log(ngx.ERR, "failed to query LAPI " .. link .. ": ".. (err or "unknown")) - return true, nil, nil, "request failed: ".. (err or "unknown") + ngx.log(ngx.ERR, "failed to query LAPI: " .. (err or "unknown")) + return true, nil, nil, err or "request failed" end - return self:live_query_process(res, ip, cache_expiration, bouncing_on_type, link) + return self:live_query_process(res, ip, cache_expiration, bouncing_on_type, full_url) end --- Process the HTTP response from the CrowdSec API for live queries diff --git a/lib/plugins/crowdsec/metrics.lua b/lib/plugins/crowdsec/metrics.lua index 7460cb4..83e00fc 100644 --- a/lib/plugins/crowdsec/metrics.lua +++ b/lib/plugins/crowdsec/metrics.lua @@ -9,7 +9,7 @@ metrics.cache = ngx.shared.crowdsec_cache -- Constructor for the store -function metrics:new(userAgent, conf, api_key_header) +function metrics:new(userAgent, conf) local info = osinfo.get_os_info() self.cache:set("metrics_data", cjson.encode({ version = userAgent, @@ -22,37 +22,43 @@ function metrics:new(userAgent, conf, api_key_header) utc_startup_timestamp = ngx.time(), })) - -- Create HTTP client for metrics (with mTLS support if configured) + -- Create HTTP client for metrics (with mTLS, API key, and user agent support if configured) self.metrics_client = nil - self.use_tls_auth = false - self.api_key_header = nil - self.api_key = nil if conf and conf["API_URL"] and conf["API_URL"] ~= "" then -- Check if mTLS is enabled - self.use_tls_auth = conf["USE_TLS_AUTH"] and - conf["TLS_CLIENT_CERT_PARSED"] ~= nil and - conf["TLS_CLIENT_KEY_PARSED"] ~= nil + local use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil - -- Store API key info for non-mTLS requests - if not self.use_tls_auth then - self.api_key_header = api_key_header or "X-Api-Key" - self.api_key = conf["API_KEY"] + -- Ensure REQUEST_TIMEOUT is a valid number, default to 3000ms if not set + local request_timeout = tonumber(conf["REQUEST_TIMEOUT"]) + if not request_timeout or request_timeout <= 0 then + request_timeout = 3000 -- Default to 3 seconds + ngx.log(ngx.WARN, "REQUEST_TIMEOUT not set or invalid, using default: " .. request_timeout .. "ms") end local client_options = { timeouts = { - connect = conf["REQUEST_TIMEOUT"] or 1000, - send = conf["REQUEST_TIMEOUT"] or 1000, - read = conf["REQUEST_TIMEOUT"] or 1000 + connect = request_timeout, + send = request_timeout, + read = request_timeout }, ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], - keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], + user_agent = userAgent, + use_tls_auth = use_tls_auth } + -- Add API key only if not using TLS auth + if not use_tls_auth then + client_options.api_key = conf["API_KEY"] + -- Use default API key header from http_client + end + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) - if self.use_tls_auth then + if use_tls_auth then client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] end @@ -60,6 +66,7 @@ function metrics:new(userAgent, conf, api_key_header) local client, err = http_client.new(conf["API_URL"], client_options) if client then self.metrics_client = client + ngx.log(ngx.DEBUG, "[METRICS] Created HTTP client with timeouts: connect=" .. request_timeout .. "ms, send=" .. request_timeout .. "ms, read=" .. request_timeout .. "ms") else ngx.log(ngx.WARN, "Failed to create metrics HTTP client: " .. (err or "unknown")) end @@ -192,7 +199,7 @@ function metrics:toJson(window) return cjson.encode({log_processors = cjson.null, remediation_components = remediation_components}) end -function metrics:sendMetrics(link, headers, ssl, window) +function metrics:sendMetrics(window, headers) local body = self:toJson(window) .. "\n" ngx.log(ngx.DEBUG, "Sending metrics to /v1/usage-metrics") ngx.log(ngx.DEBUG, "metrics: " .. body) @@ -203,18 +210,9 @@ function metrics:sendMetrics(link, headers, ssl, window) return end - -- Build headers (conditionally add API key if not using mTLS) - local request_headers = {} - if headers then - for k, v in pairs(headers) do - request_headers[k] = v - end - end - - -- Only add API key header if not using mTLS - if not self.use_tls_auth and self.api_key_header and self.api_key then - request_headers[self.api_key_header] = self.api_key - end + -- Build headers (merge any additional headers passed in) + -- HTTP client will automatically add User-Agent and API key if configured + local request_headers = headers or {} local res, err = self.metrics_client:request_uri("/v1/usage-metrics", { method = "POST", @@ -223,7 +221,7 @@ function metrics:sendMetrics(link, headers, ssl, window) }) if not res then - ngx.log(ngx.ERR, "failed to send metrics: ", err) + ngx.log(ngx.ERR, "failed to send metrics: " .. (err or "unknown")) else ngx.log(ngx.DEBUG, "metrics status: " .. res.status) ngx.log(ngx.DEBUG, "metrics body: " .. body) diff --git a/lib/plugins/crowdsec/stream.lua b/lib/plugins/crowdsec/stream.lua index c8db678..d4d0f9f 100644 --- a/lib/plugins/crowdsec/stream.lua +++ b/lib/plugins/crowdsec/stream.lua @@ -129,35 +129,46 @@ end -- Create a new stream object to query the stream API -- @param conf table: Runtime configuration table -- @param user_agent string: User agent string --- @param api_key_header string: the header to use for the API key -- @return stream: the stream object -function stream:new(conf, user_agent, api_key_header) +function stream:new(conf, user_agent) local instance = setmetatable({}, self) - instance.api_url = conf["API_URL"] - instance.api_key_header = api_key_header - instance.api_key = conf["API_KEY"] - instance.user_agent = user_agent - instance.use_tls_auth = conf["USE_TLS_AUTH"] and - conf["TLS_CLIENT_CERT_PARSED"] ~= nil and - conf["TLS_CLIENT_KEY_PARSED"] ~= nil - -- Create single HTTP client (handles mTLS if configured) + -- Create single HTTP client (handles mTLS, API key, and user agent if configured) instance.API_CLIENT = nil if conf["API_URL"] ~= "" then + local use_tls_auth = conf["USE_TLS_AUTH"] and + conf["TLS_CLIENT_CERT_PARSED"] ~= nil and + conf["TLS_CLIENT_KEY_PARSED"] ~= nil + + -- Ensure REQUEST_TIMEOUT is a valid number, default to 3000ms if not set + local request_timeout = tonumber(conf["REQUEST_TIMEOUT"]) + if not request_timeout or request_timeout <= 0 then + request_timeout = 3000 -- Default to 3 seconds + ngx.log(ngx.WARN, "REQUEST_TIMEOUT not set or invalid, using default: " .. request_timeout .. "ms") + end + local client_options = { timeouts = { - connect = conf["REQUEST_TIMEOUT"] or 1000, - send = conf["REQUEST_TIMEOUT"] or 1000, - read = conf["REQUEST_TIMEOUT"] or 1000 + connect = request_timeout, + send = request_timeout, + read = request_timeout }, ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], - keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"] + keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], + user_agent = user_agent, + use_tls_auth = use_tls_auth } + -- Add API key only if not using TLS auth + if not use_tls_auth then + client_options.api_key = conf["API_KEY"] + -- Use default API key header from http_client + end + -- Add mTLS options if TLS auth is enabled (use parsed PEM objects when available) - if instance.use_tls_auth then + if use_tls_auth then client_options.ssl_client_cert = conf["TLS_CLIENT_CERT_PARSED"] or conf["TLS_CLIENT_CERT"] client_options.ssl_client_priv_key = conf["TLS_CLIENT_KEY_PARSED"] or conf["TLS_CLIENT_KEY"] end @@ -166,6 +177,7 @@ function stream:new(conf, user_agent, api_key_header) if client then instance.API_CLIENT = client + ngx.log(ngx.DEBUG, "[STREAM] Created HTTP client with timeouts: connect=" .. request_timeout .. "ms, send=" .. request_timeout .. "ms, read=" .. request_timeout .. "ms") else ngx.log(ngx.WARN, "Failed to create API HTTP client: " .. (err or "unknown")) end @@ -185,10 +197,6 @@ function stream:stream_query(bouncing_on_type) return "HTTP client not available" end - if self.api_url == "" then - return "No API URL defined" - end - set_refreshing(true) local is_startup = stream.cache:get("startup") @@ -196,26 +204,15 @@ function stream:stream_query(bouncing_on_type) ngx.log(ngx.DEBUG, "Stream Query from worker : " .. tostring(ngx.worker.id()) .. " with startup "..tostring(is_startup)) -- Build path (base path from API_URL will be prepended by request_uri) local path = "/v1/decisions/stream?startup=" .. tostring(is_startup) - local link = self.api_url .. path - - -- Build headers: always include User-Agent, include API key only if not using mTLS - local headers = { - ['User-Agent'] = self.user_agent - } - - if not self.use_tls_auth then - headers[self.api_key_header] = self.api_key - end local res, err = self.API_CLIENT:request_uri(path, { - method = "GET", - headers = headers + method = "GET" }) if not res then set_refreshing(false) - ngx.log(ngx.ERR, "request to crowdsec lapi " .. link .. " failed: " .. (err or "unknown")) - return "request to crowdsec lapi " .. link .. " failed: " .. (err or "unknown") + ngx.log(ngx.ERR, "request to crowdsec lapi failed: " .. (err or "unknown")) + return err or "request to crowdsec lapi failed" end return self:stream_query_process(res, bouncing_on_type) From 7c8751d19516b632420fd87f25fca4335ddff3c1 Mon Sep 17 00:00:00 2001 From: Laurence Date: Tue, 6 Jan 2026 16:32:58 +0000 Subject: [PATCH 13/17] Refactor HTTP client to follow resty.http design pattern - Remove self.httpc storage, always create new instance per request - Rename _get_httpc() to _create_httpc() to reflect create-per-request pattern - Update _release_httpc() to accept httpc parameter instead of using self.httpc - Fix GET request handling: only include body in request options when present - Follow resty.http pattern: create, use, keepalive (don't store) --- lib/crowdsec.lua | 21 ++++++-- lib/plugins/crowdsec/http_client.lua | 77 ++++++++++++++-------------- 2 files changed, 54 insertions(+), 44 deletions(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 3668448..5a4759e 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -576,17 +576,23 @@ function csmod.AppSecCheck(ip) end local method = "GET" + local request_body = nil local body = get_body() if body ~= nil then if #body > 0 then method = "POST" + request_body = body if headers["content-length"] == nil then headers["content-length"] = tostring(#body) end end - else + end + + -- Remove content-length header for GET requests + if method == "GET" then headers["content-length"] = nil + headers["Content-Length"] = nil end -- Use pre-created HTTP client object (URL already parsed, connection pooling handled) @@ -597,11 +603,16 @@ function csmod.AppSecCheck(ip) -- AppSec expects requests at the base path from APPSEC_URL -- The incoming request URI is already communicated via APPSEC_URI_HEADER - local res, err = runtime.APPSEC_CLIENT:request_base({ + local request_options = { method = method, - headers = headers, - body = body - }) + headers = headers + } + -- Only include body for POST requests + if request_body ~= nil then + request_options.body = request_body + end + + local res, err = runtime.APPSEC_CLIENT:request_base(request_options) if err ~= nil or not res then ngx.log(ngx.ERR, "Fallback because of err: " .. (err or "unknown")) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index b97060a..7f7f9aa 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -198,7 +198,7 @@ function M.new(url_str, options) if timeouts.send <= 0 then timeouts.send = 3000 end if timeouts.read <= 0 then timeouts.read = 3000 end - -- Create client object with its own httpc instance + -- Create client object (no httpc instance stored - created per request) local client = setmetatable({ url_params = url_params, connection_key = connection_key, @@ -212,29 +212,31 @@ function M.new(url_str, options) api_key_header = options.api_key_header or M.DEFAULT_API_KEY_HEADER, user_agent = options.user_agent, use_tls_auth = use_tls_auth, - httpc = nil, -- HTTP client instance (created on first use) }, Client) return client, nil end ---- Get or create HTTP client instance (internal method) --- Each client object owns its httpc instance +--- Create and connect HTTP client instance (internal method) +-- Following resty.http design: create, use, keepalive (don't store) +-- resty.http handles connection pooling internally via set_keepalive() -- @return httpc: HTTP client object, or nil on error -- @return err: Error message if failed -function Client:_get_httpc() - -- After set_keepalive(), the httpc is in a closed state - -- resty.http will reuse the underlying connection from its pool when we create a new httpc - -- So we always create a new httpc instance here - -- (The actual TCP connection is reused by resty.http internally) - - -- Create new HTTP client instance - self.httpc = http.new() +function Client:_create_httpc() + -- Create new HTTP client instance (resty.http will reuse connections from pool) + local httpc = http.new() + -- Ensure timeouts are valid numbers before setting local connect_timeout = tonumber(self.timeouts.connect) or 3000 local send_timeout = tonumber(self.timeouts.send) or 3000 local read_timeout = tonumber(self.timeouts.read) or 3000 - self.httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) + + -- Validate timeouts are positive + if connect_timeout <= 0 then connect_timeout = 3000 end + if send_timeout <= 0 then send_timeout = 3000 end + if read_timeout <= 0 then read_timeout = 3000 end + + httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) ngx.log(ngx.DEBUG, "[HTTP_CLIENT] Set timeouts: connect=" .. connect_timeout .. "ms, send=" .. send_timeout .. "ms, read=" .. read_timeout .. "ms") -- Connect @@ -260,56 +262,53 @@ function Client:_get_httpc() connect_opts.ssl_client_priv_key = self.ssl_client_priv_key end - local ok, err = self.httpc:connect(connect_opts) + local ok, err = httpc:connect(connect_opts) if not ok then - self.httpc = nil return nil, "Failed to connect: " .. (err or "unknown") end - return self.httpc, nil + return httpc, nil end --- Release HTTP client back to keep-alive pool (internal method) -- Uses resty.http's built-in connection pooling via set_keepalive +-- Following resty.http design: create, use, keepalive (don't store) +-- @param httpc: HTTP client instance to release -- @return ok: boolean indicating success -- @return err: Error message if failed -function Client:_release_httpc() - if not self.httpc then +function Client:_release_httpc(httpc) + if not httpc then return true, nil end -- Check if keepalive_timeout and keepalive_pool_size are set -- If not, just close the connection (no keep-alive) if not self.keepalive_timeout or not self.keepalive_pool_size then - pcall(function() self.httpc:close() end) - self.httpc = nil + pcall(function() httpc:close() end) return true, nil end -- Try to set keepalive - use pcall to safely handle any errors local success, ok, err = pcall(function() - return self.httpc:set_keepalive(self.keepalive_timeout, self.keepalive_pool_size) + return httpc:set_keepalive(self.keepalive_timeout, self.keepalive_pool_size) end) if not success then -- pcall failed - set_keepalive threw an error (ok contains the error message) - pcall(function() self.httpc:close() end) - self.httpc = nil + pcall(function() httpc:close() end) return false, "Failed to set keepalive: " .. tostring(ok) end -- Check if set_keepalive returned success (ok is boolean, err is error message if failed) if not ok then -- set_keepalive returned false - pcall(function() self.httpc:close() end) - self.httpc = nil + pcall(function() httpc:close() end) return false, "Failed to set keepalive: " .. (tostring(err) or "unknown") end -- After set_keepalive(), the httpc object is in a "closed" state but the connection - -- is in resty.http's pool. Clear the reference - we'll create a new httpc on next request, - -- and resty.http will automatically reuse the underlying connection from its pool. - self.httpc = nil + -- is in resty.http's pool. resty.http will automatically reuse the underlying connection + -- from its pool when we create a new httpc on the next request. return true, nil end @@ -429,7 +428,9 @@ end -- @return err: Error message if failed function Client:request(method, path, headers, body) local full_url = self:build_url(path) - local httpc, err = self:_get_httpc() + + -- Create new HTTP client (resty.http will reuse connections from pool) + local httpc, err = self:_create_httpc() if not httpc then return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") end @@ -476,9 +477,8 @@ function Client:request(method, path, headers, body) }) if not res then - -- Request failed, clear httpc so we create a new one next time - pcall(function() self.httpc:close() end) - self.httpc = nil + -- Request failed, close connection + pcall(function() httpc:close() end) return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") end @@ -491,7 +491,7 @@ function Client:request(method, path, headers, body) end -- Return connection to keep-alive pool (resty.http handles pooling) - self:_release_httpc() + self:_release_httpc(httpc) return res, nil end @@ -525,8 +525,8 @@ function Client:request_uri(path, options) local full_path = self:_build_path(path) local full_url = self:build_url(path) - -- Get or create client - local httpc, err = self:_get_httpc() + -- Create new HTTP client (resty.http will reuse connections from pool) + local httpc, err = self:_create_httpc() if not httpc then return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") end @@ -574,9 +574,8 @@ function Client:request_uri(path, options) }) if not res then - -- Request failed, clear httpc so we create a new one next time - pcall(function() self.httpc:close() end) - self.httpc = nil + -- Request failed, close connection + pcall(function() httpc:close() end) return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") end @@ -590,7 +589,7 @@ function Client:request_uri(path, options) end -- Return connection to keep-alive pool (resty.http handles pooling) - self:_release_httpc() + self:_release_httpc(httpc) return res, nil end From cabbfc6b6897e8dac72a5ca452143fac076b7e4e Mon Sep 17 00:00:00 2001 From: Laurence Date: Wed, 7 Jan 2026 09:58:58 +0000 Subject: [PATCH 14/17] Refactor http_client: simplify API, improve error handling, and fix JSON decode issues - Simplified http_client API: all complexity (unix/http/https, mTLS) handled automatically - Removed all logging from http_client module - errors returned to callers - Reordered operations: prepare headers/path before connecting for efficiency - Added proper error handling for JSON decode failures in stream.lua, live.lua, and crowdsec.lua - All callers now explicitly check for errors (err ~= nil or not res) - Fixed stack trace issue when response body is empty or invalid JSON --- lib/crowdsec.lua | 14 +- lib/plugins/crowdsec/http_client.lua | 239 +++++++++++++-------------- lib/plugins/crowdsec/live.lua | 19 ++- lib/plugins/crowdsec/metrics.lua | 2 +- lib/plugins/crowdsec/stream.lua | 16 +- 5 files changed, 159 insertions(+), 131 deletions(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 5a4759e..64e819e 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -625,7 +625,19 @@ function csmod.AppSecCheck(ip) elseif res.status == 403 then ok = false ngx.log(ngx.DEBUG, "Appsec body response: " .. (res.body or "")) - local response = cjson.decode(res.body) + + -- Validate body exists and is not empty before decoding + if not res.body or res.body == "" then + ngx.log(ngx.ERR, "Empty or missing response body from APPSEC (status 403)") + return ok, remediation, status_code, "Empty or missing response body from APPSEC" + end + + local decode_ok, response = pcall(cjson.decode, res.body) + if not decode_ok then + ngx.log(ngx.ERR, "Failed to decode JSON response from APPSEC: " .. tostring(response)) + return ok, remediation, status_code, "Failed to decode JSON response from APPSEC: " .. tostring(response) + end + remediation = response.action if response.http_status ~= nil then ngx.log(ngx.DEBUG, "Got status code from APPSEC: " .. response.http_status) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 7f7f9aa..0268146 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -1,18 +1,44 @@ -- HTTP Client Module --- Provides unified HTTP client with support for: --- - HTTP/HTTPS URLs (http://host:port/path) --- - Unix sockets (unix:/path/to/socket) --- - Connection pooling and keep-alive --- - Both API key and TLS authentication +-- Provides unified HTTP client that automatically handles: +-- - HTTP/HTTPS URLs (http://host:port/path or https://host:port/path) +-- - Unix domain sockets (unix:/path/to/socket or unix:/path/to/socket/http/path) +-- - Connection pooling and keep-alive (via resty.http) +-- - API key authentication (via headers) +-- - Mutual TLS (mTLS) authentication (via client certificates) +-- +-- The client abstracts away all connection details - callers just provide a URL +-- and configuration, then make requests without worrying about the underlying +-- transport mechanism. -- -- Usage: +-- -- Create client once with URL and configuration -- local client = http_client.new("http://example.com:8081", { -- timeouts = {connect=1000, send=1000, read=1000}, -- ssl_verify = true, --- ssl_client_cert = "/path/to/cert", --- ssl_client_priv_key = "/path/to/key" +-- api_key = "your-api-key", +-- user_agent = "my-app/1.0" +-- }) +-- +-- -- For mTLS: +-- local client = http_client.new("https://example.com:8081", { +-- timeouts = {connect=1000, send=1000, read=1000}, +-- ssl_verify = true, +-- ssl_client_cert = parsed_cert_cdata, -- or "/path/to/cert.pem" +-- ssl_client_priv_key = parsed_key_cdata, -- or "/path/to/key.pem" +-- use_tls_auth = true +-- }) +-- +-- -- For Unix sockets: +-- local client = http_client.new("unix:/var/run/crowdsec.sock", { +-- timeouts = {connect=1000, send=1000, read=1000}, +-- api_key = "your-api-key" +-- }) +-- +-- -- Make requests - all complexity is handled automatically +-- local res, err = client:request_uri("/v1/decisions?ip=1.1.1.1", { +-- method = "GET", +-- headers = {["Custom-Header"] = "value"} -- }) --- local res, err = client:request_uri("/v1/decisions?ip=1.1.1.1", {headers={...}}) local http = require "resty.http" local url = require "plugins.crowdsec.url" @@ -217,9 +243,51 @@ function M.new(url_str, options) return client, nil end +--- Prepare headers for request (internal method) +-- Automatically adds Host, User-Agent, and API key headers as needed +-- @param headers table: Existing headers (may be nil) +-- @return table: Headers with required fields added +function Client:_prepare_headers(headers) + headers = headers or {} + + -- Set Host header appropriately for the connection type + if not headers["Host"] and not headers["host"] then + if self.url_params.is_unix then + -- Unix sockets typically use localhost as Host header + headers["Host"] = "localhost" + else + -- For HTTP/HTTPS, use the actual host and port (if non-standard) + local host_header = self.url_params.host + if self.url_params.port and + ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or + (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then + host_header = host_header .. ":" .. self.url_params.port + end + headers["Host"] = host_header + end + end + + -- Add User-Agent if configured + if self.user_agent and not headers["User-Agent"] and not headers["user-agent"] then + headers["User-Agent"] = self.user_agent + end + + -- Add API key header if not using TLS auth + if not self.use_tls_auth and self.api_key then + headers[self.api_key_header] = self.api_key + end + + -- Remove Connection: close header if present (we want keep-alive) + headers["Connection"] = nil + headers["connection"] = nil + + return headers +end + --- Create and connect HTTP client instance (internal method) -- Following resty.http design: create, use, keepalive (don't store) -- resty.http handles connection pooling internally via set_keepalive() +-- The connection details (unix/http/https, mTLS) are automatically handled -- @return httpc: HTTP client object, or nil on error -- @return err: Error message if failed function Client:_create_httpc() @@ -237,9 +305,8 @@ function Client:_create_httpc() if read_timeout <= 0 then read_timeout = 3000 end httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) - ngx.log(ngx.DEBUG, "[HTTP_CLIENT] Set timeouts: connect=" .. connect_timeout .. "ms, send=" .. send_timeout .. "ms, read=" .. read_timeout .. "ms") - -- Connect + -- Build connection options - automatically handles unix/http/https local connect_opts = {} if self.url_params.is_unix then @@ -250,18 +317,22 @@ function Client:_create_httpc() connect_opts.scheme = nil connect_opts.port = nil else + -- For HTTP/HTTPS, use standard connection parameters connect_opts.scheme = self.url_params.scheme connect_opts.host = self.url_params.host connect_opts.port = self.url_params.port end + -- SSL/TLS configuration connect_opts.ssl_verify = self.ssl_verify + -- Add mTLS client certificates if configured if self.ssl_client_cert and self.ssl_client_priv_key then connect_opts.ssl_client_cert = self.ssl_client_cert connect_opts.ssl_client_priv_key = self.ssl_client_priv_key end + -- Connect - resty.http will automatically reuse connections from its pool local ok, err = httpc:connect(connect_opts) if not ok then return nil, "Failed to connect: " .. (err or "unknown") @@ -419,54 +490,31 @@ function Client:build_url(path) end --- Make HTTP request with full control +-- All connection details (unix/http/https, mTLS) are handled automatically -- @param client: Client object (self) -- @param method string: HTTP method (GET, POST, etc.) -- @param path string: Request path --- @param headers table: HTTP headers +-- @param headers table: HTTP headers (optional) -- @param body string: (optional) Request body --- @return res: HTTP response object, or nil on error +-- @return res: HTTP response object with .status and .body, or nil on error -- @return err: Error message if failed function Client:request(method, path, headers, body) local full_url = self:build_url(path) - -- Create new HTTP client (resty.http will reuse connections from pool) + -- Prepare headers (adds Host, User-Agent, API key automatically) + -- This doesn't require a connection, so do it first + headers = self:_prepare_headers(headers) + + -- Build full path (handles base path and query string merging) + -- This doesn't require a connection, so do it before connecting + local full_path = self:_build_path(path) + + -- Create and connect HTTP client (resty.http handles connection pooling) + -- Only connect when we're ready to make the request local httpc, err = self:_create_httpc() if not httpc then return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") end - - -- Set Host header appropriately - if not headers then - headers = {} - end - - if self.url_params.is_unix then - if not headers["Host"] and not headers["host"] then - headers["Host"] = "localhost" - end - else - if not headers["Host"] and not headers["host"] then - local host_header = self.url_params.host - if self.url_params.port and - ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or - (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then - host_header = host_header .. ":" .. self.url_params.port - end - headers["Host"] = host_header - end - end - - -- Add User-Agent if configured - if self.user_agent and not headers["User-Agent"] and not headers["user-agent"] then - headers["User-Agent"] = self.user_agent - end - - -- Add API key header if not using TLS auth - if not self.use_tls_auth and self.api_key then - headers[self.api_key_header] = self.api_key - end - - local full_path = self:_build_path(path) -- Make request local res, err = httpc:request({ @@ -485,11 +533,14 @@ function Client:request(method, path, headers, body) -- Read response body local body_str, err = res:read_body() if err then - ngx.log(ngx.WARN, "Failed to read response body: " .. err) - else - res.body = body_str + -- Failed to read body, return error to caller + -- Return connection to keep-alive pool before returning error + self:_release_httpc(httpc) + return nil, "Failed to read response body: " .. (err or "unknown") end + res.body = body_str + -- Return connection to keep-alive pool (resty.http handles pooling) self:_release_httpc(httpc) @@ -498,100 +549,38 @@ end --- Make HTTP request using only the base path from URL (no additional path needed) -- Useful for services like AppSec that always use the configured base path +-- All connection details are handled automatically -- @param client: Client object (self) -- @param options table: Request options: -- method: string (default: "GET") --- headers: table (HTTP headers) --- body: string (request body) +-- headers: table (HTTP headers, optional) +-- body: string (request body, optional) -- @return res: HTTP response object with .status and .body, or nil on error -- @return err: Error message if failed function Client:request_base(options) return self:request_uri("", options) end ---- Make simple HTTP request (like request_uri) +--- Make HTTP request with path +-- All connection details (unix/http/https, mTLS) are handled automatically -- @param client: Client object (self) -- @param path string: Request path (can include query string) -- @param options table: Request options: -- method: string (default: "GET") --- headers: table (HTTP headers) --- body: string (request body) +-- headers: table (HTTP headers, optional) +-- body: string (request body, optional) -- @return res: HTTP response object with .status and .body, or nil on error -- @return err: Error message if failed function Client:request_uri(path, options) options = options or {} - -- Build full path (include base path and query if configured) - local full_path = self:_build_path(path) - local full_url = self:build_url(path) - - -- Create new HTTP client (resty.http will reuse connections from pool) - local httpc, err = self:_create_httpc() - if not httpc then - return nil, "Failed to connect to " .. full_url .. ": " .. (err or "unknown") - end - - -- Prepare headers - local headers = options.headers or {} - - -- Set Host header appropriately - if self.url_params.is_unix then - if not headers["Host"] and not headers["host"] then - headers["Host"] = "localhost" - end - else - if not headers["Host"] and not headers["host"] then - local host_header = self.url_params.host - if self.url_params.port and - ((self.url_params.scheme == "http" and self.url_params.port ~= 80) or - (self.url_params.scheme == "https" and self.url_params.port ~= 443)) then - host_header = host_header .. ":" .. self.url_params.port - end - headers["Host"] = host_header - end - end - - -- Add User-Agent if configured - if self.user_agent and not headers["User-Agent"] and not headers["user-agent"] then - headers["User-Agent"] = self.user_agent - end - - -- Add API key header if not using TLS auth - if not self.use_tls_auth and self.api_key then - headers[self.api_key_header] = self.api_key - end - - -- Remove Connection: close header if present (we want keep-alive) - headers["Connection"] = nil - headers["connection"] = nil - - -- Make request - local res, err = httpc:request({ - method = options.method or "GET", - path = full_path, - headers = headers, - body = options.body - }) - - if not res then - -- Request failed, close connection - pcall(function() httpc:close() end) - return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") - end - - -- Read response body - local body_str, err = res:read_body() - if err then - ngx.log(ngx.WARN, "Failed to read response body: " .. err) - res.body = "" - else - res.body = body_str - end - - -- Return connection to keep-alive pool (resty.http handles pooling) - self:_release_httpc(httpc) - - return res, nil + -- Use the unified request method which handles everything + return self:request( + options.method or "GET", + path, + options.headers, + options.body + ) end return M diff --git a/lib/plugins/crowdsec/live.lua b/lib/plugins/crowdsec/live.lua index 69b5b4c..53f309f 100644 --- a/lib/plugins/crowdsec/live.lua +++ b/lib/plugins/crowdsec/live.lua @@ -92,7 +92,7 @@ function live:live_query(ip, cache_expiration, bouncing_on_type) method = "GET" }) - if not res then + if err ~= nil or not res then ngx.log(ngx.ERR, "failed to query LAPI: " .. (err or "unknown")) return true, nil, nil, err or "request failed" end @@ -136,7 +136,22 @@ function live:live_query_process(res, ip, cache_expiration, bouncing_on_type, li end return true, nil, nil, nil end - local decision = cjson.decode(body)[1] + + -- Validate body exists and is not empty before decoding + if not body or body == "" then + return true, nil, nil, "Empty or missing response body from LAPI" + end + + local decode_ok, decoded_body = pcall(cjson.decode, body) + if not decode_ok then + return true, nil, nil, "Failed to decode JSON response from LAPI: " .. tostring(decoded_body) + end + + if not decoded_body or #decoded_body == 0 then + return true, nil, nil, "Empty decisions array from LAPI" + end + + local decision = decoded_body[1] if decision.origin == "lists" and decision.scenario ~= nil then decision.origin = "lists:" .. decision.scenario diff --git a/lib/plugins/crowdsec/metrics.lua b/lib/plugins/crowdsec/metrics.lua index 83e00fc..a4e6700 100644 --- a/lib/plugins/crowdsec/metrics.lua +++ b/lib/plugins/crowdsec/metrics.lua @@ -220,7 +220,7 @@ function metrics:sendMetrics(window, headers) body = body }) - if not res then + if err ~= nil or not res then ngx.log(ngx.ERR, "failed to send metrics: " .. (err or "unknown")) else ngx.log(ngx.DEBUG, "metrics status: " .. res.status) diff --git a/lib/plugins/crowdsec/stream.lua b/lib/plugins/crowdsec/stream.lua index d4d0f9f..daa3e08 100644 --- a/lib/plugins/crowdsec/stream.lua +++ b/lib/plugins/crowdsec/stream.lua @@ -209,7 +209,7 @@ function stream:stream_query(bouncing_on_type) method = "GET" }) - if not res then + if err ~= nil or not res then set_refreshing(false) ngx.log(ngx.ERR, "request to crowdsec lapi failed: " .. (err or "unknown")) return err or "request to crowdsec lapi failed" @@ -241,7 +241,19 @@ function stream:stream_query_process(res, bouncing_on_type) return "HTTP error while request to Local API '" .. status .. "' with message (" .. tostring(body) .. ")" end - local decisions = cjson.decode(body) + -- Validate body exists and is not empty before decoding + if not body or body == "" then + set_refreshing(false) + ngx.log(ngx.ERR, "Empty or missing response body from Local API") + return "Empty or missing response body from Local API" + end + + local decode_ok, decisions = pcall(cjson.decode, body) + if not decode_ok then + set_refreshing(false) + ngx.log(ngx.ERR, "Failed to decode JSON response from Local API: " .. tostring(decisions) .. " (body: " .. tostring(body) .. ")") + return "Failed to decode JSON response from Local API: " .. tostring(decisions) + end -- process deleted decisions local deleted = {} From 256748e8fc1ed97535059d91e5ce1ac681e2b087 Mon Sep 17 00:00:00 2001 From: Laurence Date: Wed, 7 Jan 2026 11:58:25 +0000 Subject: [PATCH 15/17] Improve timeout handling and connection management - Remove unused connection_key from client object - Use _release_httpc() consistently (set_keepalive handles closing automatically) - Support single timeout value (applied to all 3) or individual timeouts - Metrics uses resty.http default timeouts (60s) to match original behavior - Stream/Live use REQUEST_TIMEOUT for all three timeouts - AppSec uses individual timeout values (APPSEC_CONNECT/SEND/PROCESS_TIMEOUT) --- lib/plugins/crowdsec/http_client.lua | 84 ++++++++++++++++++---------- lib/plugins/crowdsec/metrics.lua | 14 +---- 2 files changed, 54 insertions(+), 44 deletions(-) diff --git a/lib/plugins/crowdsec/http_client.lua b/lib/plugins/crowdsec/http_client.lua index 0268146..7e16e34 100644 --- a/lib/plugins/crowdsec/http_client.lua +++ b/lib/plugins/crowdsec/http_client.lua @@ -180,7 +180,7 @@ end -- Parse URL once and create a reusable client object -- @param url_str string: URL (http://host:port, https://host:port, or unix:/path) -- @param options table: Client options: --- timeouts: table {connect, send, read} - timeout values in ms +-- timeouts: table {connect, send, read} - timeout values in ms, or a single number to use for all three -- ssl_verify: boolean - whether to verify SSL certificates -- ssl_client_cert: string or cdata - (optional) path to client certificate for mTLS, or parsed PEM object -- ssl_client_priv_key: string or cdata - (optional) path to client private key for mTLS, or parsed PEM object @@ -207,27 +207,47 @@ function M.new(url_str, options) use_tls_auth = (options.ssl_client_cert ~= nil and options.ssl_client_priv_key ~= nil) end - -- Build connection key with mTLS info if applicable - local connection_key = url_params.connection_key - if options.ssl_client_cert and options.ssl_client_priv_key then - connection_key = connection_key .. "|mtls" - end - -- Validate and set timeouts (ensure they're numbers and positive) - local timeouts = options.timeouts or {connect=3000, send=3000, read=3000} - timeouts.connect = tonumber(timeouts.connect) or 3000 - timeouts.send = tonumber(timeouts.send) or 3000 - timeouts.read = tonumber(timeouts.read) or 3000 - - -- Ensure timeouts are positive - if timeouts.connect <= 0 then timeouts.connect = 3000 end - if timeouts.send <= 0 then timeouts.send = 3000 end - if timeouts.read <= 0 then timeouts.read = 3000 end + -- If timeouts are not provided, use nil to let resty.http use its defaults (60 seconds) + -- Support both: single number (applied to all three) or table with individual values + local timeouts = options.timeouts + if timeouts then + -- If a single number is passed, use it for all three timeouts + if type(timeouts) == "number" then + local timeout_value = tonumber(timeouts) + if timeout_value and timeout_value > 0 then + timeouts = { + connect = timeout_value, + send = timeout_value, + read = timeout_value + } + else + timeouts = {} + end + elseif type(timeouts) == "table" then + -- Table with individual timeout values (or missing values) + timeouts.connect = tonumber(timeouts.connect) + timeouts.send = tonumber(timeouts.send) + timeouts.read = tonumber(timeouts.read) + + -- Ensure timeouts are positive if provided + if timeouts.connect and timeouts.connect <= 0 then timeouts.connect = nil end + if timeouts.send and timeouts.send <= 0 then timeouts.send = nil end + if timeouts.read and timeouts.read <= 0 then timeouts.read = nil end + else + -- Invalid type, use defaults + timeouts = {} + end + else + -- No timeouts provided - will use resty.http defaults (60 seconds) + timeouts = {} + end -- Create client object (no httpc instance stored - created per request) + -- Note: resty.http handles connection pooling automatically via set_keepalive() + -- Pool naming is automatic based on connection properties (SSL, proxy, etc.) local client = setmetatable({ url_params = url_params, - connection_key = connection_key, timeouts = timeouts, ssl_verify = options.ssl_verify ~= false, -- default true ssl_client_cert = options.ssl_client_cert, @@ -294,24 +314,26 @@ function Client:_create_httpc() -- Create new HTTP client instance (resty.http will reuse connections from pool) local httpc = http.new() - -- Ensure timeouts are valid numbers before setting - local connect_timeout = tonumber(self.timeouts.connect) or 3000 - local send_timeout = tonumber(self.timeouts.send) or 3000 - local read_timeout = tonumber(self.timeouts.read) or 3000 - - -- Validate timeouts are positive - if connect_timeout <= 0 then connect_timeout = 3000 end - if send_timeout <= 0 then send_timeout = 3000 end - if read_timeout <= 0 then read_timeout = 3000 end - - httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) + -- Set timeouts only if they were provided (otherwise use resty.http defaults: 60 seconds) + if self.timeouts and (self.timeouts.connect or self.timeouts.send or self.timeouts.read) then + local connect_timeout = tonumber(self.timeouts.connect) or 3000 + local send_timeout = tonumber(self.timeouts.send) or 3000 + local read_timeout = tonumber(self.timeouts.read) or 3000 + + -- Validate timeouts are positive + if connect_timeout <= 0 then connect_timeout = 3000 end + if send_timeout <= 0 then send_timeout = 3000 end + if read_timeout <= 0 then read_timeout = 3000 end + + httpc:set_timeouts(connect_timeout, send_timeout, read_timeout) + end + -- If timeouts are nil/not provided, resty.http will use its default (60 seconds) -- Build connection options - automatically handles unix/http/https local connect_opts = {} if self.url_params.is_unix then -- For Unix sockets, resty.http expects the full "unix:/path" in the host field - -- Use connection_key which already contains "unix:/path" connect_opts.host = self.url_params.connection_key -- Explicitly set scheme and port to nil for Unix sockets (resty.http requirement) connect_opts.scheme = nil @@ -525,8 +547,8 @@ function Client:request(method, path, headers, body) }) if not res then - -- Request failed, close connection - pcall(function() httpc:close() end) + -- Request failed, release connection (set_keepalive will close if needed) + self:_release_httpc(httpc) return nil, "Request to " .. full_url .. " failed: " .. (err or "unknown") end diff --git a/lib/plugins/crowdsec/metrics.lua b/lib/plugins/crowdsec/metrics.lua index a4e6700..a6f710c 100644 --- a/lib/plugins/crowdsec/metrics.lua +++ b/lib/plugins/crowdsec/metrics.lua @@ -31,19 +31,7 @@ function metrics:new(userAgent, conf) conf["TLS_CLIENT_CERT_PARSED"] ~= nil and conf["TLS_CLIENT_KEY_PARSED"] ~= nil - -- Ensure REQUEST_TIMEOUT is a valid number, default to 3000ms if not set - local request_timeout = tonumber(conf["REQUEST_TIMEOUT"]) - if not request_timeout or request_timeout <= 0 then - request_timeout = 3000 -- Default to 3 seconds - ngx.log(ngx.WARN, "REQUEST_TIMEOUT not set or invalid, using default: " .. request_timeout .. "ms") - end - local client_options = { - timeouts = { - connect = request_timeout, - send = request_timeout, - read = request_timeout - }, ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], @@ -66,7 +54,7 @@ function metrics:new(userAgent, conf) local client, err = http_client.new(conf["API_URL"], client_options) if client then self.metrics_client = client - ngx.log(ngx.DEBUG, "[METRICS] Created HTTP client with timeouts: connect=" .. request_timeout .. "ms, send=" .. request_timeout .. "ms, read=" .. request_timeout .. "ms") + ngx.log(ngx.DEBUG, "[METRICS] Created HTTP client (using resty.http default timeouts: 60 seconds)") else ngx.log(ngx.WARN, "Failed to create metrics HTTP client: " .. (err or "unknown")) end From ce9eeb2b6b650f8842741de7182cb26a9e43f24c Mon Sep 17 00:00:00 2001 From: Laurence Date: Wed, 7 Jan 2026 12:01:37 +0000 Subject: [PATCH 16/17] Simplify timeout configuration in live and stream - Pass single timeout value instead of table with duplicate values - http_client automatically applies single value to all three timeouts (connect, send, read) --- lib/plugins/crowdsec/live.lua | 6 +----- lib/plugins/crowdsec/stream.lua | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/plugins/crowdsec/live.lua b/lib/plugins/crowdsec/live.lua index 53f309f..2bf4ef0 100644 --- a/lib/plugins/crowdsec/live.lua +++ b/lib/plugins/crowdsec/live.lua @@ -32,11 +32,7 @@ function live:new(conf, user_agent) end local client_options = { - timeouts = { - connect = request_timeout, - send = request_timeout, - read = request_timeout - }, + timeouts = request_timeout, -- Single value applied to all three timeouts ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], diff --git a/lib/plugins/crowdsec/stream.lua b/lib/plugins/crowdsec/stream.lua index daa3e08..11204a9 100644 --- a/lib/plugins/crowdsec/stream.lua +++ b/lib/plugins/crowdsec/stream.lua @@ -149,11 +149,7 @@ function stream:new(conf, user_agent) end local client_options = { - timeouts = { - connect = request_timeout, - send = request_timeout, - read = request_timeout - }, + timeouts = request_timeout, -- Single value applied to all three timeouts ssl_verify = conf["SSL_VERIFY"], keepalive_timeout = conf["KEEPALIVE_TIMEOUT"], keepalive_pool_size = conf["KEEPALIVE_POOL_SIZE"], From f3cc7d1eb2c5f348a8a90a7dae058fe0a9fc2b81 Mon Sep 17 00:00:00 2001 From: Laurence Date: Wed, 7 Jan 2026 14:38:25 +0000 Subject: [PATCH 17/17] Make ALWAYS_SEND_TO_APPSEC boolean conversion explicit - Use explicit if/else pattern matching USE_TLS_AUTH for clarity - Ensures it defaults to false for any value other than "true" - Moved conversion to after TLS certificate parsing for better organization --- lib/crowdsec.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/crowdsec.lua b/lib/crowdsec.lua index 64e819e..438330e 100644 --- a/lib/crowdsec.lua +++ b/lib/crowdsec.lua @@ -209,7 +209,11 @@ function csmod.init(configFile, userAgent) ngx.log(ngx.ERR, "Lua shared dict (crowdsec cache) is full, please increase dict size in config") end - runtime.conf["ALWAYS_SEND_TO_APPSEC"] = runtime.conf["ALWAYS_SEND_TO_APPSEC"] ~= "false" + if runtime.conf["ALWAYS_SEND_TO_APPSEC"] == "true" then + runtime.conf["ALWAYS_SEND_TO_APPSEC"] = true + else + runtime.conf["ALWAYS_SEND_TO_APPSEC"] = false + end runtime.conf["APPSEC_ENABLED"] = false runtime.APPSEC_CLIENT = nil