diff --git a/test/load/.gitignore b/test/load/.gitignore new file mode 100644 index 000000000000..64e5f8011735 --- /dev/null +++ b/test/load/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!script.js +!README.md \ No newline at end of file diff --git a/test/load/README.md b/test/load/README.md index b8483bc3479b..f5230f312b27 100644 --- a/test/load/README.md +++ b/test/load/README.md @@ -1,7 +1,127 @@ -### Load testing +# Load testing -This is a simple load test script set up with k6s. If you have a server running locally, you can run the load test with: +This folder contains a [`k6`](https://grafana.com/docs/k6) load testing script. -```bash -$ k6 run test/load/script.js +## Running the load test + +Open the script, set input constants (environment under test, site domains), then run it: + +```sh +$ docker run --name k6-test -i --rm grafana/k6 run -e AUTH_COOKIE="_plausible_staging=..." -e STATS_API_TOKEN="..." - < ./script.js +``` + +To get a machine readable JSON summary, run it using the command: + +```sh +$ docker run --name k6-test -i grafana/k6 run -e AUTH_COOKIE="_plausible_staging=..." -e STATS_API_TOKEN="..." -e JSON_SUMMARY=true - < ./script.js; docker cp k6-test:/tmp/summary.json ./summary.json; docker rm k6-test +``` + +## Changing the load profile + +To reconfigure the load profile, edit the exported `options` constant. + +## Example output + +The output looks like this: + +``` + █ THRESHOLDS + + http_req_duration{endpoint: "/api/event", domain: "dummy.site/heavy"} + ✗ 'p(95)<1500' p(95)=7.38s + + http_req_duration{endpoint: "/api/event", domain: "dummy.site/light"} + ✗ 'p(95)<1500' p(95)=7.56s + + http_req_duration{endpoint: "/api/health"} + ✓ 'p(95)<300' p(95)=0s + + http_req_duration{endpoint: "/api/stats/:domain/pages", domain: "dummy.site/heavy"} + ✗ 'p(95)<3000' p(95)=5.53s + + http_req_duration{endpoint: "/api/stats/:domain/pages", domain: "dummy.site/light"} + ✗ 'p(95)<1500' p(95)=1.61s + + http_req_duration{endpoint: "/api/system/health/ready"} + ✓ 'p(95)<500' p(95)=226.96ms + + http_req_duration{endpoint: "/api/v2/query", domain: "dummy.site/heavy"} + ✗ 'p(95)<5000' p(95)=7.2s + + http_req_duration{endpoint: "/api/v2/query", domain: "dummy.site/light"} + ✓ 'p(95)<2500' p(95)=1.58s + + http_req_failed{endpoint: "/api/event", domain: "dummy.site/heavy"} + ✗ 'rate<0.01' rate=21.95% + + http_req_failed{endpoint: "/api/event", domain: "dummy.site/light"} + ✗ 'rate<0.01' rate=19.60% + + http_req_failed{endpoint: "/api/health"} + ✗ 'rate<0.01' rate=100.00% + + http_req_failed{endpoint: "/api/stats/:domain/pages", domain: "dummy.site/heavy"} + ✗ 'rate<0.01' rate=14.28% + + http_req_failed{endpoint: "/api/stats/:domain/pages", domain: "dummy.site/light"} + ✗ 'rate<0.01' rate=16.66% + + http_req_failed{endpoint: "/api/system/health/ready"} + ✓ 'rate<0.01' rate=0.00% + + http_req_failed{endpoint: "/api/v2/query", domain: "dummy.site/heavy"} + ✗ 'rate<0.01' rate=33.33% + + http_req_failed{endpoint: "/api/v2/query", domain: "dummy.site/light"} + ✓ 'rate<0.01' rate=0.00% + + + █ TOTAL RESULTS + + checks_total.......: 1132 36.498981/s + checks_succeeded...: 86.92% 984 out of 1132 + checks_failed......: 13.07% 148 out of 1132 + + ✗ request is successful + ↳ 81% — ✓ 13 / ✗ 3 + ✗ is accepted + ↳ 78% — ✓ 432 / ✗ 120 + ✗ is buffered + ↳ 96% — ✓ 534 / ✗ 18 + ✗ request is successful (200) + ↳ 83% — ✓ 5 / ✗ 1 + ✗ request is rate limited (429) + ↳ 0% — ✓ 0 / ✗ 6 + + HTTP + http_req_duration........................................................: avg=1.38s min=0s med=1.06s max=7.78s p(90)=1.88s p(95)=7.38s + { endpoint: "/api/event", domain: "dummy.site/heavy" }.................: avg=1.37s min=0s med=1.05s max=7.77s p(90)=1.86s p(95)=7.38s + { endpoint: "/api/event", domain: "dummy.site/light" }.................: avg=1.54s min=0s med=1.41s max=7.78s p(90)=1.91s p(95)=7.56s + { endpoint: "/api/health" }............................................: avg=0s min=0s med=0s max=0s p(90)=0s p(95)=0s + { endpoint: "/api/stats/:domain/pages", domain: "dummy.site/heavy" }...: avg=1.71s min=279.43ms med=912.92ms max=7.07s p(90)=3.99s p(95)=5.53s + { endpoint: "/api/stats/:domain/pages", domain: "dummy.site/light" }...: avg=738.31ms min=282.4ms med=437.64ms max=1.77s p(90)=1.45s p(95)=1.61s + { endpoint: "/api/system/health/ready" }...............................: avg=209.82ms min=190.77ms med=209.82ms max=228.87ms p(90)=225.06ms p(95)=226.96ms + { endpoint: "/api/v2/query", domain: "dummy.site/heavy" }..............: avg=3.86s min=1.8s med=1.99s max=7.78s p(90)=6.62s p(95)=7.2s + { endpoint: "/api/v2/query", domain: "dummy.site/light" }..............: avg=1s min=371ms med=989.72ms max=1.65s p(90)=1.52s p(95)=1.58s + { expected_response:true }.............................................: avg=1.12s min=111.27ms med=1.19s max=2.15s p(90)=1.8s p(95)=1.86s + http_req_failed..........................................................: 21.60% 124 out of 574 + { endpoint: "/api/event", domain: "dummy.site/heavy" }.................: 21.95% 110 out of 501 + { endpoint: "/api/event", domain: "dummy.site/light" }.................: 19.60% 10 out of 51 + { endpoint: "/api/health" }............................................: 100.00% 1 out of 1 + { endpoint: "/api/stats/:domain/pages", domain: "dummy.site/heavy" }...: 14.28% 1 out of 7 + { endpoint: "/api/stats/:domain/pages", domain: "dummy.site/light" }...: 16.66% 1 out of 6 + { endpoint: "/api/system/health/ready" }...............................: 0.00% 0 out of 2 + { endpoint: "/api/v2/query", domain: "dummy.site/heavy" }..............: 33.33% 1 out of 3 + { endpoint: "/api/v2/query", domain: "dummy.site/light" }..............: 0.00% 0 out of 3 + http_reqs................................................................: 574 18.507434/s + + EXECUTION + iteration_duration.......................................................: avg=6.67s min=332.62ms med=1.92s max=30.01s p(90)=30s p(95)=30s + iterations...............................................................: 574 18.507434/s + vus......................................................................: 48 min=0 max=420 + vus_max..................................................................: 8000 min=6005 max=8000 + + NETWORK + data_received............................................................: 1.7 MB 54 kB/s + data_sent................................................................: 1.0 MB 33 kB/s ``` diff --git a/test/load/script.js b/test/load/script.js index 60799791fe61..085a23175559 100644 --- a/test/load/script.js +++ b/test/load/script.js @@ -1,56 +1,279 @@ import http from "k6/http"; import { check } from "k6"; -const PAYLOAD = JSON.stringify({ - n: "pageview", - u: "http://dummy.site/some-page", - d: "dummy.site", - r: null, - w: 1666, -}); - -function newParams() { - const ip = - Math.floor(Math.random() * 255) + - 1 + - "." + - Math.floor(Math.random() * 255) + - "." + - Math.floor(Math.random() * 255) + - "." + - Math.floor(Math.random() * 255); - - return { - headers: { - "Content-Type": "application/json", - "X-Forwarded-For": ip, - "User-Agent": `${Math.random() > 0.5 ? "Mozilla/5.0" : "Mozilla/4.0"} (${Math.random() > 0.5 ? "Macintosh" : "Windows"}; ${Math.random() > 0.5 ? "Intel Mac OS X 10_15_6" : "Windows NT 10.0"}) AppleWebKit/${Math.floor(Math.random() * 1000) + 500}.36 (KHTML, like Gecko) Chrome/${Math.floor(Math.random() * 100) + 50}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)} Safari/${Math.floor(Math.random() * 1000) + 500}.${Math.floor(Math.random() * 100)} OPR/${Math.floor(Math.random() * 100)}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)}`, - }, - }; +/** INPUTS */ +/** Base URL of the environment under test (env) */ +const baseURL = "http://localhost:8000"; +/** Domain of a site registered in the env */ +const domainHeavy = "dummy.site/heavy"; +/** Domain of a site registered in the env */ +const domainLight = "dummy.site/light"; + +const endpoints = { + track: { + method: "POST", + name: "/api/event", + getUrl: () => `${baseURL}/api/event`, + getBody: ({ domain }) => { + const { n, d } = { + n: "pageview", + d: domain, + }; + const payload = { + d, + n, + u: `https://${domain}/page/${Math.floor(Math.random() * 100) + 1}`, + }; + return JSON.stringify(payload); + }, + getParams: () => { + const n = () => Math.floor(Math.random() * 255); + const ip = [n() + 1, n(), n(), n()].join("."); + return { + headers: { + "Content-Type": "application/json", + "X-Forwarded-For": ip, + "User-Agent": `${Math.random() > 0.5 ? "Mozilla/5.0" : "Mozilla/4.0"} (${Math.random() > 0.5 ? "Macintosh" : "Windows"}; ${Math.random() > 0.5 ? "Intel Mac OS X 10_15_6" : "Windows NT 10.0"}) AppleWebKit/${Math.floor(Math.random() * 1000) + 500}.36 (KHTML, like Gecko) Chrome/${Math.floor(Math.random() * 100) + 50}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)} Safari/${Math.floor(Math.random() * 1000) + 500}.${Math.floor(Math.random() * 100)} OPR/${Math.floor(Math.random() * 100)}.0.${Math.floor(Math.random() * 5000) + 1000}.${Math.floor(Math.random() * 500)}`, + }, + }; + }, + checks: { + "is accepted": (res) => res.body === "ok", + "is buffered": (res) => res.headers["X-Plausible-Dropped"] != 1, + }, + }, + internalApiPages: { + method: "GET", + name: "/api/stats/:domain/pages", + getUrl: ({ domain }) => + `${baseURL}/api/stats/${encodeURIComponent(domain)}/pages?period=all&date=${new Date().toISOString().split("T")[0]}&filters=%5B%5D`, + getParams: () => ({ headers: { Cookie: __ENV.AUTH_COOKIE } }), + checks: { + "request is successful": (res) => res.status === 200, + }, + }, + externalApiQuery: { + method: "POST", + name: "/api/v2/query", + getUrl: () => `${baseURL}/api/v2/query`, + getBody: ({ domain }) => { + const { site_id, metrics, date_range, ...rest } = { + site_id: domain, + // increase complexity / load with harder queries + metrics: ["visitors", "percentage"], + filters: [ + [ + "or", + [ + ["contains", "event:page", ["1"]], + ["not", ["contains", "event:page", ["5", "6", "7", "8", "9"]]], + ], + ], + ], + date_range: "all", + }; + + return JSON.stringify({ site_id, metrics, date_range, ...rest }); + }, + getParams: () => ({ + headers: { + Authorization: `Bearer ${__ENV.STATS_API_TOKEN}`, + "Content-Type": "application/json", + }, + }), + checks: { + "request is successful (200)": (res) => res.status === 200, + "request is rate limited (429)": (res) => res.status === 429, + }, + }, + healthReadiness: { + method: "GET", + name: "/api/system/health/ready", + getUrl: () => `${baseURL}/api/system/health/ready`, + getParams: () => ({}), + checks: { + "request is successful": (res) => res.status === 200, + }, + }, + healthLiveness: { + method: "GET", + name: "/api/health", + getUrl: () => `${baseURL}/api/health`, + getParams: () => ({}), + checks: { + "request is successful": (res) => res.status === 200, + }, + }, +}; + +function makeRequest(endpoint, opts = {}) { + const { domain } = opts; + const res = http.request( + endpoint.method, + endpoint.getUrl({ domain }), + endpoint.getBody ? endpoint.getBody({ domain }) : null, + { + tags: { endpoint: endpoint.name, ...(domain && { domain }) }, + ...endpoint.getParams({ domain }), + }, + ); + + check(res, endpoint.checks); } +export const trackHeavy = () => + makeRequest(endpoints.track, { domain: domainHeavy }); +export const trackLight = () => + makeRequest(endpoints.track, { domain: domainLight }); + +export const readinessCheck = () => makeRequest(endpoints.healthReadiness); +export const livenessCheck = () => makeRequest(endpoints.healthLiveness); + +export const pagesHeavy = () => + makeRequest(endpoints.internalApiPages, { domain: domainHeavy }); +export const pagesLight = () => + makeRequest(endpoints.internalApiPages, { domain: domainLight }); + +export const queryHeavy = () => + makeRequest(endpoints.externalApiQuery, { domain: domainHeavy }); +export const queryLight = () => + makeRequest(endpoints.externalApiQuery, { domain: domainLight }); + +const scenarioOptions = { + executor: "constant-arrival-rate", + timeUnit: "1s", + duration: "120s", +}; + +// configuring thresholds changes what stats are shown in the summary at the end +const selectors = [ + [ + `endpoint: "${endpoints.healthLiveness.name}"`, + [ + ["http_req_duration", ["p(95)<300"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.healthReadiness.name}"`, + [ + ["http_req_duration", ["p(95)<500"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.track.name}", domain: "${domainHeavy}"`, + [ + ["http_req_duration", ["p(95)<1500"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.track.name}", domain: "${domainLight}"`, + [ + ["http_req_duration", ["p(95)<1500"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.internalApiPages.name}", domain: "${domainHeavy}"`, + [ + ["http_req_duration", ["p(95)<3000"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.internalApiPages.name}", domain: "${domainLight}"`, + [ + ["http_req_duration", ["p(95)<1500"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.externalApiQuery.name}", domain: "${domainHeavy}"`, + [ + ["http_req_duration", ["p(95)<5000"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], + [ + `endpoint: "${endpoints.externalApiQuery.name}", domain: "${domainLight}"`, + [ + ["http_req_duration", ["p(95)<2500"]], + ["http_req_failed", ["rate<0.01"]], + ], + ], +]; + export const options = { + // to disable specific requests, comment out those scenarios scenarios: { - constant_rps: { - executor: "constant-arrival-rate", - rate: 6000, - timeUnit: "1s", - duration: "1m", - preAllocatedVUs: 10000, - maxVUs: 30000, + [livenessCheck.name]: { + ...scenarioOptions, + rate: 1, + preAllocatedVUs: 100, + exec: livenessCheck.name, + }, + [readinessCheck.name]: { + ...scenarioOptions, + rate: 1, + preAllocatedVUs: 100, + exec: readinessCheck.name, }, + [trackHeavy.name]: { + ...scenarioOptions, + rate: 500, + preAllocatedVUs: 6000, + exec: trackHeavy.name, + }, + [trackLight.name]: { + ...scenarioOptions, + rate: 50, + preAllocatedVUs: 600, + exec: trackLight.name, + }, + [pagesHeavy.name]: { + ...scenarioOptions, + rate: 6, + preAllocatedVUs: 400, + exec: pagesHeavy.name, + }, + [pagesLight.name]: { + ...scenarioOptions, + rate: 6, + preAllocatedVUs: 400, + exec: pagesLight.name, + }, + [queryHeavy.name]: { + ...scenarioOptions, + rate: 3, + preAllocatedVUs: 200, + exec: queryHeavy.name, + }, + [queryLight.name]: { + ...scenarioOptions, + rate: 3, + preAllocatedVUs: 200, + exec: queryLight.name, + }, + }, + thresholds: { + ...Object.fromEntries( + selectors.flatMap(([selector, thresholds]) => + thresholds.map(([metricName, thresholdValues]) => [ + `${metricName}{${selector}}`, + thresholdValues, + ]), + ), + ), }, }; -export default function () { - const res = http.post( - "http://localhost:8000/api/event", - PAYLOAD, - newParams(), - ); - - check(res, { - "is accepted": (r) => r.body === "ok", - "is buffered": (r) => r.headers["X-Plausible-Dropped"] !== "1", - }); +export function handleSummary(data) { + if (__ENV.JSON_SUMMARY) { + return { + "/tmp/summary.json": JSON.stringify(data), + }; + } + return undefined; // built-in text summary }