diff --git a/api-reference/endpoint/monitor-check-get.mdx b/api-reference/endpoint/monitor-check-get.mdx new file mode 100644 index 00000000..37e4189c --- /dev/null +++ b/api-reference/endpoint/monitor-check-get.mdx @@ -0,0 +1,4 @@ +--- +title: "Get Monitor Check" +openapi: "/api-reference/v2-openapi.json GET /monitor/{monitorId}/checks/{checkId}" +--- diff --git a/api-reference/endpoint/monitor-checks-list.mdx b/api-reference/endpoint/monitor-checks-list.mdx new file mode 100644 index 00000000..9e43cff2 --- /dev/null +++ b/api-reference/endpoint/monitor-checks-list.mdx @@ -0,0 +1,4 @@ +--- +title: "List Monitor Checks" +openapi: "/api-reference/v2-openapi.json GET /monitor/{monitorId}/checks" +--- diff --git a/api-reference/endpoint/monitor-create.mdx b/api-reference/endpoint/monitor-create.mdx new file mode 100644 index 00000000..1cd06bcc --- /dev/null +++ b/api-reference/endpoint/monitor-create.mdx @@ -0,0 +1,4 @@ +--- +title: "Create Monitor" +openapi: "/api-reference/v2-openapi.json POST /monitor" +--- diff --git a/api-reference/endpoint/monitor-delete.mdx b/api-reference/endpoint/monitor-delete.mdx new file mode 100644 index 00000000..acb9c2f0 --- /dev/null +++ b/api-reference/endpoint/monitor-delete.mdx @@ -0,0 +1,4 @@ +--- +title: "Delete Monitor" +openapi: "/api-reference/v2-openapi.json DELETE /monitor/{monitorId}" +--- diff --git a/api-reference/endpoint/monitor-get.mdx b/api-reference/endpoint/monitor-get.mdx new file mode 100644 index 00000000..749d13cf --- /dev/null +++ b/api-reference/endpoint/monitor-get.mdx @@ -0,0 +1,4 @@ +--- +title: "Get Monitor" +openapi: "/api-reference/v2-openapi.json GET /monitor/{monitorId}" +--- diff --git a/api-reference/endpoint/monitor-list.mdx b/api-reference/endpoint/monitor-list.mdx new file mode 100644 index 00000000..df2432d5 --- /dev/null +++ b/api-reference/endpoint/monitor-list.mdx @@ -0,0 +1,4 @@ +--- +title: "List Monitors" +openapi: "/api-reference/v2-openapi.json GET /monitor" +--- diff --git a/api-reference/endpoint/monitor-run.mdx b/api-reference/endpoint/monitor-run.mdx new file mode 100644 index 00000000..f47d5e8c --- /dev/null +++ b/api-reference/endpoint/monitor-run.mdx @@ -0,0 +1,4 @@ +--- +title: "Run Monitor" +openapi: "/api-reference/v2-openapi.json POST /monitor/{monitorId}/run" +--- diff --git a/api-reference/endpoint/monitor-update.mdx b/api-reference/endpoint/monitor-update.mdx new file mode 100644 index 00000000..569f271e --- /dev/null +++ b/api-reference/endpoint/monitor-update.mdx @@ -0,0 +1,4 @@ +--- +title: "Update Monitor" +openapi: "/api-reference/v2-openapi.json PATCH /monitor/{monitorId}" +--- diff --git a/api-reference/endpoint/webhook-monitor-check-completed.mdx b/api-reference/endpoint/webhook-monitor-check-completed.mdx new file mode 100644 index 00000000..c5ae94c3 --- /dev/null +++ b/api-reference/endpoint/webhook-monitor-check-completed.mdx @@ -0,0 +1,4 @@ +--- +title: "Monitor Check Completed" +openapi: "/api-reference/webhooks-openapi.json webhook monitorCheckCompleted" +--- diff --git a/api-reference/endpoint/webhook-monitor-page.mdx b/api-reference/endpoint/webhook-monitor-page.mdx new file mode 100644 index 00000000..6c7d946f --- /dev/null +++ b/api-reference/endpoint/webhook-monitor-page.mdx @@ -0,0 +1,4 @@ +--- +title: "Monitor Page" +openapi: "/api-reference/webhooks-openapi.json webhook monitorPage" +--- diff --git a/api-reference/v2-openapi.json b/api-reference/v2-openapi.json index 49989a5c..925327f5 100644 --- a/api-reference/v2-openapi.json +++ b/api-reference/v2-openapi.json @@ -16,6 +16,392 @@ } ], "paths": { + "/monitor": { + "post": { + "summary": "Create a monitor", + "operationId": "createMonitor", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorCreateRequest" + }, + "examples": { + "scrapeMonitor": { + "summary": "Scrape a URL every 30 minutes", + "value": { + "name": "Blog monitor", + "schedule": { + "text": "every 30 minutes", + "timezone": "UTC" + }, + "notification": { + "email": { + "enabled": true, + "recipients": ["alerts@example.com"], + "includeDiffs": true + } + }, + "targets": [ + { + "type": "scrape", + "urls": ["https://example.com/blog"] + } + ] + } + }, + "crawlMonitor": { + "summary": "Crawl a site on a cron schedule", + "value": { + "name": "Docs monitor", + "schedule": { + "cron": "7-59/15 * * * *", + "timezone": "UTC" + }, + "webhook": { + "url": "https://example.com/webhooks/firecrawl", + "events": ["monitor.page", "monitor.check.completed"] + }, + "targets": [ + { + "type": "crawl", + "url": "https://example.com/docs", + "crawlOptions": { + "limit": 100 + } + } + ] + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Monitor created", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorResponse" + } + } + } + }, + "400": { + "description": "Invalid monitor request" + } + } + }, + "get": { + "summary": "List monitors", + "operationId": "listMonitors", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 25 + } + }, + { + "name": "offset", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0, + "default": 0 + } + } + ], + "responses": { + "200": { + "description": "List of monitors", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorListResponse" + } + } + } + } + } + } + }, + "/monitor/{monitorId}": { + "get": { + "summary": "Get a monitor", + "operationId": "getMonitor", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + } + ], + "responses": { + "200": { + "description": "Monitor details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorResponse" + } + } + } + }, + "404": { + "description": "Monitor not found" + } + } + }, + "patch": { + "summary": "Update a monitor", + "operationId": "updateMonitor", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorUpdateRequest" + }, + "example": { + "schedule": { + "text": "every 15 minutes starting at :07" + }, + "status": "active" + } + } + } + }, + "responses": { + "200": { + "description": "Monitor updated", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorResponse" + } + } + } + }, + "404": { + "description": "Monitor not found" + } + } + }, + "delete": { + "summary": "Delete a monitor", + "operationId": "deleteMonitor", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + } + ], + "responses": { + "200": { + "description": "Monitor deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuccessResponse" + } + } + } + }, + "404": { + "description": "Monitor not found" + } + } + } + }, + "/monitor/{monitorId}/run": { + "post": { + "summary": "Run a monitor", + "operationId": "runMonitor", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + } + ], + "responses": { + "200": { + "description": "Monitor check queued", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorRunResponse" + } + } + } + }, + "409": { + "description": "A monitor check is already running" + } + } + } + }, + "/monitor/{monitorId}/checks": { + "get": { + "summary": "List monitor checks", + "operationId": "listMonitorChecks", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 25 + } + }, + { + "name": "offset", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0, + "default": 0 + } + } + ], + "responses": { + "200": { + "description": "Monitor checks", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorCheckListResponse" + } + } + } + } + } + } + }, + "/monitor/{monitorId}/checks/{checkId}": { + "get": { + "summary": "Get a monitor check", + "operationId": "getMonitorCheck", + "tags": ["Monitoring"], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/MonitorId" + }, + { + "name": "checkId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + }, + "description": "The monitor check ID" + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 25 + } + }, + { + "name": "skip", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0, + "default": 0 + }, + "description": "Number of page results to skip. Use the `next` URL from the previous response for pagination." + }, + { + "name": "status", + "in": "query", + "schema": { + "type": "string", + "enum": ["same", "new", "changed", "removed", "error"] + } + } + ], + "responses": { + "200": { + "description": "Monitor check details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MonitorCheckDetailResponse" + } + } + } + }, + "404": { + "description": "Monitor check not found" + } + } + } + }, "/scrape": { "post": { "summary": "Scrape a single URL and optionally extract information using an LLM", @@ -3641,7 +4027,398 @@ "scheme": "bearer" } }, + "parameters": { + "MonitorId": { + "name": "monitorId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + }, + "description": "The monitor ID" + } + }, "schemas": { + "SuccessResponse": { + "type": "object", + "properties": { + "success": { + "type": "boolean", + "example": true + } + } + }, + "MonitorSchedule": { + "type": "object", + "description": "Schedule for monitor checks. Provide either `cron` or `text`.", + "properties": { + "cron": { + "type": "string", + "description": "Five-field cron expression. Minimum interval is 15 minutes.", + "example": "*/30 * * * *" + }, + "text": { + "type": "string", + "description": "Natural language schedule. Supported examples include `every 30 minutes`, `every 15 minutes starting at :07`, `hourly`, `every 2 hours`, `daily`, `daily at 9:00`, and `weekly`.", + "example": "every 30 minutes" + }, + "timezone": { + "type": "string", + "default": "UTC", + "description": "IANA timezone for the schedule.", + "example": "UTC" + } + } + }, + "MonitorWebhook": { + "type": "object", + "description": "Webhook destination for monitor page and check completion events.", + "properties": { + "url": { + "type": "string", + "format": "uri", + "description": "The URL to send monitor webhooks to." + }, + "headers": { + "type": "object", + "description": "Headers to send to the webhook URL.", + "additionalProperties": { + "type": "string" + } + }, + "metadata": { + "type": "object", + "description": "Custom metadata included in webhook payloads.", + "additionalProperties": true + }, + "events": { + "type": "array", + "description": "Monitor webhook events to receive. Defaults to all monitor events.", + "items": { + "type": "string", + "enum": ["monitor.page", "monitor.check.completed"] + } + } + }, + "required": ["url"] + }, + "MonitorNotification": { + "type": "object", + "properties": { + "email": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "recipients": { + "type": "array", + "maxItems": 25, + "items": { + "type": "string", + "format": "email" + } + }, + "includeDiffs": { + "type": "boolean", + "default": false, + "description": "Include changed page details in email summaries." + } + } + } + } + }, + "MonitorTarget": { + "oneOf": [ + { + "type": "object", + "title": "Scrape target", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "Optional stable ID for this target. Generated if omitted." + }, + "type": { + "type": "string", + "enum": ["scrape"] + }, + "urls": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "format": "uri" + } + }, + "scrapeOptions": { + "$ref": "#/components/schemas/ScrapeOptions" + } + }, + "required": ["type", "urls"] + }, + { + "type": "object", + "title": "Crawl target", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "Optional stable ID for this target. Generated if omitted." + }, + "type": { + "type": "string", + "enum": ["crawl"] + }, + "url": { + "type": "string", + "format": "uri" + }, + "crawlOptions": { + "type": "object", + "description": "Crawl options such as `limit`, `maxDepth`, `includePaths`, and `excludePaths`." + }, + "scrapeOptions": { + "$ref": "#/components/schemas/ScrapeOptions" + } + }, + "required": ["type", "url"] + } + ] + }, + "MonitorCreateRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "maxLength": 256 + }, + "schedule": { + "$ref": "#/components/schemas/MonitorSchedule" + }, + "webhook": { + "$ref": "#/components/schemas/MonitorWebhook" + }, + "notification": { + "$ref": "#/components/schemas/MonitorNotification" + }, + "targets": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "$ref": "#/components/schemas/MonitorTarget" + } + }, + "retentionDays": { + "type": "integer", + "minimum": 1, + "maximum": 365, + "default": 30 + } + }, + "required": ["name", "schedule", "targets"] + }, + "MonitorUpdateRequest": { + "type": "object", + "description": "Partial monitor update payload. Include at least one field.", + "properties": { + "name": { + "type": "string", + "maxLength": 256 + }, + "schedule": { + "$ref": "#/components/schemas/MonitorSchedule" + }, + "webhook": { + "$ref": "#/components/schemas/MonitorWebhook" + }, + "notification": { + "$ref": "#/components/schemas/MonitorNotification" + }, + "targets": { + "type": "array", + "minItems": 1, + "maxItems": 50, + "items": { + "$ref": "#/components/schemas/MonitorTarget" + } + }, + "retentionDays": { + "type": "integer", + "minimum": 1, + "maximum": 365 + }, + "status": { + "type": "string", + "enum": ["active", "paused"] + } + } + }, + "MonitorSummary": { + "type": "object", + "properties": { + "totalPages": { "type": "integer" }, + "same": { "type": "integer" }, + "changed": { "type": "integer" }, + "new": { "type": "integer" }, + "removed": { "type": "integer" }, + "error": { "type": "integer" } + } + }, + "Monitor": { + "type": "object", + "properties": { + "id": { "type": "string", "format": "uuid" }, + "name": { "type": "string" }, + "status": { "type": "string", "enum": ["active", "paused", "deleted"] }, + "schedule": { + "type": "object", + "properties": { + "cron": { "type": "string" }, + "timezone": { "type": "string" } + } + }, + "nextRunAt": { "type": "string", "format": "date-time", "nullable": true }, + "lastRunAt": { "type": "string", "format": "date-time", "nullable": true }, + "currentCheckId": { "type": "string", "format": "uuid", "nullable": true }, + "targets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MonitorTarget" + } + }, + "webhook": { + "$ref": "#/components/schemas/MonitorWebhook" + }, + "notification": { + "$ref": "#/components/schemas/MonitorNotification" + }, + "retentionDays": { "type": "integer" }, + "estimatedCreditsPerMonth": { "type": "integer", "nullable": true }, + "lastCheckSummary": { + "$ref": "#/components/schemas/MonitorSummary" + }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + }, + "MonitorCheck": { + "type": "object", + "properties": { + "id": { "type": "string", "format": "uuid" }, + "monitorId": { "type": "string", "format": "uuid" }, + "status": { + "type": "string", + "enum": ["queued", "running", "completed", "failed", "partial", "skipped_overlap"] + }, + "trigger": { "type": "string", "enum": ["scheduled", "manual"] }, + "scheduledFor": { "type": "string", "format": "date-time", "nullable": true }, + "startedAt": { "type": "string", "format": "date-time", "nullable": true }, + "finishedAt": { "type": "string", "format": "date-time", "nullable": true }, + "estimatedCredits": { "type": "integer", "nullable": true }, + "reservedCredits": { "type": "integer", "nullable": true }, + "actualCredits": { "type": "integer", "nullable": true }, + "billingStatus": { "type": "string" }, + "summary": { "$ref": "#/components/schemas/MonitorSummary" }, + "targetResults": { "type": "array", "nullable": true }, + "notificationStatus": { "type": "object", "nullable": true }, + "error": { "type": "string", "nullable": true }, + "createdAt": { "type": "string", "format": "date-time" }, + "updatedAt": { "type": "string", "format": "date-time" } + } + }, + "MonitorCheckPage": { + "type": "object", + "properties": { + "id": { "type": "string", "format": "uuid" }, + "targetId": { "type": "string" }, + "url": { "type": "string", "format": "uri" }, + "status": { "type": "string", "enum": ["same", "new", "changed", "removed", "error"] }, + "previousScrapeId": { "type": "string", "format": "uuid", "nullable": true }, + "currentScrapeId": { "type": "string", "format": "uuid", "nullable": true }, + "statusCode": { "type": "integer", "nullable": true }, + "error": { "type": "string", "nullable": true }, + "metadata": { "type": "object", "nullable": true }, + "diff": { + "type": "object", + "nullable": true, + "description": "Inline diff artifact when available.", + "properties": { + "text": { "type": "string" }, + "json": { "type": "object" } + } + }, + "createdAt": { "type": "string", "format": "date-time" } + } + }, + "MonitorResponse": { + "type": "object", + "properties": { + "success": { "type": "boolean" }, + "data": { "$ref": "#/components/schemas/Monitor" } + } + }, + "MonitorListResponse": { + "type": "object", + "properties": { + "success": { "type": "boolean" }, + "data": { + "type": "array", + "items": { "$ref": "#/components/schemas/Monitor" } + } + } + }, + "MonitorRunResponse": { + "type": "object", + "properties": { + "success": { "type": "boolean" }, + "id": { "type": "string", "format": "uuid" }, + "data": { "$ref": "#/components/schemas/MonitorCheck" } + } + }, + "MonitorCheckListResponse": { + "type": "object", + "properties": { + "success": { "type": "boolean" }, + "data": { + "type": "array", + "items": { "$ref": "#/components/schemas/MonitorCheck" } + } + } + }, + "MonitorCheckDetailResponse": { + "type": "object", + "properties": { + "success": { "type": "boolean" }, + "next": { + "type": "string", + "nullable": true, + "description": "URL to fetch the next page of monitor check page results, if any." + }, + "data": { + "allOf": [ + { "$ref": "#/components/schemas/MonitorCheck" }, + { + "type": "object", + "properties": { + "pages": { + "type": "array", + "items": { "$ref": "#/components/schemas/MonitorCheckPage" } + }, + "next": { + "type": "string", + "nullable": true, + "description": "URL to fetch the next page of monitor check page results, if any." + } + } + } + ] + } + } + }, "Formats": { "type": "array", "items": { diff --git a/api-reference/webhooks-openapi.json b/api-reference/webhooks-openapi.json index 2598dc6b..34c83e9c 100644 --- a/api-reference/webhooks-openapi.json +++ b/api-reference/webhooks-openapi.json @@ -912,6 +912,176 @@ } } } + }, + "monitorPage": { + "post": { + "summary": "Monitor Page", + "operationId": "monitorPage", + "tags": [ + "Monitor" + ], + "parameters": [ + { + "$ref": "#/components/parameters/FirecrawlSignature" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "success", + "type", + "id", + "webhookId", + "data" + ], + "properties": { + "success": { + "type": "boolean", + "description": "`true` when the page scrape completed without an error." + }, + "type": { + "type": "string", + "description": "The event type.", + "const": "monitor.page" + }, + "id": { + "type": "string", + "format": "uuid", + "description": "The monitor check ID." + }, + "webhookId": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for this webhook delivery." + }, + "data": { + "$ref": "#/components/schemas/MonitorPageData" + }, + "error": { + "type": "string", + "description": "Error message when the monitored page failed." + }, + "metadata": { + "$ref": "#/components/schemas/WebhookMetadata" + } + } + }, + "example": { + "success": true, + "type": "monitor.page", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0000-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "url": "https://example.com/blog", + "status": "changed", + "previousScrapeId": "019df94f-82c3-7e41-81f0-00c72b2d9c52", + "currentScrapeId": "019df960-73ee-7ac2-97a9-fb0e442c21f1", + "error": null + }, + "metadata": {} + } + } + } + }, + "responses": { + "200": { + "description": "Return any `2xx` status code to acknowledge receipt." + } + } + } + }, + "monitorCheckCompleted": { + "post": { + "summary": "Monitor Check Completed", + "operationId": "monitorCheckCompleted", + "tags": [ + "Monitor" + ], + "parameters": [ + { + "$ref": "#/components/parameters/FirecrawlSignature" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "success", + "type", + "id", + "webhookId", + "data" + ], + "properties": { + "success": { + "type": "boolean", + "description": "`true` when the check completed without page errors, otherwise `false`." + }, + "type": { + "type": "string", + "description": "The event type.", + "const": "monitor.check.completed" + }, + "id": { + "type": "string", + "format": "uuid", + "description": "The monitor check ID." + }, + "webhookId": { + "type": "string", + "format": "uuid", + "description": "Unique identifier for this webhook delivery." + }, + "data": { + "$ref": "#/components/schemas/MonitorCheckCompletedData" + }, + "error": { + "type": "string", + "description": "Error message when the check failed." + }, + "metadata": { + "$ref": "#/components/schemas/WebhookMetadata" + } + } + }, + "example": { + "success": true, + "type": "monitor.check.completed", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0001-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "status": "completed", + "summary": { + "totalPages": 2, + "same": 1, + "changed": 1, + "new": 0, + "removed": 0, + "error": 0 + } + }, + "metadata": {} + } + } + } + }, + "responses": { + "200": { + "description": "Return any `2xx` status code to acknowledge receipt." + } + } + } } }, "components": { @@ -985,6 +1155,106 @@ } } } + }, + "MonitorPageData": { + "type": "object", + "required": [ + "monitorId", + "checkId", + "url", + "status" + ], + "properties": { + "monitorId": { + "type": "string", + "format": "uuid", + "description": "The monitor ID." + }, + "checkId": { + "type": "string", + "format": "uuid", + "description": "The monitor check ID." + }, + "url": { + "type": "string", + "format": "uri", + "description": "The page URL." + }, + "status": { + "type": "string", + "enum": [ + "same", + "new", + "changed", + "error" + ], + "description": "Page-level status for this scrape completion." + }, + "previousScrapeId": { + "type": "string", + "format": "uuid", + "nullable": true + }, + "currentScrapeId": { + "type": "string", + "format": "uuid", + "nullable": true + }, + "error": { + "type": "string", + "nullable": true + } + } + }, + "MonitorCheckCompletedData": { + "type": "object", + "required": [ + "monitorId", + "checkId", + "status", + "summary" + ], + "properties": { + "monitorId": { + "type": "string", + "format": "uuid", + "description": "The monitor ID." + }, + "checkId": { + "type": "string", + "format": "uuid", + "description": "The monitor check ID." + }, + "status": { + "type": "string", + "enum": [ + "completed", + "partial", + "failed", + "skipped_overlap" + ], + "description": "Final status of the monitor check." + }, + "summary": { + "type": "object", + "required": [ + "totalPages", + "same", + "changed", + "new", + "removed", + "error" + ], + "properties": { + "totalPages": { "type": "integer" }, + "same": { "type": "integer" }, + "changed": { "type": "integer" }, + "new": { "type": "integer" }, + "removed": { "type": "integer" }, + "error": { "type": "integer" } + } + } + } } } } diff --git a/docs.json b/docs.json index 6ec9b7ab..46d73861 100755 --- a/docs.json +++ b/docs.json @@ -82,6 +82,7 @@ "features/parse", "features/map", "features/crawl", + "features/monitoring", { "group": "Agent (Research Preview)", "icon": "robot", @@ -368,6 +369,19 @@ "api-reference/endpoint/crawl-active" ] }, + { + "group": "Monitor Endpoints", + "pages": [ + "api-reference/endpoint/monitor-create", + "api-reference/endpoint/monitor-list", + "api-reference/endpoint/monitor-get", + "api-reference/endpoint/monitor-update", + "api-reference/endpoint/monitor-delete", + "api-reference/endpoint/monitor-run", + "api-reference/endpoint/monitor-checks-list", + "api-reference/endpoint/monitor-check-get" + ] + }, { "group": "Agent Endpoints", "pages": [ @@ -413,6 +427,13 @@ "api-reference/endpoint/webhook-batch-scrape-completed" ] }, + { + "group": "Monitor", + "pages": [ + "api-reference/endpoint/webhook-monitor-page", + "api-reference/endpoint/webhook-monitor-check-completed" + ] + }, { "group": "Agent", "pages": [ diff --git a/features/monitoring.mdx b/features/monitoring.mdx new file mode 100644 index 00000000..4c3d575b --- /dev/null +++ b/features/monitoring.mdx @@ -0,0 +1,295 @@ +--- +title: "Monitoring" +description: "Schedule recurring scrapes and crawls, detect content changes, and receive webhook or email notifications" +og:title: "Monitoring | Firecrawl" +og:description: "Schedule recurring scrapes and crawls, detect content changes, and receive webhook or email notifications" +icon: "radar" +--- + +Firecrawl monitors run recurring scrapes or crawls and compare each result against the last retained snapshot. Use monitors to track product pages, docs, blogs, changelogs, competitor sites, or any page where changes matter. + +Each check records page-level results as `same`, `new`, `changed`, `removed`, or `error`. You can receive a webhook as each monitored page finishes, a webhook for every completed check, email summaries when changes or errors happen, or any combination of those notifications. + + + Monitoring requires retained snapshots and diffs. It is not available for zero data retention teams. + + +## Create a monitor + +Create a scrape monitor for one or more explicit URLs: + +```bash +curl -X POST "https://api.firecrawl.dev/v2/monitor" \ + -H "Authorization: Bearer $FIRECRAWL_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Blog monitor", + "schedule": { + "text": "every 30 minutes", + "timezone": "UTC" + }, + "notification": { + "email": { + "enabled": true, + "recipients": ["alerts@example.com"], + "includeDiffs": true + } + }, + "targets": [ + { + "type": "scrape", + "urls": ["https://example.com/blog"] + } + ] + }' +``` + +Create a crawl monitor to diff every page discovered by a crawl on each check: + +```bash +curl -X POST "https://api.firecrawl.dev/v2/monitor" \ + -H "Authorization: Bearer $FIRECRAWL_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Docs monitor", + "schedule": { + "cron": "7-59/15 * * * *", + "timezone": "UTC" + }, + "webhook": { + "url": "https://example.com/webhooks/firecrawl", + "events": ["monitor.page", "monitor.check.completed"] + }, + "targets": [ + { + "type": "crawl", + "url": "https://example.com/docs", + "crawlOptions": { + "limit": 100, + "maxDepth": 3 + } + } + ] + }' +``` + +## Schedules + +Schedules can be provided as cron or as simple natural language text. + +```json +{ + "schedule": { + "cron": "*/30 * * * *", + "timezone": "UTC" + } +} +``` + +```json +{ + "schedule": { + "text": "every 30 minutes", + "timezone": "UTC" + } +} +``` + +Supported natural language examples: + +- `every 30 minutes` +- `every 15 minutes starting at :07` +- `hourly` +- `every 2 hours` +- `daily` +- `daily at 9:00` +- `weekly` + +The minimum interval is 15 minutes. API responses always return the normalized cron expression. + +## Targets + +Monitors support two target types: + +- `scrape`: Runs one scrape per URL in `urls`. +- `crawl`: Runs a full crawl for `url` on each check, then diffs all discovered pages. + +Target scrape options are passed through to the underlying scrape jobs. Monitor-triggered scrapes default `maxAge` to `0`, so each check performs a fresh scrape unless you explicitly set a different `maxAge`. + +```json +{ + "type": "scrape", + "urls": ["https://example.com/pricing"], + "scrapeOptions": { + "formats": ["markdown"], + "maxAge": 0 + } +} +``` + +For crawl targets, use `crawlOptions` for crawl behavior and `scrapeOptions` for each page scrape: + +```json +{ + "type": "crawl", + "url": "https://example.com/docs", + "crawlOptions": { + "limit": 100, + "includePaths": ["/docs"] + }, + "scrapeOptions": { + "formats": ["markdown"] + } +} +``` + +## Notifications + +### Webhooks + +When a monitor has a `webhook`, Firecrawl can send two monitor events: + +- `monitor.page`: Sent as each monitored scrape finishes in the scrape worker. +- `monitor.check.completed`: Sent after the full check is reconciled. Includes check status and summary counts. Use `monitor.page` events or the monitor check API for page-level results. + +```json +{ + "webhook": { + "url": "https://example.com/webhooks/firecrawl", + "headers": { + "Authorization": "Bearer your-secret" + }, + "metadata": { + "environment": "production" + }, + "events": ["monitor.page", "monitor.check.completed"] + } +} +``` + +`monitor.page` payload: + +```json +{ + "success": true, + "type": "monitor.page", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0000-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "url": "https://example.com/blog", + "status": "changed", + "previousScrapeId": "019df94f-82c3-7e41-81f0-00c72b2d9c52", + "currentScrapeId": "019df960-73ee-7ac2-97a9-fb0e442c21f1", + "error": null + }, + "metadata": { + "environment": "production" + } +} +``` + +`monitor.check.completed` payload: + +```json +{ + "success": true, + "type": "monitor.check.completed", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0001-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "status": "completed", + "summary": { + "totalPages": 2, + "same": 1, + "changed": 1, + "new": 0, + "removed": 0, + "error": 0 + } + }, + "metadata": { + "environment": "production" + } +} +``` + +`success` is `true` when the check completed without page errors. It is `false` for failed or partial checks, and `error` contains the failure reason when available. + +### Email + +Email summaries are sent only when a check has changed, new, removed, or errored pages. + +```json +{ + "notification": { + "email": { + "enabled": true, + "recipients": ["alerts@example.com"], + "includeDiffs": true + } + } +} +``` + +If `recipients` is omitted, Firecrawl sends to team members who are eligible for system alert emails. + +## Check results + +Use `GET /v2/monitor/{monitorId}/checks` to list checks and `GET /v2/monitor/{monitorId}/checks/{checkId}` to inspect a check. + +```bash +curl "https://api.firecrawl.dev/v2/monitor/$MONITOR_ID/checks/$CHECK_ID?limit=25" \ + -H "Authorization: Bearer $FIRECRAWL_API_KEY" +``` + +The check detail response includes summary counts and a paginated `pages` array. Use the top-level `next` URL to fetch the next page of results, matching crawl pagination. Changed pages include inline diff data when available. + +```json +{ + "success": true, + "next": "https://api.firecrawl.dev/v2/monitor/019df960-06e7-7383-9d89-82c0113dc31a/checks/019df960-5f2a-75fb-a98b-bd2d32ca67d4?skip=25&limit=25", + "data": { + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "status": "completed", + "summary": { + "totalPages": 1, + "same": 0, + "changed": 1, + "new": 0, + "removed": 0, + "error": 0 + }, + "pages": [ + { + "url": "https://example.com/blog", + "status": "changed", + "previousScrapeId": "019df94f-82c3-7e41-81f0-00c72b2d9c52", + "currentScrapeId": "019df960-73ee-7ac2-97a9-fb0e442c21f1", + "diff": { + "text": "...", + "json": {} + } + } + ], + "next": "https://api.firecrawl.dev/v2/monitor/019df960-06e7-7383-9d89-82c0113dc31a/checks/019df960-5f2a-75fb-a98b-bd2d32ca67d4?skip=25&limit=25" + } +} +``` + +## API reference + +- [Create monitor](/api-reference/endpoint/monitor-create) +- [List monitors](/api-reference/endpoint/monitor-list) +- [Get monitor](/api-reference/endpoint/monitor-get) +- [Update monitor](/api-reference/endpoint/monitor-update) +- [Delete monitor](/api-reference/endpoint/monitor-delete) +- [Run monitor](/api-reference/endpoint/monitor-run) +- [List monitor checks](/api-reference/endpoint/monitor-checks-list) +- [Get monitor check](/api-reference/endpoint/monitor-check-get) +- [Monitor page webhook payload](/api-reference/endpoint/webhook-monitor-page) +- [Monitor check completed webhook payload](/api-reference/endpoint/webhook-monitor-check-completed) diff --git a/webhooks/events.mdx b/webhooks/events.mdx index 16a37e9f..8f024fc2 100644 --- a/webhooks/events.mdx +++ b/webhooks/events.mdx @@ -26,6 +26,8 @@ Firecrawl sends webhook events at each stage of a job's lifecycle, so you can tr | `agent.completed` | Agent finishes successfully | | `agent.failed` | Agent encounters an error | | `agent.cancelled` | Agent job is cancelled by the user | +| `monitor.page` | A monitored page scrape finishes | +| `monitor.check.completed` | Monitor check finishes and page-level changes are available | ## Payload Structure @@ -46,7 +48,7 @@ All webhook events share this structure: | `success` | boolean | Whether the operation succeeded | | `type` | string | Event type (e.g. `crawl.page`) | | `id` | string | Job ID | -| `data` | array | Event-specific data (see examples below) | +| `data` | array or object | Event-specific data (see examples below) | | `metadata` | object | Custom metadata from your webhook config | | `error` | string | Error message (when `success` is `false`) | @@ -172,6 +174,60 @@ Sent when all URLs in the batch have been processed. } ``` +## Monitor Events + +### `monitor.page` + +Sent as each monitored page scrape finishes. This event is emitted from the scrape worker path, so it arrives before the full monitor check is reconciled. + +```json +{ + "success": true, + "type": "monitor.page", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0000-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "url": "https://example.com/blog", + "status": "changed", + "previousScrapeId": "019df94f-82c3-7e41-81f0-00c72b2d9c52", + "currentScrapeId": "019df960-73ee-7ac2-97a9-fb0e442c21f1", + "error": null + }, + "metadata": {} +} +``` + +### `monitor.check.completed` + +Sent when a monitor check finishes. The `data` object contains check status and summary counts. Page-level results are only sent through `monitor.page` events or returned from the monitor check API. + +```json +{ + "success": true, + "type": "monitor.check.completed", + "id": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "webhookId": "f1e2d3c4-0001-0000-0000-000000000000", + "data": { + "monitorId": "019df960-06e7-7383-9d89-82c0113dc31a", + "checkId": "019df960-5f2a-75fb-a98b-bd2d32ca67d4", + "status": "completed", + "summary": { + "totalPages": 2, + "same": 1, + "changed": 1, + "new": 0, + "removed": 0, + "error": 0 + } + }, + "metadata": {} +} +``` + +`success` is `true` when the check completed without page errors. For partial or failed checks, `success` is `false` and `error` may contain a message. + ## Extract Events ### `extract.started` diff --git a/webhooks/overview.mdx b/webhooks/overview.mdx index cb99b53e..d1728b95 100644 --- a/webhooks/overview.mdx +++ b/webhooks/overview.mdx @@ -10,7 +10,7 @@ import CrawlWebhookCURL from "/snippets/v2/crawl-webhook/base/curl.mdx"; import BatchScrapeWebhookCURL from "/snippets/v2/batch-scrape-webhook/base/curl.mdx"; import WebhookConfigBasic from "/snippets/v2/webhook-config/basic/json.mdx"; -Get notified the moment a crawl, batch scrape, extract, or agent job starts, progresses, or finishes. Instead of polling for status, you provide an HTTPS endpoint and Firecrawl delivers events to it in real time. +Get notified the moment a crawl, batch scrape, extract, agent job, or monitor check starts, progresses, or finishes. Instead of polling for status, you provide an HTTPS endpoint and Firecrawl delivers events to it in real time. ## Supported Operations @@ -20,6 +20,7 @@ Get notified the moment a crawl, batch scrape, extract, or agent job starts, pro | Batch Scrape | `started`, `page`, `completed` | | Extract | `started`, `completed`, `failed` | | Agent | `started`, `action`, `completed`, `failed`, `cancelled` | +| Monitor | `check.completed` | See [Event Types](/webhooks/events) for full payload details and examples.