Skip to content

Initial commit of Dataminr Pulse integration for elastic #17886

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pgifford-dataminr wants to merge 2 commits into
base: main
Choose a base branch
from
+8,476 −0
Open

Initial commit of Dataminr Pulse integration for elastic #17886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3d13da4
Initial commit of Dataminr Pulse integration for elastic
pgifford-dataminr Mar 18, 2026
4ddde83
Addressed PR comments
TimofteA Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@
/packages/cyera @elastic/security-service-integrations
/packages/cylance @elastic/security-service-integrations
/packages/darktrace @elastic/security-service-integrations
/packages/dataminr_pulse @elastic/security-external-integrations
/packages/ded @elastic/sec-applied-ml @elastic/kibana-management
/packages/dga @elastic/sec-applied-ml
/packages/digital_guardian @elastic/security-service-integrations
Expand Down
284 changes: 284 additions & 0 deletions packages/dataminr_pulse/README.md
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions packages/dataminr_pulse/_dev/build/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
dependencies:
ecs:
reference: git@v8.11.0
223 changes: 223 additions & 0 deletions packages/dataminr_pulse/_devtests/program.cel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
timestamp(now).as(fetch_ts,
[408, 429, 500, 502, 503, 504, 401].as(retryable,
(state.access_token == "").as(no_token,
(int(state.expires_at) > 0).as(has_expiry,
(has_expiry ?
fetch_ts >= (timestamp("1970-01-01T00:00:00Z") +
duration(string(int(state.expires_at) - 300000) + "ms"))
: true
).as(is_expired_or_unknown,
(no_token || is_expired_or_unknown).as(need_auth,
need_auth ?
post(
"https://userauth.dataminr.com/auth/2/token",
"application/x-www-form-urlencoded",
{
"grant_type": ["api_key"],
"client_id": [state.client_id],
"client_secret": [state.client_secret]
}.format_query()
).as(auth_resp,
auth_resp.StatusCode == 200 ?
(size(auth_resp.Body) > 0 ? bytes(auth_resp.Body).decode_json() : {}).as(token_data,
has(token_data.dmaToken) && token_data.dmaToken != "" ?
{
"updated_state": state.with({
"access_token": string(token_data.dmaToken),
"expires_at": has(token_data.expire) ? int(token_data.expire) : 0
}),
"auth_event": {
"message": {
"log_type": "auth",
"status": "success",
"client_id": state.client_id,
"expires_at": has(token_data.expire) ? int(token_data.expire) : 0,
"http_status_code": auth_resp.StatusCode,
"fetch_timestamp": fetch_ts
}.encode_json()
},
"should_fetch": true,
"auth_failed": false
}
:
{
"updated_state": state,
"auth_event": {
"message": {
"log_type": "auth",
"status": "failed",
"client_id": state.client_id,
"error_message": "No valid token received",
"http_status_code": auth_resp.StatusCode,
"fetch_timestamp": fetch_ts,
"response_body": string(auth_resp.Body)
}.encode_json()
},
"should_fetch": false,
"auth_failed": true
}
)
:
(size(auth_resp.Body) > 0 ? bytes(auth_resp.Body).decode_json() : {}).as(error_body,
retryable.exists(code, code == auth_resp.StatusCode).as(is_retryable,
(has(error_body.errors) && size(error_body.errors) > 0 ?
string(error_body.errors[0].message) :
"Authentication failed with HTTP " + string(auth_resp.StatusCode)
).as(error_msg,
is_retryable ?
{
"updated_state": state,
"auth_event": {
"message": {
"log_type": "auth",
"status": "retry",
"client_id": state.client_id,
"error_message": error_msg,
"error_code": has(error_body.errors) && size(error_body.errors) > 0 ? int(error_body.errors[0].code) : 0,
"http_status_code": auth_resp.StatusCode,
"fetch_timestamp": fetch_ts
}.encode_json()
},
"should_fetch": false,
"auth_failed": false,
"want_more": true
}
:
{
"updated_state": state,
"auth_event": {
"message": {
"log_type": "auth",
"status": "failed",
"client_id": state.client_id,
"error_message": error_msg,
"error_code": has(error_body.errors) && size(error_body.errors) > 0 ? int(error_body.errors[0].code) : 0,
"http_status_code": auth_resp.StatusCode,
"fetch_timestamp": fetch_ts,
"response_body": string(auth_resp.Body)
}.encode_json()
},
"should_fetch": false,
"auth_failed": true,
"want_more": false
}
)
)
)
)
:
{
"updated_state": state,
"auth_event": {},
"should_fetch": true,
"auth_failed": false
}
).as(prep,
prep.should_fetch ?
(prep.updated_state.next_cursor != "" ?
prep.updated_state.base_url + prep.updated_state.next_cursor :
prep.updated_state.url + "?" + {"pageSize": [string(prep.updated_state.page_size)]}.format_query()
).as(request_url,
get_request(request_url).with({
"Header": {
"Authorization": ["Bearer " + prep.updated_state.access_token],
"Accept": ["application/json"],
"x-application-name": ["elastic-siem-" + state.version]
}
}).do_request().as(api_resp,
api_resp.StatusCode == 200 ?
(size(api_resp.Body) > 0 ? bytes(api_resp.Body).decode_json() : {}).as(response_body,
prep.updated_state.with({
"events": (has(prep.auth_event.message) ? [prep.auth_event] : []) +
(has(response_body.alerts) && size(response_body.alerts) > 0 ?
response_body.alerts.map(alert, {
"message": alert.with({
"cursor_used": prep.updated_state.next_cursor != "" ? prep.updated_state.next_cursor : "initial",
"fetched_alerts": size(response_body.alerts),
"fetch_timestamp": fetch_ts
}).encode_json()
}) : []
) + [{
"message": {
"log_type": "alert-fetch",
"status": "success",
"api_endpoint": request_url,
"input_alert_fetch_count": prep.updated_state.page_size,
"next_cursor": has(response_body.nextPage) && response_body.nextPage != "" ? string(response_body.nextPage) : "",
"count_of_alerts": has(response_body.alerts) ? size(response_body.alerts) : 0,
"http_status_code": api_resp.StatusCode,
"cursor_used": prep.updated_state.next_cursor != "" ? prep.updated_state.next_cursor : "initial",
"fetched_alerts": has(response_body.alerts) ? size(response_body.alerts) : 0,
"fetch_timestamp": fetch_ts
}.encode_json()
}],
"next_cursor": has(response_body.nextPage) && response_body.nextPage != "" ? string(response_body.nextPage) : "",
"access_token": prep.updated_state.access_token,
"expires_at": prep.updated_state.expires_at,
"last_fetch_timestamp": fetch_ts,
"want_more": false
})
)
:
(size(api_resp.Body) > 0 ? bytes(api_resp.Body).decode_json() : {}).as(error_body,
retryable.exists(code, code == api_resp.StatusCode).as(is_retryable,
(has(error_body.error) ?
string(error_body.error) :
"API request failed with HTTP " + string(api_resp.StatusCode)
).as(error_msg,
is_retryable ?
prep.updated_state.with({
"events": (has(prep.auth_event.message) ? [prep.auth_event] : []) + [{
"message": {
"log_type": "alert-fetch",
"status": "retry",
"api_endpoint": request_url,
"input_alert_fetch_count": prep.updated_state.page_size,
"error_message": error_msg,
"http_status_code": api_resp.StatusCode,
"cursor_used": prep.updated_state.next_cursor != "" ? prep.updated_state.next_cursor : "initial",
"fetched_alerts": 0,
"fetch_timestamp": fetch_ts
}.encode_json()
}],
"access_token": prep.updated_state.access_token,
"expires_at": prep.updated_state.expires_at,
"want_more": true
})
:
prep.updated_state.with({
"events": (has(prep.auth_event.message) ? [prep.auth_event] : []) + [{
"message": {
"log_type": "alert-fetch",
"status": "failed",
"api_endpoint": request_url,
"input_alert_fetch_count": prep.updated_state.page_size,
"error_message": error_msg,
"http_status_code": api_resp.StatusCode,
"cursor_used": prep.updated_state.next_cursor != "" ? prep.updated_state.next_cursor : "initial",
"fetched_alerts": 0,
"fetch_timestamp": fetch_ts,
"response_body": string(api_resp.Body)
}.encode_json()
}],
"access_token": prep.updated_state.access_token,
"expires_at": prep.updated_state.expires_at,
"next_cursor": api_resp.StatusCode == 400 || api_resp.StatusCode == 404 ? "" : prep.updated_state.next_cursor,
"want_more": false
})
)
)
)
)
)
:
prep.updated_state.with({
"events": [prep.auth_event],
"want_more": has(prep.want_more) ? prep.want_more : false
})
)
)
)
)
)
)
13 changes: 13 additions & 0 deletions packages/dataminr_pulse/_devtests/state.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"url": "https://api.dataminr.com/pulse/v1/alerts",
"base_url": "https://api.dataminr.com/pulse",
"client_id": "0oaqa65xf8e6Ocdsl5d7",
"client_secret": "`",
"version": "0.11.0",
"page_size": 40,
"next_cursor": "",
"access_token": "",
"expires_at": 0,
"want_more": false,
"last_fetch_timestamp": 0
}
5 changes: 5 additions & 0 deletions packages/dataminr_pulse/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- version: "1.0.0"
changes:
- description: Initial release
type: enhancement
link: https://github.com/elastic/integrations/issues/1
29 changes: 29 additions & 0 deletions packages/dataminr_pulse/config/kibana.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
server.host: "0.0.0.0"

xpack.fleet.outputs:
- id: fleet-default-output
name: default
type: elasticsearch
hosts:
- http://elasticsearch:9200
is_default: true
is_default_monitoring: true

xpack.fleet.agents.fleet_server.hosts:
- http://fleet-server:8220

xpack.fleet.packages:
- name: fleet_server
version: latest

xpack.fleet.agentPolicies:
- id: fleet-server-policy
name: Fleet Server Policy
is_default_fleet_server: true
package_policies:
- name: fleet_server-1
package:
name: fleet_server
- id: default-agent-policy
name: Default Agent Policy
is_default: true
9 changes: 9 additions & 0 deletions packages/dataminr_pulse/data_stream/alerts/_dev/deploy/docker/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
version: "2.3"
services:
elastic-siem-integration:
image: python:3.12-slim
volumes:
- ./files:/app
command: ["python", "/app/mock_server.py"]
ports:
- 8080
Loading
Loading