diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml index 7a680120979..0385afc13d4 100644 --- a/packages/cloudflare_logpush/changelog.yml +++ b/packages/cloudflare_logpush/changelog.yml @@ -1,4 +1,24 @@ # newer versions go on top +- version: "2.0.0" + changes: + - description: | + Fix several field mapping and ingest pipeline correctness bugs across data streams: + - Fix `cloudflare_logpush.gateway_http.quarantined` mapping by changing its type from `keyword` to `boolean` to match the actual values returned by Cloudflare. + - Fix `http_request` HTTP header field naming by renaming `header` to `headers` to align with the field definitions and the underlying Cloudflare payload. + - Fix `gateway_dns` resolved IP modeling by replacing the flat `cloudflare_logpush.gateway_dns.resolved_ip` field with the nested `cloudflare_logpush.gateway_dns.resolved_ip_details.ips`, so per-IP category metadata can be captured alongside the IP values. + - Fix `dlp_forensic_copies` `preserve_duplicate_custom_fields` toggle so that the correct custom fields (`triggered_rule_id`, `datetime`) — which have ECS counterparts — are removed when the toggle is OFF. Previously the remove processor referenced fields the pipeline never produced, so the toggle had no effect for these fields. + - Fix integer-to-keyword type casting for `*IDs` fields in `gateway_dns`, `gateway_http`, and `gateway_network` by replacing `rename` processors with `convert` processors of `type: string`, so values are consistently stored as strings. + type: bugfix + link: https://github.com/elastic/integrations/pull/18852 + - description: | + The fixes above are user-visible breaking changes: + - `cloudflare_logpush.gateway_http.quarantined` mapping changes from `keyword` to `boolean`. Existing indices will reject the new value type, so users must roll over the data stream (or reindex) for the new mapping to take effect. + - The `http_request` HTTP header fields are renamed from `header` to `headers`. Saved searches, dashboards, queries, and detection rules referencing `cloudflare_logpush.http_request.*.header` will no longer return data and must be updated to use `headers`. + - The `cloudflare_logpush.gateway_dns.resolved_ip` field is removed; resolved IPs now live under `cloudflare_logpush.gateway_dns.resolved_ip_details.ips`. Saved searches, dashboards, queries, and detection rules referencing the old field must be updated. + - In the `dlp_forensic_copies` data stream, when `preserve_duplicate_custom_fields` is OFF (the default), `cloudflare_logpush.dlp_forensic_copies.triggered_rule_id` and `cloudflare_logpush.dlp_forensic_copies.datetime` are no longer present in documents. Users that depend on these custom fields must turn `preserve_duplicate_custom_fields` ON, or migrate to the corresponding ECS fields. + - The `*IDs` fields in `gateway_dns`, `gateway_http`, and `gateway_network` are now indexed as strings rather than the raw JSON integers Cloudflare emits. Queries, dashboards, and detection rules comparing these fields against numeric values must be updated to use string values. + type: breaking-change + link: https://github.com/elastic/integrations/pull/18852 - version: "1.44.1" changes: - description: | diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml index 4a19388b999..4ec6f82503d 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml @@ -113,10 +113,8 @@ processors: ignore_missing: true - remove: field: - - cloudflare_logpush.dlp_forensic_copies.action - - cloudflare_logpush.dlp_forensic_copies.host - - cloudflare_logpush.dlp_forensic_copies.url - - cloudflare_logpush.dlp_forensic_copies.timestamp + - cloudflare_logpush.dlp_forensic_copies.triggered_rule_id + - cloudflare_logpush.dlp_forensic_copies.datetime if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields') ignore_failure: true ignore_missing: true diff --git a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json index e351281cf96..1f4a5b3e527 100644 --- a/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "698be645-ef86-47e6-9089-9dfbd966825f", - "id": "0b854708-2754-4561-bf95-a02f8101cc03", - "name": "elastic-agent-23353", + "ephemeral_id": "fbe0b509-8ef1-485f-81aa-b8133c0fbc0b", + "id": "888c1d7d-2129-4b5f-8d55-f5717851b204", + "name": "elastic-agent-96884", "type": "filebeat", "version": "8.17.1" }, @@ -24,14 +24,14 @@ }, "data_stream": { "dataset": "cloudflare_logpush.dlp_forensic_copies", - "namespace": "31861", + "namespace": "86795", "type": "logs" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "0b854708-2754-4561-bf95-a02f8101cc03", + "id": "888c1d7d-2129-4b5f-8d55-f5717851b204", "snapshot": false, "version": "8.17.1" }, @@ -41,7 +41,7 @@ "network" ], "dataset": "cloudflare_logpush.dlp_forensic_copies", - "ingested": "2026-05-07T09:57:30Z", + "ingested": "2026-05-07T10:11:01Z", "kind": "event", "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}", "type": [ diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json index fbbd247e365..cecda777840 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json @@ -69,11 +69,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -166,7 +168,9 @@ ], "ip": [ "67.43.156.2", - "89.160.20.129" + "89.160.20.129", + "67.43.156.1", + "67.43.156.3" ], "user": [ "166befbb-00e3-5e20-bd6e-27245000000", @@ -267,11 +271,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -364,7 +370,9 @@ ], "ip": [ "67.43.156.2", - "89.160.20.129" + "89.160.20.129", + "67.43.156.1", + "67.43.156.3" ], "user": [ "166befbb-00e3-5e20-bd6e-27245000000", @@ -465,11 +473,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -562,7 +572,9 @@ ], "ip": [ "67.43.156.2", - "89.160.20.129" + "89.160.20.129", + "67.43.156.1", + "67.43.156.3" ], "user": [ "166befbb-00e3-5e20-bd6e-27245000000", diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml index f4b69d1fbbe..11535516bc0 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml @@ -184,14 +184,6 @@ processors: field: dns.answers copy_from: cloudflare_logpush.gateway_dns.answers ignore_empty_value: true - - rename: - field: json.ResolvedIPs - target_field: cloudflare_logpush.gateway_dns.resolved_ip - ignore_missing: true - - set: - field: dns.resolved_ip - copy_from: cloudflare_logpush.gateway_dns.resolved_ip - ignore_empty_value: true - rename: field: json.SrcIP target_field: cloudflare_logpush.gateway_dns.source.ip @@ -272,9 +264,10 @@ processors: - append: field: error.message value: '{{{_ingest.on_failure_message}}}' - - rename: + - convert: field: json.CNAMECategoryIDs target_field: cloudflare_logpush.gateway_dns.cname_category.ids + type: string ignore_missing: true - rename: field: json.CNAMECategoryNames @@ -321,13 +314,15 @@ processors: field: json.DoTSubdomain target_field: cloudflare_logpush.gateway_dns.dot_subdomain ignore_missing: true - - rename: + - convert: field: json.EDEErrors target_field: cloudflare_logpush.gateway_dns.extended_dns_error_codes + type: string ignore_missing: true - - rename: + - convert: field: json.InitialCategoryIDs target_field: cloudflare_logpush.gateway_dns.initial_category.ids + type: string ignore_missing: true - rename: field: json.InitialCategoryNames @@ -358,9 +353,10 @@ processors: field: json.MatchedCategoryNames target_field: cloudflare_logpush.gateway_dns.matched.category.names ignore_missing: true - - rename: + - convert: field: json.MatchedIndicatorFeedIDs target_field: cloudflare_logpush.gateway_dns.matched.indicator_feed.ids + type: string ignore_missing: true - rename: field: json.MatchedIndicatorFeedNames @@ -416,9 +412,10 @@ processors: field: json.QueryType target_field: cloudflare_logpush.gateway_dns.question.type_id ignore_missing: true - - rename: + - convert: field: json.ResolvedIPCategoryIDs target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids + type: string ignore_missing: true - rename: field: json.ResolvedIPCategoryNames @@ -441,6 +438,10 @@ processors: - append: field: error.message value: '{{{_ingest.on_failure_message}}}' + - set: + field: dns.resolved_ip + copy_from: cloudflare_logpush.gateway_dns.resolved_ip_details.ips + ignore_empty_value: true - rename: field: json.ResolverPolicyID target_field: cloudflare_logpush.gateway_dns.resolver.policy.ids @@ -504,6 +505,14 @@ processors: value: '{{{user.email}}}' if: ctx.user?.email != null allow_duplicates: false + - foreach: + field: cloudflare_logpush.gateway_dns.resolved_ip_details.ips + if: ctx.cloudflare_logpush?.gateway_dns?.resolved_ip_details?.ips instanceof List + processor: + append: + field: related.ip + value: '{{{_ingest._value}}}' + allow_duplicates: false # Clean resulting event - remove: tag: remove_json_conf @@ -523,7 +532,7 @@ processors: - cloudflare_logpush.gateway_dns.question.type - cloudflare_logpush.gateway_dns.response_code - cloudflare_logpush.gateway_dns.answers - - cloudflare_logpush.gateway_dns.resolved_ip + - cloudflare_logpush.gateway_dns.resolved_ip_details.ips - cloudflare_logpush.gateway_dns.source.ip - cloudflare_logpush.gateway_dns.source.port - cloudflare_logpush.gateway_dns.timezone diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml index d5ee196f461..e6c8b93eea0 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml @@ -182,9 +182,6 @@ - name: type_id type: long description: ID of the type of DNS query. - - name: resolved_ip - type: ip - description: The resolved IPs in the response, if any. - name: resolved_ip_details type: group fields: diff --git a/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json index 283d11fd030..540422a98da 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2023-05-02T22:49:53.000Z", "agent": { - "ephemeral_id": "1c3b663b-0c05-4eda-a264-e5fff8a1643e", - "id": "3849f512-e036-495f-808b-d21769ada35a", - "name": "elastic-agent-19241", + "ephemeral_id": "d2c75bea-8493-42ae-ab47-64c6ff4e96db", + "id": "4d4f44f3-246a-4333-aa9e-d382ee402851", + "name": "elastic-agent-27416", "type": "filebeat", "version": "8.17.1" }, @@ -74,11 +74,13 @@ "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -96,7 +98,7 @@ }, "data_stream": { "dataset": "cloudflare_logpush.gateway_dns", - "namespace": "73427", + "namespace": "57015", "type": "logs" }, "destination": { @@ -151,7 +153,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "3849f512-e036-495f-808b-d21769ada35a", + "id": "4d4f44f3-246a-4333-aa9e-d382ee402851", "snapshot": false, "version": "8.17.1" }, @@ -161,7 +163,7 @@ "network" ], "dataset": "cloudflare_logpush.gateway_dns", - "ingested": "2026-05-07T09:59:42Z", + "ingested": "2026-05-07T10:11:59Z", "kind": "event", "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}", "outcome": "success", @@ -187,7 +189,9 @@ ], "ip": [ "67.43.156.2", - "89.160.20.129" + "89.160.20.129", + "67.43.156.1", + "67.43.156.3" ], "user": [ "166befbb-00e3-5e20-bd6e-27245000000", diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json index aeaf9838def..bb9a613bd0a 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json @@ -522,8 +522,8 @@ }, "category": { "ids": [ - 26, - 81 + "26", + "81" ], "names": [ "Technology", diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml index 0bc4b1904c9..5d52bfcfb53 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml @@ -251,9 +251,10 @@ processors: field: json.AccountID target_field: cloudflare_logpush.gateway_http.account_id ignore_missing: true - - rename: + - convert: field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_http.application.ids + type: string ignore_missing: true - rename: field: json.ApplicationNames @@ -279,9 +280,10 @@ processors: field: json.BlockedFileType target_field: cloudflare_logpush.gateway_http.blocked_file.type ignore_missing: true - - rename: + - convert: field: json.CategoryIDs target_field: cloudflare_logpush.gateway_http.category.ids + type: string ignore_missing: true - rename: field: json.CategoryNames diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml b/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml index 4afcc1306e2..2998a321026 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml @@ -108,7 +108,7 @@ type: keyword description: The proxy endpoint used on this network session, if any. - name: quarantined - type: keyword + type: boolean description: If the request content was quarantined. - name: request type: group diff --git a/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json index 88389f74ec6..57a2a0b8e2c 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_http/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2023-05-03T20:55:05.000Z", "agent": { - "ephemeral_id": "bd73bd3a-7603-42f5-8954-36f29479b358", - "id": "7d1a5dea-3b66-4fff-9548-749ee57d9181", - "name": "elastic-agent-95974", + "ephemeral_id": "1b35a5c6-4363-4ae7-ae0d-d3b91627d126", + "id": "73be74d9-7303-434d-8732-938ede199c77", + "name": "elastic-agent-76276", "type": "filebeat", "version": "8.17.1" }, @@ -78,7 +78,7 @@ }, "data_stream": { "dataset": "cloudflare_logpush.gateway_http", - "namespace": "14950", + "namespace": "21052", "type": "logs" }, "destination": { @@ -107,7 +107,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "7d1a5dea-3b66-4fff-9548-749ee57d9181", + "id": "73be74d9-7303-434d-8732-938ede199c77", "snapshot": false, "version": "8.17.1" }, @@ -118,7 +118,7 @@ "network" ], "dataset": "cloudflare_logpush.gateway_http", - "ingested": "2026-05-06T10:52:15Z", + "ingested": "2026-05-06T11:38:12Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a348\",\"Action\":\"block\",\"BlockedFileHash\":\"91dc1db739a705105e1c763bfdbdaa84c0de8\",\"BlockedFileName\":\"downloaded_test\",\"BlockedFileReason\":\"malware\",\"BlockedFileSize\":43,\"BlockedFileType\":\"bin\",\"Datetime\":\"2023-05-03T20:55:05Z\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"DownloadedFileNames\":[\"downloaded_file\",\"downloaded_test\"],\"Email\":\"user@example.com\",\"FileInfo\":{\"files\":[{\"name\":\"downloaded_file\",\"size\":43},{\"name\":\"downloaded_test\",\"size\":341}]},\"HTTPHost\":\"guce.yahoo.com\",\"HTTPMethod\":\"GET\",\"HTTPStatusCode\":302,\"HTTPVersion\":\"HTTP/2\",\"IsIsolated\":false,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"Block Yahoo\",\"Referer\":\"https://www.example.com/\",\"RequestID\":\"1884fec9b600007fb06a299400000001\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.123\",\"SourcePort\":47924,\"URL\":\"https://test.com\",\"UntrustedCertificateAction\":\"none\",\"UploadedFileNames\":[\"uploaded_file\",\"uploaded_test\"],\"UserAgent\":\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml index 379981aaff7..336fc46e565 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/gateway_network/elasticsearch/ingest_pipeline/default.yml @@ -221,17 +221,19 @@ processors: field: json.AccountID target_field: cloudflare_logpush.gateway_network.account_id ignore_missing: true - - rename: + - convert: field: json.ApplicationIDs target_field: cloudflare_logpush.gateway_network.application.ids + type: string ignore_missing: true - rename: field: json.ApplicationNames target_field: cloudflare_logpush.gateway_network.application.names ignore_missing: true - - rename: + - convert: field: json.CategoryIDs target_field: cloudflare_logpush.gateway_network.category.ids + type: string ignore_missing: true - rename: field: json.CategoryNames diff --git a/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json b/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json index 15c0fe51e26..e648f377b19 100644 --- a/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/gateway_network/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2023-05-18T21:12:57.058Z", "agent": { - "ephemeral_id": "d71417a3-d888-4ce0-ba97-63c6026ea45a", - "id": "ebc82c7e-1496-4828-b2e0-0fdc859a362c", - "name": "elastic-agent-90140", + "ephemeral_id": "fb95d17b-66d7-4324-864c-01c810a0065a", + "id": "e8ff3f1c-c19c-4c53-971a-b13b0cfb6c9f", + "name": "elastic-agent-76120", "type": "filebeat", "version": "8.17.1" }, @@ -44,7 +44,7 @@ }, "data_stream": { "dataset": "cloudflare_logpush.gateway_network", - "namespace": "96293", + "namespace": "54252", "type": "logs" }, "destination": { @@ -74,7 +74,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "ebc82c7e-1496-4828-b2e0-0fdc859a362c", + "id": "e8ff3f1c-c19c-4c53-971a-b13b0cfb6c9f", "snapshot": false, "version": "8.17.1" }, @@ -86,7 +86,7 @@ ], "dataset": "cloudflare_logpush.gateway_network", "id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", - "ingested": "2026-05-06T10:53:10Z", + "ingested": "2026-05-06T11:39:12Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"Action\":\"allowedOnNoRuleMatch\",\"Datetime\":1684444377058000000,\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"Email\":\"user@test.com\",\"OverrideIP\":\"175.16.199.4\",\"OverridePort\":8080,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"My policy\",\"SNI\":\"www.elastic.co\",\"SessionID\":\"5f2d04be-3512-11e8-b467-0ed5f89f718b\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.3\",\"SourcePort\":47924,\"Transport\":\"tcp\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ diff --git a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml index c6f21150469..56620c9064c 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/http_request/elasticsearch/ingest_pipeline/default.yml @@ -758,11 +758,11 @@ processors: ignore_empty_value: true - rename: field: json.RequestHeaders - target_field: cloudflare_logpush.http_request.request.header + target_field: cloudflare_logpush.http_request.request.headers ignore_missing: true - rename: field: json.ResponseHeaders - target_field: cloudflare_logpush.http_request.response.header + target_field: cloudflare_logpush.http_request.response.headers ignore_missing: true - rename: field: json.SecurityLevel diff --git a/packages/cloudflare_logpush/data_stream/http_request/sample_event.json b/packages/cloudflare_logpush/data_stream/http_request/sample_event.json index 3ae94cf5db1..f05fd4f8f8a 100644 --- a/packages/cloudflare_logpush/data_stream/http_request/sample_event.json +++ b/packages/cloudflare_logpush/data_stream/http_request/sample_event.json @@ -1,9 +1,9 @@ { "@timestamp": "2022-05-25T13:25:26.000Z", "agent": { - "ephemeral_id": "5c30f62a-7113-4fb3-bc34-184a2fffbb14", - "id": "629ad3f2-1781-41b4-9fb6-008cff3ef87b", - "name": "elastic-agent-16148", + "ephemeral_id": "01f408e6-b8bc-406b-9524-fb8b929604ef", + "id": "547a6b9b-150b-4eb1-8b51-f15546edcdcf", + "name": "elastic-agent-11246", "type": "filebeat", "version": "8.17.1" }, @@ -191,7 +191,7 @@ }, "data_stream": { "dataset": "cloudflare_logpush.http_request", - "namespace": "16525", + "namespace": "47436", "type": "logs" }, "destination": { @@ -201,7 +201,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "629ad3f2-1781-41b4-9fb6-008cff3ef87b", + "id": "547a6b9b-150b-4eb1-8b51-f15546edcdcf", "snapshot": false, "version": "8.17.1" }, @@ -212,7 +212,7 @@ ], "dataset": "cloudflare_logpush.http_request", "id": "710e98d9367f357d", - "ingested": "2026-05-06T10:54:13Z", + "ingested": "2026-05-06T11:40:14Z", "kind": "event", "original": "{\"BotDetectionIDs\":[7,8,9],\"BotScore\":20,\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":[\"bing\",\"api\"],\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityAction\":\"unknown\",\"SecurityLevel\":\"off\",\"SecurityRuleDescription\":\"matchad variable message\",\"SecurityRuleID\":\"98d93d5\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAttackScore\":50,\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRCEAttackScore\":1,\"WAFSQLiAttackScore\":99,\"WAFXSSAttackScore\":90,\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}", "type": [ diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md index 27b8ca30662..ba7bf19280b 100644 --- a/packages/cloudflare_logpush/docs/README.md +++ b/packages/cloudflare_logpush/docs/README.md @@ -882,9 +882,9 @@ An example event for `dlp_forensic_copies` looks as following: { "@timestamp": "2023-05-04T11:29:14.000Z", "agent": { - "ephemeral_id": "698be645-ef86-47e6-9089-9dfbd966825f", - "id": "0b854708-2754-4561-bf95-a02f8101cc03", - "name": "elastic-agent-23353", + "ephemeral_id": "fbe0b509-8ef1-485f-81aa-b8133c0fbc0b", + "id": "888c1d7d-2129-4b5f-8d55-f5717851b204", + "name": "elastic-agent-96884", "type": "filebeat", "version": "8.17.1" }, @@ -905,14 +905,14 @@ An example event for `dlp_forensic_copies` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.dlp_forensic_copies", - "namespace": "31861", + "namespace": "86795", "type": "logs" }, "ecs": { "version": "8.11.0" }, "elastic_agent": { - "id": "0b854708-2754-4561-bf95-a02f8101cc03", + "id": "888c1d7d-2129-4b5f-8d55-f5717851b204", "snapshot": false, "version": "8.17.1" }, @@ -922,7 +922,7 @@ An example event for `dlp_forensic_copies` looks as following: "network" ], "dataset": "cloudflare_logpush.dlp_forensic_copies", - "ingested": "2026-05-07T09:57:30Z", + "ingested": "2026-05-07T10:11:01Z", "kind": "event", "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}", "type": [ @@ -1756,9 +1756,9 @@ An example event for `gateway_dns` looks as following: { "@timestamp": "2023-05-02T22:49:53.000Z", "agent": { - "ephemeral_id": "1c3b663b-0c05-4eda-a264-e5fff8a1643e", - "id": "3849f512-e036-495f-808b-d21769ada35a", - "name": "elastic-agent-19241", + "ephemeral_id": "d2c75bea-8493-42ae-ab47-64c6ff4e96db", + "id": "4d4f44f3-246a-4333-aa9e-d382ee402851", + "name": "elastic-agent-27416", "type": "filebeat", "version": "8.17.1" }, @@ -1829,11 +1829,13 @@ An example event for `gateway_dns` looks as following: "type": "A", "type_id": 1 }, - "resolved_ip": [ - "67.43.156.1", - "67.43.156.2", - "67.43.156.3" - ], + "resolved_ip_details": { + "ips": [ + "67.43.156.1", + "67.43.156.2", + "67.43.156.3" + ] + }, "resolver_decision": "allowedOnNoPolicyMatch", "response_code": "0", "source": { @@ -1851,7 +1853,7 @@ An example event for `gateway_dns` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_dns", - "namespace": "73427", + "namespace": "57015", "type": "logs" }, "destination": { @@ -1906,7 +1908,7 @@ An example event for `gateway_dns` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "3849f512-e036-495f-808b-d21769ada35a", + "id": "4d4f44f3-246a-4333-aa9e-d382ee402851", "snapshot": false, "version": "8.17.1" }, @@ -1916,7 +1918,7 @@ An example event for `gateway_dns` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_dns", - "ingested": "2026-05-07T09:59:42Z", + "ingested": "2026-05-07T10:11:59Z", "kind": "event", "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}", "outcome": "success", @@ -1942,7 +1944,9 @@ An example event for `gateway_dns` looks as following: ], "ip": [ "67.43.156.2", - "89.160.20.129" + "89.160.20.129", + "67.43.156.1", + "67.43.156.3" ], "user": [ "166befbb-00e3-5e20-bd6e-27245000000", @@ -2035,7 +2039,6 @@ An example event for `gateway_dns` looks as following: | cloudflare_logpush.gateway_dns.question.size | The size of the DNS request in bytes. | long | | cloudflare_logpush.gateway_dns.question.type | The type of DNS query. | keyword | | cloudflare_logpush.gateway_dns.question.type_id | ID of the type of DNS query. | long | -| cloudflare_logpush.gateway_dns.resolved_ip | The resolved IPs in the response, if any. | ip | | cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids | ID or IDs of category that the IPs in the response belongs to. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.category.names | Name or names of category that the IPs in the response belongs to. | keyword | | cloudflare_logpush.gateway_dns.resolved_ip_details.continent_codes | Continent code of each resolved IP, if any. | keyword | @@ -2085,9 +2088,9 @@ An example event for `gateway_http` looks as following: { "@timestamp": "2023-05-03T20:55:05.000Z", "agent": { - "ephemeral_id": "bd73bd3a-7603-42f5-8954-36f29479b358", - "id": "7d1a5dea-3b66-4fff-9548-749ee57d9181", - "name": "elastic-agent-95974", + "ephemeral_id": "1b35a5c6-4363-4ae7-ae0d-d3b91627d126", + "id": "73be74d9-7303-434d-8732-938ede199c77", + "name": "elastic-agent-76276", "type": "filebeat", "version": "8.17.1" }, @@ -2162,7 +2165,7 @@ An example event for `gateway_http` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_http", - "namespace": "14950", + "namespace": "21052", "type": "logs" }, "destination": { @@ -2191,7 +2194,7 @@ An example event for `gateway_http` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "7d1a5dea-3b66-4fff-9548-749ee57d9181", + "id": "73be74d9-7303-434d-8732-938ede199c77", "snapshot": false, "version": "8.17.1" }, @@ -2202,7 +2205,7 @@ An example event for `gateway_http` looks as following: "network" ], "dataset": "cloudflare_logpush.gateway_http", - "ingested": "2026-05-06T10:52:15Z", + "ingested": "2026-05-06T11:38:12Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a348\",\"Action\":\"block\",\"BlockedFileHash\":\"91dc1db739a705105e1c763bfdbdaa84c0de8\",\"BlockedFileName\":\"downloaded_test\",\"BlockedFileReason\":\"malware\",\"BlockedFileSize\":43,\"BlockedFileType\":\"bin\",\"Datetime\":\"2023-05-03T20:55:05Z\",\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"DownloadedFileNames\":[\"downloaded_file\",\"downloaded_test\"],\"Email\":\"user@example.com\",\"FileInfo\":{\"files\":[{\"name\":\"downloaded_file\",\"size\":43},{\"name\":\"downloaded_test\",\"size\":341}]},\"HTTPHost\":\"guce.yahoo.com\",\"HTTPMethod\":\"GET\",\"HTTPStatusCode\":302,\"HTTPVersion\":\"HTTP/2\",\"IsIsolated\":false,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"Block Yahoo\",\"Referer\":\"https://www.example.com/\",\"RequestID\":\"1884fec9b600007fb06a299400000001\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.123\",\"SourcePort\":47924,\"URL\":\"https://test.com\",\"UntrustedCertificateAction\":\"none\",\"UploadedFileNames\":[\"uploaded_file\",\"uploaded_test\"],\"UserAgent\":\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64) Firefox/112.0\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ @@ -2318,7 +2321,7 @@ An example event for `gateway_http` looks as following: | cloudflare_logpush.gateway_http.policy.name | The name of the gateway policy applied to the request, if any. | keyword | | cloudflare_logpush.gateway_http.private_app_aud | The private app AUD, if any. | keyword | | cloudflare_logpush.gateway_http.proxy_endpoint | The proxy endpoint used on this network session, if any. | keyword | -| cloudflare_logpush.gateway_http.quarantined | If the request content was quarantined. | keyword | +| cloudflare_logpush.gateway_http.quarantined | If the request content was quarantined. | boolean | | cloudflare_logpush.gateway_http.request.host | Content of the host header in the HTTP request. | keyword | | cloudflare_logpush.gateway_http.request.method | HTTP request method. | keyword | | cloudflare_logpush.gateway_http.request.referrer | Contents of the referer header in the HTTP request. | keyword | @@ -2371,9 +2374,9 @@ An example event for `gateway_network` looks as following: { "@timestamp": "2023-05-18T21:12:57.058Z", "agent": { - "ephemeral_id": "d71417a3-d888-4ce0-ba97-63c6026ea45a", - "id": "ebc82c7e-1496-4828-b2e0-0fdc859a362c", - "name": "elastic-agent-90140", + "ephemeral_id": "fb95d17b-66d7-4324-864c-01c810a0065a", + "id": "e8ff3f1c-c19c-4c53-971a-b13b0cfb6c9f", + "name": "elastic-agent-76120", "type": "filebeat", "version": "8.17.1" }, @@ -2414,7 +2417,7 @@ An example event for `gateway_network` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.gateway_network", - "namespace": "96293", + "namespace": "54252", "type": "logs" }, "destination": { @@ -2444,7 +2447,7 @@ An example event for `gateway_network` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "ebc82c7e-1496-4828-b2e0-0fdc859a362c", + "id": "e8ff3f1c-c19c-4c53-971a-b13b0cfb6c9f", "snapshot": false, "version": "8.17.1" }, @@ -2456,7 +2459,7 @@ An example event for `gateway_network` looks as following: ], "dataset": "cloudflare_logpush.gateway_network", "id": "5f2d04be-3512-11e8-b467-0ed5f89f718b", - "ingested": "2026-05-06T10:53:10Z", + "ingested": "2026-05-06T11:39:12Z", "kind": "event", "original": "{\"AccountID\":\"e1836771179f98aabb828da5ea69a111\",\"Action\":\"allowedOnNoRuleMatch\",\"Datetime\":1684444377058000000,\"DestinationIP\":\"89.160.20.129\",\"DestinationPort\":443,\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b100cff\",\"DeviceName\":\"zt-test-vm1\",\"Email\":\"user@test.com\",\"OverrideIP\":\"175.16.199.4\",\"OverridePort\":8080,\"PolicyID\":\"85063bec-74cb-4546-85a3-e0cde2cdfda2\",\"PolicyName\":\"My policy\",\"SNI\":\"www.elastic.co\",\"SessionID\":\"5f2d04be-3512-11e8-b467-0ed5f89f718b\",\"SourceIP\":\"67.43.156.2\",\"SourceInternalIP\":\"192.168.1.3\",\"SourcePort\":47924,\"Transport\":\"tcp\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245723949f\"}", "type": [ @@ -2596,9 +2599,9 @@ An example event for `http_request` looks as following: { "@timestamp": "2022-05-25T13:25:26.000Z", "agent": { - "ephemeral_id": "5c30f62a-7113-4fb3-bc34-184a2fffbb14", - "id": "629ad3f2-1781-41b4-9fb6-008cff3ef87b", - "name": "elastic-agent-16148", + "ephemeral_id": "01f408e6-b8bc-406b-9524-fb8b929604ef", + "id": "547a6b9b-150b-4eb1-8b51-f15546edcdcf", + "name": "elastic-agent-11246", "type": "filebeat", "version": "8.17.1" }, @@ -2786,7 +2789,7 @@ An example event for `http_request` looks as following: }, "data_stream": { "dataset": "cloudflare_logpush.http_request", - "namespace": "16525", + "namespace": "47436", "type": "logs" }, "destination": { @@ -2796,7 +2799,7 @@ An example event for `http_request` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "629ad3f2-1781-41b4-9fb6-008cff3ef87b", + "id": "547a6b9b-150b-4eb1-8b51-f15546edcdcf", "snapshot": false, "version": "8.17.1" }, @@ -2807,7 +2810,7 @@ An example event for `http_request` looks as following: ], "dataset": "cloudflare_logpush.http_request", "id": "710e98d9367f357d", - "ingested": "2026-05-06T10:54:13Z", + "ingested": "2026-05-06T11:40:14Z", "kind": "event", "original": "{\"BotDetectionIDs\":[7,8,9],\"BotScore\":20,\"BotScoreSrc\":\"Verified Bot\",\"BotTags\":[\"bing\",\"api\"],\"CacheCacheStatus\":\"dynamic\",\"CacheResponseBytes\":983828,\"CacheResponseStatus\":200,\"CacheTieredFill\":false,\"ClientASN\":43766,\"ClientCountry\":\"sa\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"175.16.199.0\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"Fingerprint\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRequestBytes\":5800,\"ClientRequestHost\":\"xyz.example.com\",\"ClientRequestMethod\":\"POST\",\"ClientRequestPath\":\"/xyz/checkout\",\"ClientRequestProtocol\":\"HTTP/1.1\",\"ClientRequestReferer\":\"https://example.com/s/example/default?sourcerer=(default:(id:!n,selectedPatterns:!(example,%27logs-endpoint.*-example%27,%27logs-system.*-example%27,%27logs-windows.*-example%27)))\\u0026timerange=(global:(linkTo:!(),timerange:(from:%272022-05-16T06:26:36.340Z%27,fromStr:now-24h,kind:relative,to:%272022-05-17T06:26:36.340Z%27,toStr:now)),timeline:(linkTo:!(),timerange:(from:%272022-04-17T22:00:00.000Z%27,kind:absolute,to:%272022-04-18T21:59:59.999Z%27)))\\u0026timeline=(activeTab:notes,graphEventId:%27%27,id:%279844bdd4-4dd6-5b22-ab40-3cd46fce8d6b%27,isOpen:!t)\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"edgeWorkerFetch\",\"ClientRequestURI\":\"/s/example/api/telemetry/v2/clusters/_stats\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36\",\"ClientSSLCipher\":\"NONE\",\"ClientSSLProtocol\":\"TLSv1.2\",\"ClientSrcPort\":0,\"ClientTCPRTTMs\":0,\"ClientXRequestedWith\":\"Request With\",\"Cookies\":{\"key\":\"value\"},\"EdgeCFConnectingO2O\":false,\"EdgeColoCode\":\"RUH\",\"EdgeColoID\":339,\"EdgeEndTimestamp\":\"2022-05-25T13:25:32Z\",\"EdgePathingOp\":\"wl\",\"EdgePathingSrc\":\"macro\",\"EdgePathingStatus\":\"nr\",\"EdgeRateLimitAction\":\"unknown\",\"EdgeRateLimitID\":0,\"EdgeRequestHost\":\"abc.example.com\",\"EdgeResponseBodyBytes\":980397,\"EdgeResponseBytes\":981308,\"EdgeResponseCompressionRatio\":0,\"EdgeResponseContentType\":\"application/json\",\"EdgeResponseStatus\":200,\"EdgeServerIP\":\"1.128.0.0\",\"EdgeStartTimestamp\":\"2022-05-25T13:25:26Z\",\"EdgeTimeToFirstByteMs\":5333,\"OriginDNSResponseTimeMs\":3,\"OriginIP\":\"67.43.156.0\",\"OriginRequestHeaderSendDurationMs\":0,\"OriginResponseBytes\":0,\"OriginResponseDurationMs\":5319,\"OriginResponseHTTPExpires\":\"2022-05-27T13:25:26Z\",\"OriginResponseHTTPLastModified\":\"2022-05-26T13:25:26Z\",\"OriginResponseHeaderReceiveDurationMs\":5155,\"OriginResponseStatus\":200,\"OriginResponseTime\":5232000000,\"OriginSSLProtocol\":\"TLSv1.2\",\"OriginTCPHandshakeDurationMs\":24,\"OriginTLSHandshakeDurationMs\":53,\"ParentRayID\":\"710e98d93d50357d\",\"RayID\":\"710e98d9367f357d\",\"SecurityAction\":\"unknown\",\"SecurityLevel\":\"off\",\"SecurityRuleDescription\":\"matchad variable message\",\"SecurityRuleID\":\"98d93d5\",\"SmartRouteColoID\":20,\"UpperTierColoID\":0,\"WAFAttackScore\":50,\"WAFFlags\":\"0\",\"WAFMatchedVar\":\"example\",\"WAFProfile\":\"unknown\",\"WAFRCEAttackScore\":1,\"WAFSQLiAttackScore\":99,\"WAFXSSAttackScore\":90,\"WorkerCPUTime\":0,\"WorkerStatus\":\"unknown\",\"WorkerSubrequest\":true,\"WorkerSubrequestCount\":0,\"ZoneID\":393347122,\"ZoneName\":\"example.com\"}", "type": [ diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml index 9eec26c027f..41722ed71e2 100644 --- a/packages/cloudflare_logpush/manifest.yml +++ b/packages/cloudflare_logpush/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cloudflare_logpush title: Cloudflare Logpush -version: "1.44.1" +version: "2.0.0" description: Collect and parse logs from Cloudflare API with Elastic Agent. type: integration categories: