[cloudflare_logpush] fix breaking-change bugs

packages/cloudflare_logpush/changelog.yml

@@ -1,4 +1,24 @@
 # newer versions go on top
+- version: "2.0.0"
+  changes:
+    - description: |
+        Fix several field mapping and ingest pipeline correctness bugs across data streams:
+          - Fix `cloudflare_logpush.gateway_http.quarantined` mapping by changing its type from `keyword` to `boolean` to match the actual values returned by Cloudflare.
+          - Fix `http_request` HTTP header field naming by renaming `header` to `headers` to align with the field definitions and the underlying Cloudflare payload.
+          - Fix `gateway_dns` resolved IP modeling by replacing the flat `cloudflare_logpush.gateway_dns.resolved_ip` field with the nested `cloudflare_logpush.gateway_dns.resolved_ip_details.ips`, so per-IP category metadata can be captured alongside the IP values.
+          - Fix `dlp_forensic_copies` `preserve_duplicate_custom_fields` toggle so that the correct custom fields (`triggered_rule_id`, `datetime`) — which have ECS counterparts — are removed when the toggle is OFF. Previously the remove processor referenced fields the pipeline never produced, so the toggle had no effect for these fields.
+          - Fix integer-to-keyword type casting for `*IDs` fields in `gateway_dns`, `gateway_http`, and `gateway_network` by replacing `rename` processors with `convert` processors of `type: string`, so values are consistently stored as strings.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/18852
+    - description: |
+        The fixes above are user-visible breaking changes:
+          - `cloudflare_logpush.gateway_http.quarantined` mapping changes from `keyword` to `boolean`. Existing indices will reject the new value type, so users must roll over the data stream (or reindex) for the new mapping to take effect.
+          - The `http_request` HTTP header fields are renamed from `header` to `headers`. Saved searches, dashboards, queries, and detection rules referencing `cloudflare_logpush.http_request.*.header` will no longer return data and must be updated to use `headers`.
+          - The `cloudflare_logpush.gateway_dns.resolved_ip` field is removed; resolved IPs now live under `cloudflare_logpush.gateway_dns.resolved_ip_details.ips`. Saved searches, dashboards, queries, and detection rules referencing the old field must be updated.
+          - In the `dlp_forensic_copies` data stream, when `preserve_duplicate_custom_fields` is OFF (the default), `cloudflare_logpush.dlp_forensic_copies.triggered_rule_id` and `cloudflare_logpush.dlp_forensic_copies.datetime` are no longer present in documents. Users that depend on these custom fields must turn `preserve_duplicate_custom_fields` ON, or migrate to the corresponding ECS fields.
+          - The `*IDs` fields in `gateway_dns`, `gateway_http`, and `gateway_network` are now indexed as strings rather than the raw JSON integers Cloudflare emits. Queries, dashboards, and detection rules comparing these fields against numeric values must be updated to use string values.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/18852
 - version: "1.44.1"
   changes:
     - description: |

packages/cloudflare_logpush/data_stream/dlp_forensic_copies/elasticsearch/ingest_pipeline/default.yml

@@ -113,10 +113,8 @@ processors:
       ignore_missing: true
   - remove:
       field:
-        - cloudflare_logpush.dlp_forensic_copies.action
-        - cloudflare_logpush.dlp_forensic_copies.host
-        - cloudflare_logpush.dlp_forensic_copies.url
-        - cloudflare_logpush.dlp_forensic_copies.timestamp
+        - cloudflare_logpush.dlp_forensic_copies.triggered_rule_id
+        - cloudflare_logpush.dlp_forensic_copies.datetime
       if: ctx.tags == null || !ctx.tags.contains('preserve_duplicate_custom_fields')
       ignore_failure: true
       ignore_missing: true

packages/cloudflare_logpush/data_stream/dlp_forensic_copies/sample_event.json

@@ -1,9 +1,9 @@
 {
     "@timestamp": "2023-05-04T11:29:14.000Z",
     "agent": {
-        "ephemeral_id": "698be645-ef86-47e6-9089-9dfbd966825f",
-        "id": "0b854708-2754-4561-bf95-a02f8101cc03",
-        "name": "elastic-agent-23353",
+        "ephemeral_id": "fbe0b509-8ef1-485f-81aa-b8133c0fbc0b",
+        "id": "888c1d7d-2129-4b5f-8d55-f5717851b204",
+        "name": "elastic-agent-96884",
         "type": "filebeat",
         "version": "8.17.1"
     },
@@ -24,14 +24,14 @@
     },
     "data_stream": {
         "dataset": "cloudflare_logpush.dlp_forensic_copies",
-        "namespace": "31861",
+        "namespace": "86795",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "0b854708-2754-4561-bf95-a02f8101cc03",
+        "id": "888c1d7d-2129-4b5f-8d55-f5717851b204",
         "snapshot": false,
         "version": "8.17.1"
     },
@@ -41,7 +41,7 @@
             "network"
         ],
         "dataset": "cloudflare_logpush.dlp_forensic_copies",
-        "ingested": "2026-05-07T09:57:30Z",
+        "ingested": "2026-05-07T10:11:01Z",
         "kind": "event",
         "original": "{\"AccountID\":\"acc-id\",\"Datetime\":\"2023-05-04T11:29:14Z\",\"ForensicCopyID\":\"copy-id\",\"GatewayRequestID\":\"req-id\",\"Headers\":{\"key1\":\"val1\",\"key2\":\"val2\"},\"Payload\":\"Tm90aGluZyB0byBzZWUgaGVyZS4gTW92ZSBhbG9uZy4K\",\"Phase\":\"request\",\"TriggeredRuleID\":\"9\"}",
         "type": [

packages/cloudflare_logpush/data_stream/gateway_dns/_dev/test/pipeline/test-pipeline-gateway-dns.log-expected.json

@@ -69,11 +69,13 @@
                         "type": "A",
                         "type_id": 1
                     },
-                    "resolved_ip": [
-                        "67.43.156.1",
-                        "67.43.156.2",
-                        "67.43.156.3"
-                    ],
+                    "resolved_ip_details": {
+                        "ips": [
+                            "67.43.156.1",
+                            "67.43.156.2",
+                            "67.43.156.3"
+                        ]
+                    },
                     "resolver_decision": "allowedOnNoPolicyMatch",
                     "response_code": "0",
                     "source": {
@@ -166,7 +168,9 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "67.43.156.1",
+                    "67.43.156.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245000000",
@@ -267,11 +271,13 @@
                         "type": "A",
                         "type_id": 1
                     },
-                    "resolved_ip": [
-                        "67.43.156.1",
-                        "67.43.156.2",
-                        "67.43.156.3"
-                    ],
+                    "resolved_ip_details": {
+                        "ips": [
+                            "67.43.156.1",
+                            "67.43.156.2",
+                            "67.43.156.3"
+                        ]
+                    },
                     "resolver_decision": "allowedOnNoPolicyMatch",
                     "response_code": "0",
                     "source": {
@@ -364,7 +370,9 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "67.43.156.1",
+                    "67.43.156.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245000000",
@@ -465,11 +473,13 @@
                         "type": "A",
                         "type_id": 1
                     },
-                    "resolved_ip": [
-                        "67.43.156.1",
-                        "67.43.156.2",
-                        "67.43.156.3"
-                    ],
+                    "resolved_ip_details": {
+                        "ips": [
+                            "67.43.156.1",
+                            "67.43.156.2",
+                            "67.43.156.3"
+                        ]
+                    },
                     "resolver_decision": "allowedOnNoPolicyMatch",
                     "response_code": "0",
                     "source": {
@@ -562,7 +572,9 @@
                 ],
                 "ip": [
                     "67.43.156.2",
-                    "89.160.20.129"
+                    "89.160.20.129",
+                    "67.43.156.1",
+                    "67.43.156.3"
                 ],
                 "user": [
                     "166befbb-00e3-5e20-bd6e-27245000000",

packages/cloudflare_logpush/data_stream/gateway_dns/elasticsearch/ingest_pipeline/default.yml

@@ -184,14 +184,6 @@ processors:
       field: dns.answers
       copy_from: cloudflare_logpush.gateway_dns.answers
       ignore_empty_value: true
-  - rename:
-      field: json.ResolvedIPs
-      target_field: cloudflare_logpush.gateway_dns.resolved_ip
-      ignore_missing: true
-  - set:
-      field: dns.resolved_ip
-      copy_from: cloudflare_logpush.gateway_dns.resolved_ip
-      ignore_empty_value: true
   - rename:
       field: json.SrcIP
       target_field: cloudflare_logpush.gateway_dns.source.ip
@@ -272,9 +264,10 @@ processors:
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
-  - rename:
+  - convert:
       field: json.CNAMECategoryIDs
       target_field: cloudflare_logpush.gateway_dns.cname_category.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.CNAMECategoryNames
@@ -321,13 +314,15 @@ processors:
       field: json.DoTSubdomain
       target_field: cloudflare_logpush.gateway_dns.dot_subdomain
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.EDEErrors
       target_field: cloudflare_logpush.gateway_dns.extended_dns_error_codes
+      type: string
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.InitialCategoryIDs
       target_field: cloudflare_logpush.gateway_dns.initial_category.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.InitialCategoryNames
@@ -358,9 +353,10 @@ processors:
       field: json.MatchedCategoryNames
       target_field: cloudflare_logpush.gateway_dns.matched.category.names
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.MatchedIndicatorFeedIDs
       target_field: cloudflare_logpush.gateway_dns.matched.indicator_feed.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.MatchedIndicatorFeedNames
@@ -416,9 +412,10 @@ processors:
       field: json.QueryType
       target_field: cloudflare_logpush.gateway_dns.question.type_id
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.ResolvedIPCategoryIDs
       target_field: cloudflare_logpush.gateway_dns.resolved_ip_details.category.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.ResolvedIPCategoryNames
@@ -441,6 +438,10 @@ processors:
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
+  - set:
+      field: dns.resolved_ip
+      copy_from: cloudflare_logpush.gateway_dns.resolved_ip_details.ips
+      ignore_empty_value: true
   - rename:
       field: json.ResolverPolicyID
       target_field: cloudflare_logpush.gateway_dns.resolver.policy.ids
@@ -504,6 +505,14 @@ processors:
       value: '{{{user.email}}}'
       if: ctx.user?.email != null
       allow_duplicates: false
+  - foreach:
+      field: cloudflare_logpush.gateway_dns.resolved_ip_details.ips
+      if: ctx.cloudflare_logpush?.gateway_dns?.resolved_ip_details?.ips instanceof List
+      processor:
+        append:
+          field: related.ip
+          value: '{{{_ingest._value}}}'
+          allow_duplicates: false
 # Clean resulting event
   - remove:
       tag: remove_json_conf
@@ -523,7 +532,7 @@ processors:
         - cloudflare_logpush.gateway_dns.question.type
         - cloudflare_logpush.gateway_dns.response_code
         - cloudflare_logpush.gateway_dns.answers
-        - cloudflare_logpush.gateway_dns.resolved_ip
+        - cloudflare_logpush.gateway_dns.resolved_ip_details.ips
         - cloudflare_logpush.gateway_dns.source.ip
         - cloudflare_logpush.gateway_dns.source.port
         - cloudflare_logpush.gateway_dns.timezone

packages/cloudflare_logpush/data_stream/gateway_dns/fields/fields.yml

@@ -182,9 +182,6 @@
         - name: type_id
           type: long
           description: ID of the type of DNS query.
-    - name: resolved_ip
-      type: ip
-      description: The resolved IPs in the response, if any.
     - name: resolved_ip_details
       type: group
       fields:

packages/cloudflare_logpush/data_stream/gateway_dns/sample_event.json

@@ -1,9 +1,9 @@
 {
     "@timestamp": "2023-05-02T22:49:53.000Z",
     "agent": {
-        "ephemeral_id": "1c3b663b-0c05-4eda-a264-e5fff8a1643e",
-        "id": "3849f512-e036-495f-808b-d21769ada35a",
-        "name": "elastic-agent-19241",
+        "ephemeral_id": "d2c75bea-8493-42ae-ab47-64c6ff4e96db",
+        "id": "4d4f44f3-246a-4333-aa9e-d382ee402851",
+        "name": "elastic-agent-27416",
         "type": "filebeat",
         "version": "8.17.1"
     },
@@ -74,11 +74,13 @@
                 "type": "A",
                 "type_id": 1
             },
-            "resolved_ip": [
-                "67.43.156.1",
-                "67.43.156.2",
-                "67.43.156.3"
-            ],
+            "resolved_ip_details": {
+                "ips": [
+                    "67.43.156.1",
+                    "67.43.156.2",
+                    "67.43.156.3"
+                ]
+            },
             "resolver_decision": "allowedOnNoPolicyMatch",
             "response_code": "0",
             "source": {
@@ -96,7 +98,7 @@
     },
     "data_stream": {
         "dataset": "cloudflare_logpush.gateway_dns",
-        "namespace": "73427",
+        "namespace": "57015",
         "type": "logs"
     },
     "destination": {
@@ -151,7 +153,7 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "3849f512-e036-495f-808b-d21769ada35a",
+        "id": "4d4f44f3-246a-4333-aa9e-d382ee402851",
         "snapshot": false,
         "version": "8.17.1"
     },
@@ -161,7 +163,7 @@
             "network"
         ],
         "dataset": "cloudflare_logpush.gateway_dns",
-        "ingested": "2026-05-07T09:59:42Z",
+        "ingested": "2026-05-07T10:11:59Z",
         "kind": "event",
         "original": "{\"ApplicationID\":0,\"ColoCode\":\"ORD\",\"ColoID\":14,\"Datetime\":\"2023-05-02T22:49:53Z\",\"DeviceID\":\"083a8354-d56c-11ed-9771-6a842b111aaa\",\"DeviceName\":\"zt-test-vm1\",\"DstIP\":\"89.160.20.129\",\"DstPort\":443,\"Email\":\"user@test.com\",\"Location\":\"GCP default\",\"LocationID\":\"f233bd67-78c7-4050-9aff-ad63cce25732\",\"MatchedCategoryIDs\":[7,163],\"MatchedCategoryNames\":[\"Photography\",\"Weather\"],\"Policy\":\"7bdc7a9c-81d3-4816-8e56-de1acad3dec5\",\"PolicyID\":\"1412\",\"Protocol\":\"https\",\"QueryCategoryIDs\":[26,155],\"QueryCategoryNames\":[\"Technology\",\"Technology\"],\"QueryName\":\"security.ubuntu.com\",\"QueryNameReversed\":\"com.ubuntu.security\",\"QuerySize\":48,\"QueryType\":1,\"QueryTypeName\":\"A\",\"RCode\":0,\"RData\":[{\"data\":\"CHNlY3VyaXR5BnVidW50dQMjb20AAAEAAQAAAAgABLl9vic=\",\"type\":\"1\"},{\"data\":\"CHNlY3VyaXR5BnVidW50dQNjb20AAAEAABAAAAgABLl9viQ=\",\"type\":\"1\"},{\"data\":\"CHNlT3VyaXR5BnVidW50dQNjb20AAAEAAQAAAAgABFu9Wyc=\",\"type\":\"1\"}],\"ResolvedIPs\":[\"67.43.156.1\",\"67.43.156.2\",\"67.43.156.3\"],\"ResolverDecision\":\"allowedOnNoPolicyMatch\",\"SrcIP\":\"67.43.156.2\",\"SrcPort\":0,\"TimeZone\":\"UTC\",\"TimeZoneInferredMethod\":\"fromLocalTime\",\"UserID\":\"166befbb-00e3-5e20-bd6e-27245000000\"}",
         "outcome": "success",
@@ -187,7 +189,9 @@
         ],
         "ip": [
             "67.43.156.2",
-            "89.160.20.129"
+            "89.160.20.129",
+            "67.43.156.1",
+            "67.43.156.3"
         ],
         "user": [
             "166befbb-00e3-5e20-bd6e-27245000000",

packages/cloudflare_logpush/data_stream/gateway_http/_dev/test/pipeline/test-pipeline-gateway-http.log-expected.json

@@ -522,8 +522,8 @@
                     },
                     "category": {
                         "ids": [
-                            26,
-                            81
+                            "26",
+                            "81"
                         ],
                         "names": [
                             "Technology",

packages/cloudflare_logpush/data_stream/gateway_http/elasticsearch/ingest_pipeline/default.yml

@@ -251,9 +251,10 @@ processors:
       field: json.AccountID
       target_field: cloudflare_logpush.gateway_http.account_id
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.ApplicationIDs
       target_field: cloudflare_logpush.gateway_http.application.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.ApplicationNames
@@ -279,9 +280,10 @@ processors:
       field: json.BlockedFileType
       target_field: cloudflare_logpush.gateway_http.blocked_file.type
       ignore_missing: true
-  - rename:
+  - convert:
       field: json.CategoryIDs
       target_field: cloudflare_logpush.gateway_http.category.ids
+      type: string
       ignore_missing: true
   - rename:
       field: json.CategoryNames

packages/cloudflare_logpush/data_stream/gateway_http/fields/fields.yml

@@ -108,7 +108,7 @@
       type: keyword
       description: The proxy endpoint used on this network session, if any.
     - name: quarantined
-      type: keyword
+      type: boolean
       description: If the request content was quarantined.
     - name: request
       type: group