[AWS] Handle duplicate fields in Network Firewall Logs data stream

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.3.1"
+  changes:
+    - description: Fix handling of duplicate fields in Network Firewall Logs data stream.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/13676
 - version: "3.3.0"
   changes:
     - description: Remove deprecated httpjson input for cloudtrail data stream.

packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log

@@ -1,2 +1,3 @@
 {"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636381332","event":{"timestamp":"2021-11-08T14:22:12.637611+0000","flow_id":706471429191862,"event_type":"alert","src_ip":"81.2.69.143","src_port":51254,"dest_ip":"216.160.83.57","dest_port":80,"proto":"TCP","alert":{"action":"blocked","signature_id":1000003,"rev":1,"signature":"Deny all other TCP traffic","category":"","severity":3},"http":{"hostname":"216.160.83.57","url":"/","http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36","http_method":"GET","protocol":"HTTP/1.1","length":0},"app_proto":"http"}}
-{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636395643","event":{"timestamp":"2021-11-08T18:20:43.324542+0000","flow_id":625409144331688,"event_type":"netflow","src_ip":"216.160.83.61","src_port":61953,"dest_ip":"89.160.20.156","dest_port":5060,"proto":"TCP","netflow":{"pkts":1,"bytes":40,"start":"2021-11-08T18:18:11.990632+0000","end":"2021-11-08T18:18:11.990632+0000","age":0,"min_ttl":221,"max_ttl":221},"tcp":{"tcp_flags":"02","syn":true}}}
+{"firewall_name":"AWSNetworkFirewall","availability_zone":"us-east-2a","event_timestamp":"1636395643","event":{"timestamp":"2021-11-08T18:20:43.324542+0000","flow_id":625409144331688,"event_type":"netflow","src_ip":"216.160.83.61","src_port":61953,"dest_ip":"89.160.20.156","dest_port":5060,"proto":"TCP","netflow":{"pkts":1,"bytes":40,"start":"2021-11-08T18:18:11.990632+0000","end":"2021-11-08T18:18:11.990632+0000","age":0,"min_ttl":221,"max_ttl":221},"tcp":{"tcp_flags":"02","syn":true}}}
+{"firewall_name":"outbound","availability_zone":"eu-central-1a","event_timestamp":"1674225318","event":{"app_proto":"tls","src_ip":"216.160.83.61","src_port":5302,"event_type":"alert","alert":{"severity":3,"signature_id":4,"rev":0,"signature":"aws:alert_established action","action":"allowed","category":""},"flow_id":1829676028160453,"dest_ip":"89.160.20.156","proto":"TCP","tls":{"subject":"CN=sqs.eu-central-1.amazonaws.com","issuerdn":"C=US, O=Amazon, OU=Server CA 1B, CN=Amazon","subject":"CN=sqs.eu-central-1.amazonaws.com","issuerdn":"C=US, O=Amazon, OU=Server CA 1B, CN=Amazon","serial":"08:C9:76:68:CB:30:31:1C:B2:24:A4:B3:22:F0:16:29","fingerprint":"34:36:32:fb:05:65:83:55:a0:8e:24:7b:41:52:bf:98:88:3b:bf:9b","sni":"sqs.eu-central-1.amazonaws.com","version":"TLS 1.2","notbefore":"2022-11-03T00:00:00","notafter":"2023-10-16T23:59:59","ja3":{},"ja3s":{}},"dest_port":443,"timestamp":"2023-01-20T14:35:18.307095+0000"}}

packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json

@@ -217,6 +217,118 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2023-01-20T14:35:18.307Z",
+            "aws": {
+                "firewall": {
+                    "flow": {
+                        "id": "1829676028160453"
+                    }
+                }
+            },
+            "cloud": {
+                "availability_zone": "eu-central-1a",
+                "region": "eu-central-1"
+            },
+            "destination": {
+                "address": "89.160.20.156",
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "domain": "sqs.eu-central-1.amazonaws.com",
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.156",
+                "port": 443
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "kind": "alert",
+                "original": "{\"firewall_name\":\"outbound\",\"availability_zone\":\"eu-central-1a\",\"event_timestamp\":\"1674225318\",\"event\":{\"app_proto\":\"tls\",\"src_ip\":\"216.160.83.61\",\"src_port\":5302,\"event_type\":\"alert\",\"alert\":{\"severity\":3,\"signature_id\":4,\"rev\":0,\"signature\":\"aws:alert_established action\",\"action\":\"allowed\",\"category\":\"\"},\"flow_id\":1829676028160453,\"dest_ip\":\"89.160.20.156\",\"proto\":\"TCP\",\"tls\":{\"subject\":\"CN=sqs.eu-central-1.amazonaws.com\",\"issuerdn\":\"C=US, O=Amazon, OU=Server CA 1B, CN=Amazon\",\"subject\":\"CN=sqs.eu-central-1.amazonaws.com\",\"issuerdn\":\"C=US, O=Amazon, OU=Server CA 1B, CN=Amazon\",\"serial\":\"08:C9:76:68:CB:30:31:1C:B2:24:A4:B3:22:F0:16:29\",\"fingerprint\":\"34:36:32:fb:05:65:83:55:a0:8e:24:7b:41:52:bf:98:88:3b:bf:9b\",\"sni\":\"sqs.eu-central-1.amazonaws.com\",\"version\":\"TLS 1.2\",\"notbefore\":\"2022-11-03T00:00:00\",\"notafter\":\"2023-10-16T23:59:59\",\"ja3\":{},\"ja3s\":{}},\"dest_port\":443,\"timestamp\":\"2023-01-20T14:35:18.307095+0000\"}}",
+                "severity": 3,
+                "type": [
+                    "connection",
+                    "allowed"
+                ]
+            },
+            "message": "",
+            "network": {
+                "community_id": "1:RHGtRWh3v3jsfbwi5QJWf4w5P6w=",
+                "protocol": "tls",
+                "transport": "tcp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "name": "outbound",
+                "product": "Network Firewall",
+                "type": "firewall",
+                "vendor": "AWS"
+            },
+            "related": {
+                "ip": [
+                    "216.160.83.61",
+                    "89.160.20.156"
+                ]
+            },
+            "rule": {
+                "id": "4",
+                "name": "aws:alert_established action"
+            },
+            "source": {
+                "address": "216.160.83.61",
+                "as": {
+                    "number": 209
+                },
+                "geo": {
+                    "city_name": "Milton",
+                    "continent_name": "North America",
+                    "country_iso_code": "US",
+                    "country_name": "United States",
+                    "location": {
+                        "lat": 47.2513,
+                        "lon": -122.3149
+                    },
+                    "region_iso_code": "US-WA",
+                    "region_name": "Washington"
+                },
+                "ip": "216.160.83.61",
+                "port": 5302
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "tls": {
+                "client": {
+                    "server_name": "sqs.eu-central-1.amazonaws.com"
+                },
+                "server": {
+                    "x509": {
+                        "not_after": "2023-10-16T23:59:59.000Z",
+                        "not_before": "2022-11-03T00:00:00.000Z"
+                    }
+                },
+                "version": "1.2",
+                "version_protocol": "tls"
+            }
         }
     ]
-}
+}

packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml

@@ -17,6 +17,7 @@ processors:
       description: 'The `message` field is no longer required if the document has an `event.original` field.'
   - json:
       field: event.original
+      allow_duplicate_keys: true
       target_field: json
   - date:
       field: json.event.timestamp

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.3.1
 name: aws
 title: AWS
-version: 3.3.0
+version: 3.3.1
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories: