From ee437f5c119ebd62fa18332cced9aaf7a5b0ad36 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Thu, 7 May 2026 14:23:14 -0400 Subject: [PATCH 01/19] [Azure Logs] Add aadgraphactivitylogs data stream Adds the azure.aadgraphactivitylogs data stream to ingest the AzureADGraphActivityLogs diagnostic category from Microsoft Entra ID, parallel to azure.graphactivitylogs for Microsoft Graph. Without this, AAD Graph events fall through to azure.platformlogs and the AAD-Graph-specific properties survive only inside event.original. --- packages/azure/changelog.yml | 5 + .../test/pipeline/test-aadgraph-activity.log | 4 + .../test-aadgraph-activity.log-expected.json | 556 +++++++++++++++ .../_dev/test/pipeline/test-common-config.yml | 7 + .../agent/stream/azure-eventhub.yml.hbs | 89 +++ .../agent/stream/log.yml.hbs | 20 + .../ingest_pipeline/azure-shared-pipeline.yml | 101 +++ .../elasticsearch/ingest_pipeline/default.yml | 644 ++++++++++++++++++ .../aadgraphactivitylogs/fields/agent.yml | 12 + .../fields/base-fields.yml | 20 + .../fields/ecs-extended.yml | 12 + .../aadgraphactivitylogs/fields/ecs.yml | 6 + .../aadgraphactivitylogs/fields/fields.yml | 116 ++++ .../fields/package-fields.yml | 42 ++ .../aadgraphactivitylogs/manifest.yml | 165 +++++ .../aadgraphactivitylogs/sample_event.json | 138 ++++ .../elasticsearch/ingest_pipeline/default.yml | 4 + .../data_stream/events/routing_rules.yml | 5 + packages/azure/manifest.yml | 2 +- 19 files changed, 1947 insertions(+), 1 deletion(-) create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/agent/stream/azure-eventhub.yml.hbs create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/agent.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/base-fields.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/ecs.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/fields/package-fields.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/manifest.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/sample_event.json diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 00b47f187ad..e0c885c1ad0 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.37.0-preview1" + changes: + - description: Add azure.aadgraphactivitylogs dataset to ingest legacy Azure AD Graph (graph.windows.net) AzureADGraphActivityLogs diagnostic events. Local TRaDE research build, not for upstream. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 - version: "1.36.1" changes: - description: Add missing event.kind pipeline_error handling to ingest pipeline on_failure handlers. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log new file mode 100644 index 00000000000..96dc0d56a20 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log @@ -0,0 +1,4 @@ +{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000001-0001-0001-0001-000000000001","durationMs":59,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":59,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000001-0001-0001-0001-000000000001","requestId":"00000001-0001-0001-0001-000000000001","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users","responseSizeBytes":54662,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:19:33.5368860Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:19:33.5368860Z"} +{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000002-0002-0002-0002-000000000002","durationMs":18,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":18,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000002-0002-0002-0002-000000000002","requestId":"00000002-0002-0002-0002-000000000002","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles","responseSizeBytes":258,"responseStatusCode":400,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:15:45.0849115Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"400","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:15:45.0849115Z"} +{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000003-0003-0003-0003-000000000003","durationMs":57,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":57,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000003-0003-0003-0003-000000000003","requestId":"00000003-0003-0003-0003-000000000003","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails","responseSizeBytes":13324,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"BBBBBBBBBBBBBBBBBBBBBB==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T16:12:53.7641499Z","tokenIssuedAt":"2026-05-07T16:05:34Z","userAgent":"azure-graph-test-probe/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T16:12:53.7641499Z"} +{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000004-0004-0004-0004-000000000004","durationMs":71,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":71,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000004-0004-0004-0004-000000000004","requestId":"00000004-0004-0004-0004-000000000004","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications","responseSizeBytes":10026,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:15:44.4909654Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:15:44.4909654Z"} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json new file mode 100644 index 00000000000..46231ac5589 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json @@ -0,0 +1,556 @@ +{ + "expected": [ + { + "@timestamp": "2026-05-07T15:19:33.536Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000001-0001-0001-0001-000000000001", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", + "time_generated": "2026-05-07T15:19:33.536Z", + "token_issued_at": "2026-05-07T13:50:39.000Z" + }, + "result_signature": "200" + }, + "correlation_id": "00000001-0001-0001-0001-000000000001", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "users-read", + "category": [ + "iam", + "web" + ], + "duration": 59000000, + "id": "00000001-0001-0001-0001-000000000001", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "outcome": "success", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000001-0001-0001-0001-000000000001", + "method": "GET" + }, + "response": { + "bytes": 54662, + "status_code": 200 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-client/1.0" + } + }, + { + "@timestamp": "2026-05-07T15:15:45.084Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000002-0002-0002-0002-000000000002", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", + "time_generated": "2026-05-07T15:15:45.084Z", + "token_issued_at": "2026-05-07T13:50:39.000Z" + }, + "result_signature": "400" + }, + "correlation_id": "00000002-0002-0002-0002-000000000002", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "directory-roles-read", + "category": [ + "iam", + "web" + ], + "duration": 18000000, + "id": "00000002-0002-0002-0002-000000000002", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000002-0002-0002-0002-000000000002\",\"durationMs\":18,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":18,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000002-0002-0002-0002-000000000002\",\"requestId\":\"00000002-0002-0002-0002-000000000002\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles\",\"responseSizeBytes\":258,\"responseStatusCode\":400,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:15:45.0849115Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"400\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:15:45.0849115Z\"}", + "outcome": "failure", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000002-0002-0002-0002-000000000002", + "method": "GET" + }, + "response": { + "bytes": 258, + "status_code": 400 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-client/1.0" + } + }, + { + "@timestamp": "2026-05-07T16:12:53.764Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000003-0003-0003-0003-000000000003", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "BBBBBBBBBBBBBBBBBBBBBB==", + "time_generated": "2026-05-07T16:12:53.764Z", + "token_issued_at": "2026-05-07T16:05:34.000Z" + }, + "result_signature": "200" + }, + "correlation_id": "00000003-0003-0003-0003-000000000003", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "tenant-details-read", + "category": [ + "iam", + "web" + ], + "duration": 57000000, + "id": "00000003-0003-0003-0003-000000000003", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000003-0003-0003-0003-000000000003\",\"durationMs\":57,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":57,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000003-0003-0003-0003-000000000003\",\"requestId\":\"00000003-0003-0003-0003-000000000003\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails\",\"responseSizeBytes\":13324,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"BBBBBBBBBBBBBBBBBBBBBB==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T16:12:53.7641499Z\",\"tokenIssuedAt\":\"2026-05-07T16:05:34Z\",\"userAgent\":\"azure-graph-test-probe/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T16:12:53.7641499Z\"}", + "outcome": "success", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000003-0003-0003-0003-000000000003", + "method": "GET" + }, + "response": { + "bytes": 13324, + "status_code": 200 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "BBBBBBBBBBBBBBBBBBBBBB==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-probe/1.0" + } + }, + { + "@timestamp": "2026-05-07T15:15:44.490Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000004-0004-0004-0004-000000000004", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", + "time_generated": "2026-05-07T15:15:44.490Z", + "token_issued_at": "2026-05-07T13:50:39.000Z" + }, + "result_signature": "200" + }, + "correlation_id": "00000004-0004-0004-0004-000000000004", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "applications-read", + "category": [ + "iam", + "web" + ], + "duration": 71000000, + "id": "00000004-0004-0004-0004-000000000004", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000004-0004-0004-0004-000000000004\",\"durationMs\":71,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":71,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000004-0004-0004-0004-000000000004\",\"requestId\":\"00000004-0004-0004-0004-000000000004\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications\",\"responseSizeBytes\":10026,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:15:44.4909654Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:15:44.4909654Z\"}", + "outcome": "success", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000004-0004-0004-0004-000000000004", + "method": "GET" + }, + "response": { + "bytes": 10026, + "status_code": 200 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-client/1.0" + } + } + ] +} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..f00b11ca2b6 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,7 @@ +fields: + tags: + - preserve_original_event +dynamic_fields: + # This can be removed after ES 8.14 is the minimum version. + # Relates: https://github.com/elastic/elasticsearch/pull/105689 + url.extension: '^.*$' diff --git a/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/azure-eventhub.yml.hbs new file mode 100644 index 00000000000..266f1fcff76 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/azure-eventhub.yml.hbs @@ -0,0 +1,89 @@ +{{#if auth_type}} +auth_type: {{auth_type}} +{{/if}} +{{#if connection_string}} +connection_string: {{connection_string}} +{{/if}} +{{#if eventhub_namespace}} +eventhub_namespace: {{eventhub_namespace}} +{{/if}} +{{#if tenant_id}} +tenant_id: {{tenant_id}} +{{/if}} +{{#if client_id}} +client_id: {{client_id}} +{{/if}} +{{#if client_secret}} +client_secret: {{client_secret}} +{{/if}} +{{#contains "client_secret" auth_type}} +{{#if authority_host}} +authority_host: {{authority_host}} +{{/if}} +{{/contains}} +{{#if storage_account_container}} +storage_account_container: {{storage_account_container}} +{{else}} +{{#if eventhub}} +storage_account_container: filebeat-aadgraphactivitylogs-{{eventhub}} +{{/if}} +{{/if}} +{{#if eventhub}} +eventhub: {{eventhub}} +{{/if}} +{{#if consumer_group}} +consumer_group: {{consumer_group}} +{{/if}} +{{#if storage_account}} +storage_account: {{storage_account}} +{{/if}} +{{#if storage_account_key}} +storage_account_key: {{storage_account_key}} +storage_account_connection_string: DefaultEndpointsProtocol=https;AccountName={{storage_account}};AccountKey={{storage_account_key}};EndpointSuffix={{endpoint_suffix}} +{{/if}} +{{#if resource_manager_endpoint}} +resource_manager_endpoint: {{resource_manager_endpoint}} +{{/if}} +{{#if processor_version}} +processor_version: {{processor_version}} +{{/if}} +{{#if migrate_checkpoint}} +migrate_checkpoint: {{migrate_checkpoint}} +{{/if}} +{{#if processor_update_interval}} +processor_update_interval: {{processor_update_interval}} +{{/if}} +{{#if processor_start_position}} +processor_start_position: {{processor_start_position}} +{{/if}} +{{#if partition_receive_timeout}} +partition_receive_timeout: {{partition_receive_timeout}} +{{/if}} +{{#if partition_receive_count}} +partition_receive_count: {{partition_receive_count}} +{{/if}} +{{#if transport}} +transport: {{transport}} +{{/if}} + +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +sanitize_options: +{{#if sanitize_newlines}} + - NEW_LINES +{{/if}} +{{#if sanitize_singlequotes}} + - SINGLE_QUOTES +{{/if}} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..89d35fecd80 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs @@ -0,0 +1,20 @@ +paths: + {{#each paths as |path|}} +- {{path}} + {{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +allow_deprecated_use: true +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml new file mode 100644 index 00000000000..d7d26e8d31d --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml @@ -0,0 +1,101 @@ +--- +description: Pipeline for parsing Azure shared log fields. +processors: + - set: + field: cloud.provider + value: azure + - grok: + field: azure.resource_id + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule} + - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + PROVIDERNAME: .+ + NAMESPACE: .+ + RULE: .+ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+. + NAME: ((?!AUTHORIZATIONRULES).)*$ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+ + NAME: .+ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /providers/%{PROVIDER:azure.resource.provider} + - /PROVIDERS/%{PROVIDER:azure.resource.provider} + pattern_definitions: + PROVIDER: .+ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider} + - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group} + - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + ignore_failure: true + - grok: + field: azure.resource_id + if: 'ctx.azure?.subscription_id == null' + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id} + - /subscriptions/%{SUBID:azure.subscription_id} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + ignore_failure: true + - rename: + field: azure.resource_id + target_field: azure.resource.id + ignore_missing: true + - lowercase: + field: event.outcome + ignore_missing: true +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..2d46cce9060 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,644 @@ +--- +description: Pipeline for parsing azure AAD Graph activity logs. +processors: + - set: + field: ecs.version + value: '8.11.0' + tag: set-ecs_version + - set: + field: event.kind + value: event + tag: set-event_kind + - append: + field: event.type + value: access + tag: append-event_type + - rename: + field: azure + target_field: azure-eventhub + ignore_missing: true + tag: rename-azure + - remove: + field: routing.category + ignore_missing: true + description: Remove the routing.category field if it exists. + - rename: + field: message + target_field: event.original + ignore_missing: true + if: 'ctx.event?.original == null' + tag: rename-message + description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' + - remove: + field: message + ignore_missing: true + if: 'ctx.event?.original != null' + description: 'The `message` field is no longer required if the document has an `event.original` field.' + tag: remove-message + - json: + field: event.original + target_field: azure.aadgraphactivitylogs + tag: json-event_original + - script: + description: Convert Azure JSON keys to snake case. + tag: azure-json-keys-to-snake-case + lang: painless + source: | + Map keysToSnakeCase(Map m) { + def regex = /_?([a-z])([A-Z]+)/; + def out = [:]; + + for (entry in m.entrySet()) { + def k = entry.getKey(); + def v = entry.getValue(); + + if (v instanceof Map) { + v = keysToSnakeCase(v); + } else if (v instanceof List) { + for (int i = 0; i < v.size(); i++) { + def item = v.get(i); + if (item instanceof Map) { + v.set(i, keysToSnakeCase(item)); + } + } + } + + k = regex.matcher(k).replaceAll('$1_$2').toLowerCase(); + out.put(k, v); + } + + return out; + } + + ctx.azure['aadgraphactivitylogs'] = keysToSnakeCase(ctx.azure.aadgraphactivitylogs); + - date: + field: azure.aadgraphactivitylogs.time + target_field: '@timestamp' + if: ctx.azure?.aadgraphactivitylogs?.time != null + tag: date-time + # Order is prioritized by observed frequency for this log + # category. + # We expect the first format to be used in 99% of cases. + # However, we include the other formats seen in other + # categories as a necessary fallback. + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" + - "M/d/yyyy h:mm:ss a" + - "M/d/yyyy H:mm:ss" + - yyyy-MM-dd'T'H:mm:ss.SSS'Z' + - remove: + field: azure.aadgraphactivitylogs.time + ignore_missing: true + tag: remove-time + - date: + field: azure.aadgraphactivitylogs.properties.token_issued_at + target_field: azure.aadgraphactivitylogs.properties.token_issued_at + if: ctx.azure?.aadgraphactivitylogs?.properties?.token_issued_at != null + tag: date-token_issued_at + # Order is prioritized by observed frequency for this log + # category. + # We expect the first format to be used in 99% of cases. + # However, we include the other formats seen in other + # categories as a necessary fallback. + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" + - "M/d/yyyy h:mm:ss a" + - "M/d/yyyy H:mm:ss" + - yyyy-MM-dd'T'H:mm:ss.SSS'Z' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: azure.aadgraphactivitylogs.resource_id + target_field: azure.resource_id + ignore_missing: true + tag: rename-resource_id + - convert: + field: azure.aadgraphactivitylogs.caller_ip_address + target_field: source.ip + type: ip + ignore_missing: true + tag: convert-caller_ip_address + on_failure: + - rename: + field: azure.aadgraphactivitylogs.caller_ip_address + target_field: source.address + ignore_missing: true + tag: rename-caller_ip_address + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: azure.aadgraphactivitylogs.properties.ip_address + target_field: source.ip + type: ip + ignore_missing: true + tag: convert-properties_ip_address + if: ctx.source?.ip == null + on_failure: + - rename: + field: azure.aadgraphactivitylogs.properties.ip_address + target_field: source.address + ignore_missing: true + tag: rename-properties_ip_address + if: ctx.source?.address == null + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + - remove: + field: + - azure.aadgraphactivitylogs.caller_ip_address + - azure.aadgraphactivitylogs.properties.ip_address + if: 'ctx.source?.ip != null' + tag: remove-caller_ip_address + ignore_missing: true + - set: + field: client.ip + value: '{{source.ip}}' + tag: set-client_ip + ignore_empty_value: true + - set: + field: service.id + copy_from: azure.aadgraphactivitylogs.properties.service_principal_id + tag: set_service_id + ignore_empty_value: true + - set: + field: user.id + copy_from: azure.aadgraphactivitylogs.properties.service_principal_id + tag: set_user_id_from_service_principal_id + if: ctx.user?.id == null + ignore_empty_value: true + - set: + field: device.id + copy_from: azure.aadgraphactivitylogs.properties.device_id + tag: set_device_id + ignore_empty_value: true + - set: + field: session.id + copy_from: azure.aadgraphactivitylogs.properties.session_id + tag: set_session_id + ignore_empty_value: true + - set: + field: token.id + copy_from: azure.aadgraphactivitylogs.properties.sign_in_activity_id + tag: set_token_id + ignore_empty_value: true + - append: + field: related.ip + tag: append-related_ip + value: '{{source.ip}}' + allow_duplicates: false + if: 'ctx.source?.ip != null' + - convert: + field: azure.aadgraphactivitylogs.level + target_field: log.level + type: string + tag: convert-level + ignore_missing: true + - remove: + field: azure.aadgraphactivitylogs.level + ignore_missing: true + tag: remove-level + - convert: + field: azure.aadgraphactivitylogs.duration_ms + target_field: event.duration + type: long + if: ctx.azure?.aadgraphactivitylogs?.duration_ms != null + tag: convert-duration_ms + ignore_failure: true + - convert: + field: azure.aadgraphactivitylogs.properties.duration_ms + target_field: event.duration + type: long + if: ctx.azure?.aadgraphactivitylogs?.properties?.duration_ms != null && ctx.event?.duration == null + tag: convert-properties-duration_ms + ignore_failure: true + - script: + lang: painless + source: | + if (ctx.event.duration!= null) { + ctx.event.duration = ctx.event.duration * 1000000; + } + ignore_failure: true + tag: script-duration + - remove: + field: + - azure.aadgraphactivitylogs.duration_ms + - azure.aadgraphactivitylogs.properties.duration_ms + tag: remove-duration_ms + if: ctx.event?.duration != null + ignore_missing: true + - rename: + field: azure.aadgraphactivitylogs.location + target_field: destination.geo.region_name + ignore_missing: true + tag: rename-location + - rename: + field: azure.aadgraphactivitylogs.properties.location + target_field: destination.geo.region_name + if: ctx.destination?.geo?.region_name == null && ctx.azure?.aadgraphactivitylogs?.properties?.location != null + tag: rename-properties-location + - remove: + field: + - azure.aadgraphactivitylogs.location + - azure.aadgraphactivitylogs.properties.location + tag: remove-location + if: ctx.destination?.geo?.region_name != null + ignore_missing: true + - convert: + field: azure.aadgraphactivitylogs.operation_name + target_field: event.action + type: string + ignore_missing: true + tag: convert-operation_name + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - rename: + field: azure.aadgraphactivitylogs.tenant_id + target_field: azure.tenant_id + ignore_missing: true + tag: rename-tenant_id + - rename: + field: azure.aadgraphactivitylogs.properties.tenant_id + target_field: azure.tenant_id + ignore_missing: true + if: ctx.azure?.tenant_id == null + tag: rename-properties_tenant_id + - remove: + field: + - azure.aadgraphactivitylogs.tenant_id + - azure.aadgraphactivitylogs.properties.tenant_id + tag: remove-tenant_id + if: ctx.azure?.tenant_id != null + ignore_missing: true + - rename: + field: azure.aadgraphactivitylogs.correlation_id + target_field: azure.correlation_id + ignore_missing: true + tag: rename-correlation_id + - rename: + field: azure.aadgraphactivitylogs.properties.request_id + target_field: http.request.id + ignore_missing: true + tag: rename-properties_request_id + - rename: + field: azure.aadgraphactivitylogs.properties.request_method + target_field: http.request.method + ignore_missing: true + tag: rename-properties_request_methods + - uri_parts: + field: azure.aadgraphactivitylogs.properties.request_uri + if: ctx.azure?.aadgraphactivitylogs?.properties?.request_uri != null + keep_original: true + tag: uri_parts-properties-request_uri + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: azure.aadgraphactivitylogs.properties.request_uri + tag: remove-properties_request_uri + if: ctx.url?.original != null + ignore_missing: true + - rename: + field: azure.aadgraphactivitylogs.properties.response_size_bytes + target_field: http.response.bytes + ignore_missing: true + tag: rename-properties_response_bytes + - rename: + field: azure.aadgraphactivitylogs.properties.response_status_code + target_field: http.response.status_code + ignore_missing: true + tag: rename-properties_response_statuscode + - split: + field: azure.aadgraphactivitylogs.properties.roles + if: ctx.azure?.aadgraphactivitylogs?.properties?.roles != null && ctx.azure.aadgraphactivitylogs.properties.roles != '' + separator: ' ' + tag: split-properties_roles + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - split: + field: azure.aadgraphactivitylogs.properties.scopes + if: ctx.azure?.aadgraphactivitylogs?.properties?.scopes != null && ctx.azure.aadgraphactivitylogs.properties.scopes != '' + separator: ' ' + tag: split-properties_scopes + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - split: + field: azure.aadgraphactivitylogs.properties.wids + if: ctx.azure?.aadgraphactivitylogs?.properties?.wids != null && ctx.azure.aadgraphactivitylogs.properties.wids != '' + separator: ' ' + tag: split-properties_wids + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - user_agent: + field: azure.aadgraphactivitylogs.properties.user_agent + if: ctx.azure?.aadgraphactivitylogs?.properties?.user_agent != null && ctx.azure.aadgraphactivitylogs.properties.user_agent != '' + tag: user_agent-properties_user_agent + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - remove: + field: azure.aadgraphactivitylogs.properties.user_agent + tag: remove-properties_user_agent + if: ctx.user_agent != null + ignore_missing: true + - rename: + field: azure.aadgraphactivitylogs.properties.user_id + target_field: user.id + tag: rename-properties_user_id + if: ctx.azure?.aadgraphactivitylogs?.properties?.user_id != null && ctx.azure.aadgraphactivitylogs.properties.user_id != '' + - set: + field: client.user.id + value: '{{user.id}}' + tag: set-client_ip + ignore_empty_value: true + - append: + field: related.user + value: '{{user.id}}' + tag: append-related_user + allow_duplicates: false + if: 'ctx.user?.id != null' + - rename: + field: azure.aadgraphactivitylogs.properties._billed_size + target_field: azure.aadgraphactivitylogs.properties.billed_size + if: ctx.azure?.aadgraphactivitylogs?.properties?._billed_size != null + tag: rename-properties_billed_size + - rename: + field: azure.aadgraphactivitylogs.properties._is_billable + target_field: azure.aadgraphactivitylogs.properties.is_billable + if: ctx.azure?.aadgraphactivitylogs?.properties?._is_billable != null + tag: rename-properties_is_billable + - convert: + field: azure.aadgraphactivitylogs.properties.is_billable + type: boolean + tag: convert-properties_is_billable + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - convert: + field: azure.aadgraphactivitylogs.properties.client_auth_method + type: integer + tag: convert-properties_client_auth_method + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: azure.aadgraphactivitylogs.properties.time_generated + target_field: azure.aadgraphactivitylogs.properties.time_generated + if: ctx.azure?.aadgraphactivitylogs?.properties?.time_generated != null + tag: date-properties_time_generated + # Order is prioritized by observed frequency for this log + # category. + # We expect the first format to be used in 99% of cases. + # However, we include the other formats seen in other + # categories as a necessary fallback. + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" + - "M/d/yyyy h:mm:ss a" + - "M/d/yyyy H:mm:ss" + - yyyy-MM-dd'T'H:mm:ss.SSS'Z' + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + # ECS enrichment. Runs after url.path, http.request.method, and + # http.response.status_code are populated. Derives a semantic event.action + # from method + directory collection (the operation_name field from the + # diagnostic stream is the literal string "AAD Graph Activity" and isn't + # useful for detection by itself), and sets event.category / event.type / + # event.outcome / event.id following ECS conventions used by sibling Entra + # datasets (signinlogs, auditlogs). + - set: + field: event.category + value: + - iam + - web + tag: set-event_category + - set: + field: event.type + value: + - access + - info + tag: set-event_type-info + if: ctx.http?.request?.method == null || ctx.http.request.method == 'GET' || ctx.http.request.method == 'HEAD' + - set: + field: event.type + value: + - access + - change + tag: set-event_type-change + if: ctx.http?.request?.method == 'POST' || ctx.http?.request?.method == 'PATCH' || ctx.http?.request?.method == 'PUT' + - set: + field: event.type + value: + - access + - deletion + tag: set-event_type-deletion + if: ctx.http?.request?.method == 'DELETE' + - set: + field: event.outcome + value: success + tag: set-event_outcome-success + if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 200 && ctx.http.response.status_code < 300 + - set: + field: event.outcome + value: failure + tag: set-event_outcome-failure + if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 400 + - set: + field: event.outcome + value: unknown + tag: set-event_outcome-unknown + if: ctx.http?.response?.status_code != null && ctx.http.response.status_code >= 300 && ctx.http.response.status_code < 400 + - set: + field: event.id + copy_from: http.request.id + tag: set-event_id + ignore_empty_value: true + + # actor_type and app_id stay in azure.aadgraphactivitylogs.properties.* — + # ECS doesn't have clean canonical fields for "OAuth client/app id" or for + # "person vs service principal" (sibling integrations like AWS CloudTrail + # and Okta also keep these in their dataset namespaces). Both fields are + # declared in fields/fields.yml and are top-level queryable. + # + # We do append app_id to related.user so cross-event correlation pivots on + # the OAuth-client dimension work alongside the user.id pivot. This is the + # same pattern the M365/O365 integration uses for application_id. + - append: + field: related.user + value: '{{{azure.aadgraphactivitylogs.properties.app_id}}}' + tag: append-related_user-app_id + allow_duplicates: false + if: ctx.azure?.aadgraphactivitylogs?.properties?.app_id != null && ctx.azure.aadgraphactivitylogs.properties.app_id != '' + - script: + lang: painless + tag: derive-event_action + description: >- + Derive event.action from HTTP method + directory collection in the + request URI. Examples: + GET /v2/{tid}/users -> users-read, + GET /v2/{tid}/directoryRoles -> directory-roles-read, + PATCH /v2/{tid}/applications/{id} -> applications-update, + POST /v2/{tid}/$batch -> batch-execute. + Leaves the prior event.action (operation_name) in place if the URI + doesn't match the expected shape. + source: | + String path = ctx.url?.path; + String method = ctx.http?.request?.method; + if (path == null || method == null) { return; } + String[] parts = path.splitOnToken('/'); + // expected shape: ["", "v2", "", "", ...] + if (parts.length < 4) { return; } + String collection = parts[3]; + if (collection == null || collection.isEmpty()) { return; } + if (collection.startsWith('$')) { collection = collection.substring(1); } + // CamelCase -> hyphen-lower (servicePrincipals -> service-principals) + StringBuilder hyphenated = new StringBuilder(); + for (int i = 0; i < collection.length(); i++) { + char c = collection.charAt(i); + if (i > 0 && Character.isUpperCase(c) && !Character.isUpperCase(collection.charAt(i - 1))) { + hyphenated.append('-'); + } + hyphenated.append(Character.toLowerCase(c)); + } + String verb; + if (method == 'GET' || method == 'HEAD') { verb = 'read'; } + else if (method == 'POST') { verb = 'execute'; } + else if (method == 'PATCH' || method == 'PUT') { verb = 'update'; } + else if (method == 'DELETE') { verb = 'delete'; } + else { verb = method.toLowerCase(); } + ctx.event.action = hyphenated.toString() + '-' + verb; + ignore_failure: true + + - pipeline: + name: '{{ IngestPipeline "azure-shared-pipeline" }}' + tag: pipeline-azure-shared-pipeline + - fingerprint: + fields: + - azure.aadgraphactivitylogs.properties.request_uri + - http.request.id + - azure.tenant_id + - azure.aadgraphactivitylogs.properties.time_generated + target_field: _id + ignore_missing: true + tag: fingerprint-id + - set: + field: cloud.account.id + value: '{{azure.tenant_id}}' + tag: set-cloud_account_id + ignore_empty_value: true + - set: + field: cloud.region + value: '{{destination.geo.region_name}}' + tag: set-cloud_region + ignore_empty_value: true + - set: + field: cloud.service.name + value: 'Azure AD Graph' + tag: set-cloud_region + ignore_empty_value: true + - geoip: + field: client.ip + target_field: client.geo + ignore_missing: true + - geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + # IP Autonomous System (AS) Lookup + - geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + - rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + - rename: + field: client.as.asn + target_field: client.as.number + ignore_missing: true + - rename: + field: client.as.organization_name + target_field: client.as.organization.name + ignore_missing: true + - script: + tag: script_to_drop_null_values + lang: painless + description: Drops null/empty values recursively. + source: |- + boolean drop(Object o) { + if (o == null || o == '') { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); + - set: + field: event.kind + value: pipeline_error + tag: set_pipeline_error + if: ctx.error?.message != null + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/agent.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/agent.yml new file mode 100644 index 00000000000..4b15225a4d4 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/agent.yml @@ -0,0 +1,12 @@ +- name: cloud.image.id + type: keyword + description: Image ID for the cloud instance. +- name: host.containerized + type: boolean + description: If the host is a container. +- name: host.os.build + type: keyword + description: OS build information. +- name: host.os.codename + type: keyword + description: OS codename, if any. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/base-fields.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/base-fields.yml new file mode 100644 index 00000000000..35f4928764f --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset name. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: event.module + type: constant_keyword + description: Event module + value: azure +- name: event.dataset + type: constant_keyword + description: Event dataset + value: azure.aadgraphactivitylogs diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml new file mode 100644 index 00000000000..4fe793d10ad --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml @@ -0,0 +1,12 @@ +- name: session + type: group + fields: + - name: id + type: keyword + description: The unique identifier for the authentication session. +- name: token + type: group + fields: + - name: id + type: keyword + description: The unique token identifier of the API call. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs.yml new file mode 100644 index 00000000000..3b027f35f0d --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: client.geo.location + external: ecs +- name: source.geo.location + external: ecs +- name: destination.geo.region_name + external: ecs diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml new file mode 100644 index 00000000000..d26170c2fb6 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml @@ -0,0 +1,116 @@ +- name: azure.aadgraphactivitylogs + type: group + fields: + - name: category + type: keyword + description: | + Azure Event Category. For Azure AD Graph Activity Logs, this is `AzureADGraphActivityLogs`. + - name: operation_version + type: keyword + description: | + The legacy Azure AD Graph API version of the event (e.g. `1.6`). + - name: operation_name + type: keyword + description: | + Operation name. + - name: result_signature + type: keyword + description: | + Result signature. + - name: properties + type: group + description: | + Event properties. + fields: + - name: api_version + type: keyword + description: | + The API version of the event. + - name: app_id + type: keyword + description: | + The identifier for the application. + - name: at_content + type: keyword + description: | + Reserved for future use. + - name: billed_size + type: double + description: | + The record size in bytes. + - name: actor_type + type: keyword + description: | + Type of identity that issued the request, e.g. `User`, `ServicePrincipal`. + - name: device_id + type: keyword + description: | + The identifier of the device from which the authentication request originated. + - name: session_id + type: keyword + description: | + The unique identifier for the authentication session. + - name: client_auth_method + type: integer + description: | + Indicates how the client was authenticated. For a public client, the value is 0. If client ID and client secret are used, the value is 1. If a client certificate was used for authentication, the value is 2. + - name: client_request_id + type: keyword + description: | + The client request identifier when sent. If no client request identifier is sent, the value will be equal to the operation identifier. + - name: identity_provider + type: keyword + description: | + The identity provider that authenticated the subject of the token. + - name: is_billable + type: boolean + description: | + Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account. + - name: operation_id + type: keyword + description: | + The identifier for the batch. For non-batched requests, this will be unique per request. For batched requests, this will be the same for all requests in the batch. + - name: request_uri + type: keyword + description: | + The URI of the request. + - name: roles + type: keyword + description: | + The roles in token claims. + - name: scopes + type: keyword + description: | + The scopes in token claims. + - name: service_principal_id + type: keyword + description: | + The identifier of the servicePrincipal making the request. + - name: sign_in_activity_id + type: keyword + description: | + The identifier representing the sign-in activitys. + - name: source_system + type: keyword + description: | + The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics. + - name: time_generated + type: date + description: | + The date and time the request was received. + - name: token_issued_at + type: date + description: | + The timestamp the token was issued at. + - name: type + type: keyword + description: | + The name of the table. + - name: user_agent + type: keyword + description: | + The user agent information related to request. + - name: wids + type: keyword + description: | + Denotes the tenant-wide roles assigned to this user. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/package-fields.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/package-fields.yml new file mode 100644 index 00000000000..44792684f83 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/package-fields.yml @@ -0,0 +1,42 @@ +- name: azure + type: group + fields: + - name: subscription_id + type: keyword + description: | + Azure subscription ID. + - name: correlation_id + type: keyword + description: | + Correlation ID. + - name: tenant_id + type: keyword + description: | + tenant ID. + - name: resource + type: group + fields: + - name: id + type: keyword + description: | + Resource ID. + - name: group + type: keyword + description: | + Resource group. + - name: provider + type: keyword + description: | + Resource type/namespace. + - name: namespace + type: keyword + description: | + Resource type/namespace. + - name: name + type: keyword + description: | + Name. + - name: authorization_rule + type: keyword + description: | + Authorization rule. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/manifest.yml b/packages/azure/data_stream/aadgraphactivitylogs/manifest.yml new file mode 100644 index 00000000000..99a745621f4 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/manifest.yml @@ -0,0 +1,165 @@ +type: logs +title: Azure AD Graph Activity Logs +streams: + - input: "azure-eventhub" + enabled: false + template_path: "azure-eventhub.yml.hbs" + title: "Azure AD Graph Activity Logs" + description: "Collect Azure AD Graph Activity Logs using azure-eventhub input" + vars: + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: storage_account_container + type: text + title: Storage Account Container + multi: false + required: false + show_user: false + description: >- + The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-aadgraphactivitylogs + - forwarded + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: sanitize_newlines + type: bool + title: Sanitizes New Lines + description: Removes new lines in logs to ensure proper formatting of JSON data and avoid parsing issues during processing. + multi: false + required: false + show_user: false + default: false + - name: sanitize_singlequotes + required: true + show_user: false + title: Sanitizes Single Quotes + description: Replaces single quotes with double quotes (single quotes inside double quotes are omitted) in logs to ensure proper formatting of JSON data and avoid parsing issues during processing. + type: bool + multi: false + default: false + # + # Processor v2 only settings + # + - name: processor_version + type: select + title: Processor version + multi: false + required: false + show_user: false + default: v2 + options: + - text: v1 (legacy) + value: v1 + - text: v2 + value: v2 + description: > + The processor version that the integration should use. Possible values are v1 (legacy) and v2 (recommended). + The v2 event hub processor is recommended for typical use cases. Defaults to v2. + - name: processor_update_interval + type: text + title: Processor update interval + multi: false + required: false + show_user: false + default: 10s + description: >- + (Processor v2 only) How often the processor should attempt to claim partitions. + + Default is `10` seconds. + - name: processor_start_position + type: select + title: Processor start position + multi: false + required: false + show_user: false + default: earliest + options: + - text: earliest + value: earliest + - text: latest + value: latest + description: >- + (Processor v2 only) Controls from what position in the event hub the processor should start processing messages for all partitions. + + Possible values are `earliest` and `latest`. + + `earliest` starts processing messages from the last checkpoint, or the beginning of the event hub if no checkpoint is available. + + `latest` starts processing messages from the the latest event in the event hub and continues to process new events as they arrive. + + Default is `earliest`. + - name: partition_receive_timeout + type: text + title: Partition receive timeout + multi: false + required: false + show_user: false + default: 5s + description: >- + (Processor v2 only) Maximum time to wait before processing the messages received from the event hub. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `5` seconds. + - name: partition_receive_count + type: text + title: Partition receive count + multi: false + required: false + show_user: false + default: 100 + description: >- + (Processor v2 only) Maximum number of messages from the event hub to wait for before processing them. + + The partition consumer waits up to a "receive count" or a "receive timeout", whichever comes first. + + Default is `100` messages. + - name: migrate_checkpoint + type: bool + title: Migrate checkpoint information + multi: false + required: false + show_user: false + default: true + description: >- + (Processor v2 only) Controls processor behavior upon the initial transition from v1 to v2. If you are + starting directly with v2, you can disregard this setting. + + Defaults to `true`, which means the processor will automatically migrate checkpoint information from the + v1 format to v2. + + If set to `false`, the processor will bypass existing v1 checkpoints and replay all events in the + Event Hub from the start of the retention window. For example, if set to `false` on an Event Hub with + a 1-hour retention period, the processor will replay the last hour's worth of data. + - name: endpoint_suffix + type: text + default: core.windows.net + required: true + title: Storage account endpoint suffix + show_user: false + description: >- + (Processor v2 only) Override the default storage account endpoint suffix. + +# Ensures agents have permissions to write data to `logs-*-*` +elasticsearch: + dynamic_dataset: true + dynamic_namespace: true diff --git a/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json b/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json new file mode 100644 index 00000000000..ea530b79392 --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json @@ -0,0 +1,138 @@ +{ + "@timestamp": "2026-05-07T15:19:33.536Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000001-0001-0001-0001-000000000001", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", + "time_generated": "2026-05-07T15:19:33.536Z", + "token_issued_at": "2026-05-07T13:50:39.000Z" + }, + "result_signature": "200" + }, + "correlation_id": "00000001-0001-0001-0001-000000000001", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "users-read", + "category": [ + "iam", + "web" + ], + "duration": 59000000, + "id": "00000001-0001-0001-0001-000000000001", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "outcome": "success", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000001-0001-0001-0001-000000000001", + "method": "GET" + }, + "response": { + "bytes": 54662, + "status_code": 200 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-client/1.0" + } +} \ No newline at end of file diff --git a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 44debced8b2..1122b2c196d 100644 --- a/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -104,6 +104,10 @@ processors: field: event.dataset value: azure.graphactivitylogs if: 'ctx.routing?.category == "MicrosoftGraphActivityLogs"' + - set: + field: event.dataset + value: azure.aadgraphactivitylogs + if: 'ctx.routing?.category == "AzureADGraphActivityLogs"' - set: field: event.dataset value: azure.identity_protection diff --git a/packages/azure/data_stream/events/routing_rules.yml b/packages/azure/data_stream/events/routing_rules.yml index e204a0a5aad..d0e06f60040 100644 --- a/packages/azure/data_stream/events/routing_rules.yml +++ b/packages/azure/data_stream/events/routing_rules.yml @@ -30,6 +30,11 @@ namespace: - "{{data_stream.namespace}}" - default + - target_dataset: azure.aadgraphactivitylogs + if: ctx.event?.dataset == 'azure.aadgraphactivitylogs' + namespace: + - "{{data_stream.namespace}}" + - default - target_dataset: azure.identity_protection if: ctx.event?.dataset == 'azure.identity_protection' namespace: diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index cceab32a0f7..de01f4d4e00 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.36.1" +version: "1.37.0-preview1" description: This Elastic integration collects logs from Azure type: integration icons: From 2c8805279c7879d86e6c68771c45e4e6a5418080 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Thu, 7 May 2026 14:32:52 -0400 Subject: [PATCH 02/19] Update changelog for azure.aadgraphactivitylogs dataset --- packages/azure/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index e0c885c1ad0..398c88ff87a 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,8 +1,8 @@ - version: "1.37.0-preview1" changes: - - description: Add azure.aadgraphactivitylogs dataset to ingest legacy Azure AD Graph (graph.windows.net) AzureADGraphActivityLogs diagnostic events. Local TRaDE research build, not for upstream. + - description: Add azure.aadgraphactivitylogs dataset to ingest legacy Azure AD Graph (graph.windows.net) AzureADGraphActivityLogs diagnostic events. type: enhancement - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/18880 - version: "1.36.1" changes: - description: Add missing event.kind pipeline_error handling to ingest pipeline on_failure handlers. From 7f60d1d402b7377aab7fff8a8b379cbfd37a3ea9 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Thu, 7 May 2026 14:33:45 -0400 Subject: [PATCH 03/19] Apply suggestion from @terrancedejesus --- packages/azure/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 398c88ff87a..ad2c5b53ff4 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,4 +1,4 @@ -- version: "1.37.0-preview1" +- version: "1.37.0" changes: - description: Add azure.aadgraphactivitylogs dataset to ingest legacy Azure AD Graph (graph.windows.net) AzureADGraphActivityLogs diagnostic events. type: enhancement From c1a75c025daa9912ebc071a8a915a2e4abe804bb Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Thu, 7 May 2026 14:33:51 -0400 Subject: [PATCH 04/19] Apply suggestion from @terrancedejesus --- packages/azure/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index de01f4d4e00..b7a2c4fe61f 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.37.0-preview1" +version: "1.37.0" description: This Elastic integration collects logs from Azure type: integration icons: From 82dd31b23f839b65b6f8b24fa6bf6ebb332f7fab Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Fri, 8 May 2026 00:18:12 -0400 Subject: [PATCH 05/19] azure/aadgraphactivitylogs: update codeowners --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 04ae437ee86..7baa529e143 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -100,6 +100,7 @@ /packages/awsfargate @elastic/obs-infraobs-integrations /packages/awsfirehose @elastic/obs-ds-hosted-services /packages/azure @elastic/obs-infraobs-integrations @elastic/obs-ds-hosted-services @elastic/security-service-integrations +/packages/azure/data_stream/aadgraphactivitlogs @elastic/security-service-integrations /packages/azure/data_stream/activitylogs @elastic/obs-infraobs-integrations /packages/azure/data_stream/application_gateway @elastic/security-service-integrations /packages/azure/data_stream/auditlogs @elastic/obs-infraobs-integrations From 357c09f77ac1a8f4534f2996b471803d6c7778a2 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Fri, 8 May 2026 00:26:06 -0400 Subject: [PATCH 06/19] azure/aadgraphactivitylogs: update codeowners --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7baa529e143..843bd17b8b2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -100,7 +100,7 @@ /packages/awsfargate @elastic/obs-infraobs-integrations /packages/awsfirehose @elastic/obs-ds-hosted-services /packages/azure @elastic/obs-infraobs-integrations @elastic/obs-ds-hosted-services @elastic/security-service-integrations -/packages/azure/data_stream/aadgraphactivitlogs @elastic/security-service-integrations +/packages/azure/data_stream/aadgraphactivitylogs @elastic/security-service-integrations /packages/azure/data_stream/activitylogs @elastic/obs-infraobs-integrations /packages/azure/data_stream/application_gateway @elastic/security-service-integrations /packages/azure/data_stream/auditlogs @elastic/obs-infraobs-integrations From 30046ce36184e5916a2b5f69455b6c5b0fe976f6 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 10:02:49 -0400 Subject: [PATCH 07/19] azure/aadgraphactivitylogs: remove log YAML HBS file --- .../agent/stream/log.yml.hbs | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs diff --git a/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs deleted file mode 100644 index 89d35fecd80..00000000000 --- a/packages/azure/data_stream/aadgraphactivitylogs/agent/stream/log.yml.hbs +++ /dev/null @@ -1,20 +0,0 @@ -paths: - {{#each paths as |path|}} -- {{path}} - {{/each}} -exclude_files: [".gz$"] -tags: -{{#if preserve_original_event}} - - preserve_original_event -{{/if}} -{{#each tags as |tag|}} - - {{tag}} -{{/each}} -{{#contains "forwarded" tags}} -publisher_pipeline.disable_host: true -{{/contains}} -allow_deprecated_use: true -{{#if processors}} -processors: -{{processors}} -{{/if}} From 0a4f6973f89ad1c070d269aeab138e7be02bd14b Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:17:25 -0400 Subject: [PATCH 08/19] azure/aadgraphactivitylogs: adding docs and updated README --- packages/azure/_dev/build/docs/README.md | 4 +- .../_dev/build/docs/aadgraphactivitylogs.md | 89 ++++++ packages/azure/docs/README.md | 4 +- packages/azure/docs/aadgraphactivitylogs.md | 281 ++++++++++++++++++ 4 files changed, 374 insertions(+), 4 deletions(-) create mode 100644 packages/azure/_dev/build/docs/aadgraphactivitylogs.md create mode 100644 packages/azure/docs/aadgraphactivitylogs.md diff --git a/packages/azure/_dev/build/docs/README.md b/packages/azure/_dev/build/docs/README.md index 154475d1c24..fd1421df752 100644 --- a/packages/azure/_dev/build/docs/README.md +++ b/packages/azure/_dev/build/docs/README.md @@ -1,6 +1,6 @@ # Azure Logs Integration -The Azure Logs integration collects logs for specific Azure services like Microsoft Entra ID (Sign-in, Audit, Identity Protection, and Provisioning logs), Azure Spring Apps, Azure Firewall, Microsoft Graph Activity, and several others using the Activity and Platform logs. +The Azure Logs integration collects logs for specific Azure services like Microsoft Entra ID (Sign-in, Audit, Identity Protection, and Provisioning logs), Azure Spring Apps, Azure Firewall, Microsoft Graph Activity, Azure AD Graph Activity, and several others using the Activity and Platform logs. You can then visualize that data in Kibana, create alerts if something goes wrong, and reference data when troubleshooting an issue. @@ -16,7 +16,7 @@ fail to start due to an exceed quota limit. The Azure Logs integration collects logs. **Logs** help you keep a record of events that happen on your Azure account. -Log data streams collected by the Azure Logs integration include Activity, Platform, Microsoft Entra ID (Sign-in, Audit, Identity Protection, Provisioning), Microsoft Graph Activity, and Spring Apps logs. +Log data streams collected by the Azure Logs integration include Activity, Platform, Microsoft Entra ID (Sign-in, Audit, Identity Protection, Provisioning), Microsoft Graph Activity, Azure AD Graph Activity, and Spring Apps logs. ## Requirements diff --git a/packages/azure/_dev/build/docs/aadgraphactivitylogs.md b/packages/azure/_dev/build/docs/aadgraphactivitylogs.md new file mode 100644 index 00000000000..83db4c02976 --- /dev/null +++ b/packages/azure/_dev/build/docs/aadgraphactivitylogs.md @@ -0,0 +1,89 @@ +# Azure AD Graph Activity Logs + +Azure AD Graph Activity Logs provide an audit trail of all HTTP requests that the legacy Azure AD Graph service (`graph.windows.net`) has received and processed for a tenant. Although Microsoft has deprecated Azure AD Graph in favor of Microsoft Graph, the API is still actively used by Microsoft first-party tooling, older line-of-business applications, third-party SaaS connectors, and adversary tooling (for example ROADtools, AzureHound v1, AADInternals). Refer to the [AADGraphActivityLogs table reference](https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/aadgraphactivitylogs) for the canonical schema. + +Tenant administrators can configure the collection and storage destinations of Azure AD Graph Activity Logs through Diagnostic Setting in the Entra Portal. This integration uses Azure Event Hubs destination to stream Azure AD Graph Activity Logs to Elastic. + +## Requirements and Setup + +### What do I need to use this integration? + +The following privileges are required to collect Azure AD Graph Activity Logs: +- A Microsoft Entra ID P1 or P2 tenant license in your tenant. +- A `Security Administrator` or `Global Administrator` Microsoft Entra ID role to configure the diagnostic settings. + +### Setup + +Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for more information about setting up and using this integration. + +### Limitations + +- Activities of multi-tenant applications belonging to another tenant are not available. +- In rare cases, events might take up to 2 hours to be delivered to Event Hubs. +- Azure AD Graph is deprecated by Microsoft. New workloads should target Microsoft Graph; this dataset is intended for visibility into legacy traffic that still exists in the tenant. + +## Settings + +`eventhub` : + _string_ +It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name; in this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (e.g., the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. +Default value `insights-operational-logs`. + +`consumer_group` : +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` : +_string_ +The connection string required to communicate with Event Hubs, steps [here](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). + +A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. + +`storage_account` : +_string_ +The name of the storage account the state/offsets will be stored and updated. + +`storage_account_key` : +_string_ +The storage account key, this key will be used to authorize access to data in your storage account. + +`storage_account_container` : +_string_ +The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified. + +`resource_manager_endpoint` : +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different Azure environment. + +Resource manager endpoints: + +```text +# Azure ChinaCloud +https://management.chinacloudapi.cn/ + +# Azure GermanCloud +https://management.microsoftazure.de/ + +# Azure PublicCloud +https://management.azure.com/ + +# Azure USGovernmentCloud +https://management.usgovcloudapi.net/ +``` + +## Logs + +### aadgraphactivitylogs + +The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`; the events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. + +Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled; only new events are routed to the dedicated dataset. + +{{event "aadgraphactivitylogs"}} + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +{{fields "aadgraphactivitylogs"}} diff --git a/packages/azure/docs/README.md b/packages/azure/docs/README.md index 154475d1c24..fd1421df752 100644 --- a/packages/azure/docs/README.md +++ b/packages/azure/docs/README.md @@ -1,6 +1,6 @@ # Azure Logs Integration -The Azure Logs integration collects logs for specific Azure services like Microsoft Entra ID (Sign-in, Audit, Identity Protection, and Provisioning logs), Azure Spring Apps, Azure Firewall, Microsoft Graph Activity, and several others using the Activity and Platform logs. +The Azure Logs integration collects logs for specific Azure services like Microsoft Entra ID (Sign-in, Audit, Identity Protection, and Provisioning logs), Azure Spring Apps, Azure Firewall, Microsoft Graph Activity, Azure AD Graph Activity, and several others using the Activity and Platform logs. You can then visualize that data in Kibana, create alerts if something goes wrong, and reference data when troubleshooting an issue. @@ -16,7 +16,7 @@ fail to start due to an exceed quota limit. The Azure Logs integration collects logs. **Logs** help you keep a record of events that happen on your Azure account. -Log data streams collected by the Azure Logs integration include Activity, Platform, Microsoft Entra ID (Sign-in, Audit, Identity Protection, Provisioning), Microsoft Graph Activity, and Spring Apps logs. +Log data streams collected by the Azure Logs integration include Activity, Platform, Microsoft Entra ID (Sign-in, Audit, Identity Protection, Provisioning), Microsoft Graph Activity, Azure AD Graph Activity, and Spring Apps logs. ## Requirements diff --git a/packages/azure/docs/aadgraphactivitylogs.md b/packages/azure/docs/aadgraphactivitylogs.md new file mode 100644 index 00000000000..702e847414d --- /dev/null +++ b/packages/azure/docs/aadgraphactivitylogs.md @@ -0,0 +1,281 @@ +# Azure AD Graph Activity Logs + +Azure AD Graph Activity Logs provide an audit trail of all HTTP requests that the legacy Azure AD Graph service (`graph.windows.net`) has received and processed for a tenant. Although Microsoft has deprecated Azure AD Graph in favor of Microsoft Graph, the API is still actively used by Microsoft first-party tooling, older line-of-business applications, third-party SaaS connectors, and adversary tooling (for example ROADtools, AzureHound v1, AADInternals). Refer to the [AADGraphActivityLogs table reference](https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/aadgraphactivitylogs) for the canonical schema. + +Tenant administrators can configure the collection and storage destinations of Azure AD Graph Activity Logs through Diagnostic Setting in the Entra Portal. This integration uses Azure Event Hubs destination to stream Azure AD Graph Activity Logs to Elastic. + +## Requirements and Setup + +### What do I need to use this integration? + +The following privileges are required to collect Azure AD Graph Activity Logs: +- A Microsoft Entra ID P1 or P2 tenant license in your tenant. +- A `Security Administrator` or `Global Administrator` Microsoft Entra ID role to configure the diagnostic settings. + +### Setup + +Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for more information about setting up and using this integration. + +### Limitations + +- Activities of multi-tenant applications belonging to another tenant are not available. +- In rare cases, events might take up to 2 hours to be delivered to Event Hubs. +- Azure AD Graph is deprecated by Microsoft. New workloads should target Microsoft Graph; this dataset is intended for visibility into legacy traffic that still exists in the tenant. + +## Settings + +`eventhub` : + _string_ +It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name; in this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (e.g., the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. +Default value `insights-operational-logs`. + +`consumer_group` : +_string_ + The publish/subscribe mechanism of Event Hubs is enabled through consumer groups. A consumer group is a view (state, position, or offset) of an entire event hub. Consumer groups enable multiple consuming applications to each have a separate view of the event stream, and to read the stream independently at their own pace and with their own offsets. +Default value: `$Default` + +`connection_string` : +_string_ +The connection string required to communicate with Event Hubs, steps [here](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). + +A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. + +`storage_account` : +_string_ +The name of the storage account the state/offsets will be stored and updated. + +`storage_account_key` : +_string_ +The storage account key, this key will be used to authorize access to data in your storage account. + +`storage_account_container` : +_string_ +The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified. + +`resource_manager_endpoint` : +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different Azure environment. + +Resource manager endpoints: + +```text +# Azure ChinaCloud +https://management.chinacloudapi.cn/ + +# Azure GermanCloud +https://management.microsoftazure.de/ + +# Azure PublicCloud +https://management.azure.com/ + +# Azure USGovernmentCloud +https://management.usgovcloudapi.net/ +``` + +## Logs + +### aadgraphactivitylogs + +The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`; the events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. + +Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled; only new events are routed to the dedicated dataset. + +An example event for `aadgraphactivitylogs` looks as following: + +```json +{ + "@timestamp": "2026-05-07T15:19:33.536Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "operation_version": "1.6", + "properties": { + "actor_type": "User", + "api_version": "1.6", + "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "client_auth_method": 0, + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "operation_id": "00000001-0001-0001-0001-000000000001", + "scopes": [ + "62e90394-69f5-4237-9190-012177145e10" + ], + "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", + "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", + "time_generated": "2026-05-07T15:19:33.536Z", + "token_issued_at": "2026-05-07T13:50:39.000Z" + }, + "result_signature": "200" + }, + "correlation_id": "00000001-0001-0001-0001-000000000001", + "resource": { + "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", + "provider": "MICROSOFT.AADIAM" + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "client": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143", + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "service": { + "name": "Azure AD Graph" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "users-read", + "category": [ + "iam", + "web" + ], + "duration": 59000000, + "id": "00000001-0001-0001-0001-000000000001", + "kind": "event", + "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "outcome": "success", + "type": [ + "access", + "info" + ] + }, + "http": { + "request": { + "id": "00000001-0001-0001-0001-000000000001", + "method": "GET" + }, + "response": { + "bytes": 54662, + "status_code": 200 + } + }, + "log": { + "level": "4" + }, + "related": { + "ip": [ + "81.2.69.143" + ], + "user": [ + "b37ec517-0a34-4266-b627-f7bb0d679d70", + "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + ] + }, + "session": { + "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + }, + "source": { + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.143" + }, + "tags": [ + "preserve_original_event" + ], + "token": { + "id": "AAAAAAAAAAAAAAAAAAAAAA==" + }, + "url": { + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" + }, + "user": { + "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "azure-graph-test-client/1.0" + } +} +``` + +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| azure.aadgraphactivitylogs.category | Azure Event Category. For Azure AD Graph Activity Logs, this is `AzureADGraphActivityLogs`. | keyword | +| azure.aadgraphactivitylogs.operation_name | Operation name. For this category the value is always the literal string `AAD Graph Activity`; rely on the derived `event.action` (HTTP method + directory collection from `requestUri`) for detection. | keyword | +| azure.aadgraphactivitylogs.properties.actor_type | Type of identity that issued the request, e.g. `User`, `ServicePrincipal`. | keyword | +| azure.aadgraphactivitylogs.properties.api_version | The API version of the event. | keyword | +| azure.aadgraphactivitylogs.properties.app_id | The identifier for the application. | keyword | +| azure.aadgraphactivitylogs.properties.billed_size | The record size in bytes. | double | +| azure.aadgraphactivitylogs.properties.client_auth_method | Indicates how the client was authenticated. For a public client, the value is 0. If client ID and client secret are used, the value is 1. If a client certificate was used for authentication, the value is 2. | integer | +| azure.aadgraphactivitylogs.properties.device_id | The identifier of the device from which the authentication request originated. | keyword | +| azure.aadgraphactivitylogs.properties.direct_access_source | The path through which the request reached the AAD Graph service (e.g. `Gateway`). | keyword | +| azure.aadgraphactivitylogs.properties.env_cloud_role | The Microsoft cloud role identifier for the service handling the request (e.g. `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. | keyword | +| azure.aadgraphactivitylogs.properties.identity_provider | The identity provider that authenticated the subject of the token. | keyword | +| azure.aadgraphactivitylogs.properties.is_billable | Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account. | boolean | +| azure.aadgraphactivitylogs.properties.request_uri | The URI of the request. | keyword | +| azure.aadgraphactivitylogs.properties.roles | The roles in token claims. | keyword | +| azure.aadgraphactivitylogs.properties.scopes | The scopes in token claims. | keyword | +| azure.aadgraphactivitylogs.properties.service_principal_id | The identifier of the servicePrincipal making the request. | keyword | +| azure.aadgraphactivitylogs.properties.session_id | The unique identifier for the authentication session. | keyword | +| azure.aadgraphactivitylogs.properties.sign_in_activity_id | Identifier of the Microsoft Entra ID sign-in event that established the authentication context for this AAD Graph request. Correlates this request with an entry in Entra ID Sign-In Logs. | keyword | +| azure.aadgraphactivitylogs.properties.source_system | The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics. | keyword | +| azure.aadgraphactivitylogs.properties.time_generated | The date and time the request was received. | date | +| azure.aadgraphactivitylogs.properties.token_issued_at | The timestamp the token was issued at. | date | +| azure.aadgraphactivitylogs.properties.type | The name of the table. | keyword | +| azure.aadgraphactivitylogs.properties.user_agent | The user agent information related to request. | keyword | +| azure.aadgraphactivitylogs.properties.wids | Denotes the tenant-wide roles assigned to this user. | keyword | +| azure.correlation_id | Correlation ID. | keyword | +| azure.resource.authorization_rule | Authorization rule. | keyword | +| azure.resource.group | Resource group. | keyword | +| azure.resource.id | Resource ID. | keyword | +| azure.resource.name | Name. | keyword | +| azure.resource.namespace | Resource type/namespace. | keyword | +| azure.resource.provider | Resource type/namespace. | keyword | +| azure.subscription_id | Azure subscription ID. | keyword | +| azure.tenant_id | tenant ID. | keyword | +| client.geo.location | Longitude and latitude. | geo_point | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset name. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.geo.region_name | Region name. | keyword | +| event.dataset | Event dataset | constant_keyword | +| event.module | Event module | constant_keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| session.id | The unique identifier for the authentication session. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | + From 20ba136297e2bc2d0e950cfdb17032c56efcc328 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:18:22 -0400 Subject: [PATCH 09/19] azure/aadgraphactivitylogs: adding separate manifest for aadgraphactivitylogs --- packages/azure/manifest.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index b7a2c4fe61f..4d36c4ee9d2 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -292,6 +292,29 @@ policy_templates: title: microsoft graph activity overview size: 5002x2666 type: image/png + - name: aadgraphactivitylogs + title: Azure AD Graph Activity Logs + description: Azure AD Graph Activity Logs integration + fips_compatible: false + data_streams: + - aadgraphactivitylogs + categories: + - security + inputs: + - type: "azure-eventhub" + title: "Collect Azure AD Graph Activity Logs (v1)" + description: "Collecting Azure AD Graph activity logs from Azure instances (input: azure-eventhub)" + input_group: logs + icons: + - src: /img/microsoft-entra-id-logo.svg + title: Microsoft Entra ID logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/filebeat-azure-overview.png + title: filebeat azure overview + size: 5002x2666 + type: image/png - name: springcloudlogs title: Azure Spring Apps logs description: Azure Spring Apps logs integration From 8e1dc867400bfc1b8fac53dee0560715e0a538df Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:20:14 -0400 Subject: [PATCH 10/19] azure/aadgraphactivitylogs: adjust data stream test pipeline events and field schemas --- .../test/pipeline/test-aadgraph-activity.log | 9 +- .../test-aadgraph-activity.log-expected.json | 345 ++++++++++++------ .../elasticsearch/ingest_pipeline/default.yml | 171 ++++----- .../fields/ecs-extended.yml | 6 - .../aadgraphactivitylogs/fields/fields.yml | 28 +- 5 files changed, 317 insertions(+), 242 deletions(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log index 96dc0d56a20..95921738b7e 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log @@ -1,4 +1,5 @@ -{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000001-0001-0001-0001-000000000001","durationMs":59,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":59,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000001-0001-0001-0001-000000000001","requestId":"00000001-0001-0001-0001-000000000001","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users","responseSizeBytes":54662,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:19:33.5368860Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:19:33.5368860Z"} -{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000002-0002-0002-0002-000000000002","durationMs":18,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":18,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000002-0002-0002-0002-000000000002","requestId":"00000002-0002-0002-0002-000000000002","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles","responseSizeBytes":258,"responseStatusCode":400,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:15:45.0849115Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"400","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:15:45.0849115Z"} -{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000003-0003-0003-0003-000000000003","durationMs":57,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":57,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000003-0003-0003-0003-000000000003","requestId":"00000003-0003-0003-0003-000000000003","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails","responseSizeBytes":13324,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"BBBBBBBBBBBBBBBBBBBBBB==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T16:12:53.7641499Z","tokenIssuedAt":"2026-05-07T16:05:34Z","userAgent":"azure-graph-test-probe/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T16:12:53.7641499Z"} -{"Level":4,"callerIpAddress":"81.2.69.143","category":"AzureADGraphActivityLogs","correlationId":"00000004-0004-0004-0004-000000000004","durationMs":71,"operationName":"AAD Graph Activity","operationVersion":"1.6","properties":{"actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","clientAuthMethod":"0","durationMs":71,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","ipAddress":"81.2.69.143","operationId":"00000004-0004-0004-0004-000000000004","requestId":"00000004-0004-0004-0004-000000000004","requestMethod":"GET","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications","responseSizeBytes":10026,"responseStatusCode":200,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:15:44.4909654Z","tokenIssuedAt":"2026-05-07T13:50:39Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"resourceId":"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM","resultSignature":"200","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","time":"2026-05-07T15:15:44.4909654Z"} +{"category":"AzureADGraphActivityLogs","location":"WestUS","operationName":"AAD Graph Activity","properties":{"__UDI_RequiredFields_EventTime":639140000000000000,"__UDI_RequiredFields_RegionScope":"NA","__UDI_RequiredFields_TenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","__UDI_RequiredFields_UniqueId":"00000001-0001-0001-0001-000000000001","actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","callerIpAddress":"81.2.69.143","clientAuthMethod":0,"deviceId":"","directAccessSource":"Gateway","durationMs":59,"env_cloud_role":"restdirectoryservice","httpMethod":"GET","httpStatusCode":200,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","issuedAt":"5/7/2026 1:50:39 PM","location":"WestUS","requestId":"00000001-0001-0001-0001-000000000001","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users","responseSizeBytes":54662,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:19:33.5368860Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb"} +{"category":"AzureADGraphActivityLogs","location":"WestUS","operationName":"AAD Graph Activity","properties":{"__UDI_RequiredFields_EventTime":639140000000000001,"__UDI_RequiredFields_RegionScope":"NA","__UDI_RequiredFields_TenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","__UDI_RequiredFields_UniqueId":"00000002-0002-0002-0002-000000000002","actorType":"User","apiVersion":"1.61-internal","appId":"1b730954-1685-4b74-9bfd-dac224a7b894","callerIpAddress":"81.2.69.143","clientAuthMethod":0,"deviceId":"","directAccessSource":"Gateway","durationMs":25,"env_cloud_role":"restdirectoryservice","httpMethod":"GET","httpStatusCode":403,"identityProvider":"","issuedAt":"5/8/2026 4:11:19 PM","location":"WestUS","requestId":"00000002-0002-0002-0002-000000000002","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails","responseSizeBytes":-1,"roles":"","scopes":"user_impersonation","servicePrincipalId":"","sessionId":"004d8baa-891d-139e-633f-6c194e566b78","signInActivityId":"BCF50s3Ik0q7Ke-mIBAGAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-08T16:37:21.2823997Z","userAgent":"AADInternals","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb"} +{"category":"AzureADGraphActivityLogs","location":"WestUS","operationName":"AAD Graph Activity","properties":{"__UDI_RequiredFields_EventTime":639141000000000000,"__UDI_RequiredFields_RegionScope":"NA","__UDI_RequiredFields_TenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","__UDI_RequiredFields_UniqueId":"00000003-0003-0003-0003-000000000003","actorType":"Application","apiVersion":"AGSbeta-internal","appId":"65d91a3d-ab74-42e6-8a2f-0add61688c74","callerIpAddress":"81.2.69.144","clientAuthMethod":0,"deviceId":"","directAccessSource":"Gateway","durationMs":44,"env_cloud_role":"restdirectoryservice","httpMethod":"GET","httpStatusCode":200,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","issuedAt":"5/11/2026 12:35:58 PM","location":"WestUS","requestId":"00000003-0003-0003-0003-000000000003","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/groups","responseSizeBytes":695,"roles":" Group.Read.All,","scopes":"","servicePrincipalId":"fa5cc200-f464-4068-88b9-c6b8335c8c70","sessionId":"","signInActivityId":"Rr_bbVOVu0WHOTzl0BhgAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-11T12:41:10.9430757Z","userAgent":"","userId":"","wids":" 0997a1d0-0d1d-4acb-b408-d5ca73121e90,"},"tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb"} +{"category":"AzureADGraphActivityLogs","location":"SouthCentralUS","operationName":"AAD Graph Activity","properties":{"__UDI_RequiredFields_EventTime":639141000000000001,"__UDI_RequiredFields_RegionScope":"NA","__UDI_RequiredFields_TenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","__UDI_RequiredFields_UniqueId":"00000004-0004-0004-0004-000000000004","actorType":"Application","apiVersion":"1.61-internal","appId":"66244124-575c-4284-92bc-fdd00e669cea","callerIpAddress":"89.160.20.156","clientAuthMethod":0,"deviceId":"","directAccessSource":"Gateway","durationMs":152,"env_cloud_role":"restdirectoryservice","httpMethod":"POST","httpStatusCode":202,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","issuedAt":"5/9/2026 12:28:05 PM","location":"SouthCentralUS","requestId":"00000004-0004-0004-0004-000000000004","requestUri":"/ab30785b-417f-42a4-b5dc-8f9051718acb/$batch","responseSizeBytes":9580,"roles":"","scopes":"","servicePrincipalId":"4ed697af-4af9-43f7-8bf2-7b2172300068","sessionId":"","signInActivityId":"-TZsVeDWj0e-jamRtuUnAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-09T12:33:05.9908603Z","userAgent":"","userId":"","wids":""},"tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb"} +{"category":"AzureADGraphActivityLogs","location":"WestUS","operationName":"AAD Graph Activity","properties":{"__UDI_RequiredFields_EventTime":639142000000000000,"__UDI_RequiredFields_RegionScope":"NA","__UDI_RequiredFields_TenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","__UDI_RequiredFields_UniqueId":"00000005-0005-0005-0005-000000000005","actorType":"User","apiVersion":"1.6","appId":"04b07795-8ddb-461a-bbee-02f9e1bf7b46","callerIpAddress":"81.2.69.143","clientAuthMethod":0,"deviceId":"","directAccessSource":"Gateway","durationMs":71,"env_cloud_role":"restdirectoryservice","httpMethod":"PATCH","httpStatusCode":204,"identityProvider":"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/","issuedAt":"5/7/2026 1:50:39 PM","location":"WestUS","requestId":"00000005-0005-0005-0005-000000000005","requestUri":"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications/fda2d70f-7df3-4556-8bcb-30acc0a45082","responseSizeBytes":0,"roles":"","scopes":"62e90394-69f5-4237-9190-012177145e10","servicePrincipalId":"","sessionId":"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a","signInActivityId":"AAAAAAAAAAAAAAAAAAAAAA==","tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb","timeGenerated":"2026-05-07T15:15:44.4909654Z","userAgent":"azure-graph-test-client/1.0","userId":"b37ec517-0a34-4266-b627-f7bb0d679d70","wids":""},"tenantId":"ab30785b-417f-42a4-b5dc-8f9051718acb"} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json index 46231ac5589..dbb1b8aed21 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-aadgraph-activity.log-expected.json @@ -6,14 +6,14 @@ "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { "actor_type": "User", "api_version": "1.6", "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000001-0001-0001-0001-000000000001", "scopes": [ "62e90394-69f5-4237-9190-012177145e10" ], @@ -21,13 +21,7 @@ "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", "time_generated": "2026-05-07T15:19:33.536Z", "token_issued_at": "2026-05-07T13:50:39.000Z" - }, - "result_signature": "200" - }, - "correlation_id": "00000001-0001-0001-0001-000000000001", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -54,10 +48,16 @@ "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, @@ -70,7 +70,7 @@ "duration": 59000000, "id": "00000001-0001-0001-0001-000000000001", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639140000000000000,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000001-0001-0001-0001-000000000001\",\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"callerIpAddress\":\"81.2.69.143\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":59,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"GET\",\"httpStatusCode\":200,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/7/2026 1:50:39 PM\",\"location\":\"WestUS\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "success", "type": [ "access", @@ -87,9 +87,6 @@ "status_code": 200 } }, - "log": { - "level": "4" - }, "related": { "ip": [ "81.2.69.143" @@ -120,9 +117,6 @@ "tags": [ "preserve_original_event" ], - "token": { - "id": "AAAAAAAAAAAAAAAAAAAAAA==" - }, "url": { "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" @@ -139,33 +133,26 @@ } }, { - "@timestamp": "2026-05-07T15:15:45.084Z", + "@timestamp": "2026-05-08T16:37:21.282Z", "azure": { "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { "actor_type": "User", - "api_version": "1.6", - "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "api_version": "1.61-internal", + "app_id": "1b730954-1685-4b74-9bfd-dac224a7b894", "client_auth_method": 0, - "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000002-0002-0002-0002-000000000002", + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "scopes": [ - "62e90394-69f5-4237-9190-012177145e10" + "user_impersonation" ], - "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", - "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", - "time_generated": "2026-05-07T15:15:45.084Z", - "token_issued_at": "2026-05-07T13:50:39.000Z" - }, - "result_signature": "400" - }, - "correlation_id": "00000002-0002-0002-0002-000000000002", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + "session_id": "004d8baa-891d-139e-633f-6c194e566b78", + "sign_in_activity_id": "BCF50s3Ik0q7Ke-mIBAGAA==", + "time_generated": "2026-05-08T16:37:21.282Z", + "token_issued_at": "2026-05-08T16:11:19.000Z" + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -192,23 +179,29 @@ "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, "event": { - "action": "directory-roles-read", + "action": "tenant-details-read", "category": [ "iam", "web" ], - "duration": 18000000, + "duration": 25000000, "id": "00000002-0002-0002-0002-000000000002", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000002-0002-0002-0002-000000000002\",\"durationMs\":18,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":18,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000002-0002-0002-0002-000000000002\",\"requestId\":\"00000002-0002-0002-0002-000000000002\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles\",\"responseSizeBytes\":258,\"responseStatusCode\":400,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:15:45.0849115Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"400\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:15:45.0849115Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639140000000000001,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000002-0002-0002-0002-000000000002\",\"actorType\":\"User\",\"apiVersion\":\"1.61-internal\",\"appId\":\"1b730954-1685-4b74-9bfd-dac224a7b894\",\"callerIpAddress\":\"81.2.69.143\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":25,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"GET\",\"httpStatusCode\":403,\"identityProvider\":\"\",\"issuedAt\":\"5/8/2026 4:11:19 PM\",\"location\":\"WestUS\",\"requestId\":\"00000002-0002-0002-0002-000000000002\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails\",\"responseSizeBytes\":-1,\"roles\":\"\",\"scopes\":\"user_impersonation\",\"servicePrincipalId\":\"\",\"sessionId\":\"004d8baa-891d-139e-633f-6c194e566b78\",\"signInActivityId\":\"BCF50s3Ik0q7Ke-mIBAGAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-08T16:37:21.2823997Z\",\"userAgent\":\"AADInternals\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "failure", "type": [ "access", @@ -221,24 +214,21 @@ "method": "GET" }, "response": { - "bytes": 258, - "status_code": 400 + "bytes": -1, + "status_code": 403 } }, - "log": { - "level": "4" - }, "related": { "ip": [ "81.2.69.143" ], "user": [ "b37ec517-0a34-4266-b627-f7bb0d679d70", - "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + "1b730954-1685-4b74-9bfd-dac224a7b894" ] }, "session": { - "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + "id": "004d8baa-891d-139e-633f-6c194e566b78" }, "source": { "geo": { @@ -258,12 +248,9 @@ "tags": [ "preserve_original_event" ], - "token": { - "id": "AAAAAAAAAAAAAAAAAAAAAA==" - }, "url": { - "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles", - "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/directoryRoles" + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails" }, "user": { "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" @@ -273,37 +260,34 @@ "name": "Other" }, "name": "Other", - "original": "azure-graph-test-client/1.0" + "original": "AADInternals" } }, { - "@timestamp": "2026-05-07T16:12:53.764Z", + "@timestamp": "2026-05-11T12:41:10.943Z", "azure": { "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { - "actor_type": "User", - "api_version": "1.6", - "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", + "actor_type": "Application", + "api_version": "AGSbeta-internal", + "app_id": "65d91a3d-ab74-42e6-8a2f-0add61688c74", "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000003-0003-0003-0003-000000000003", - "scopes": [ - "62e90394-69f5-4237-9190-012177145e10" + "roles": [ + "Group.Read.All" ], - "session_id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a", - "sign_in_activity_id": "BBBBBBBBBBBBBBBBBBBBBB==", - "time_generated": "2026-05-07T16:12:53.764Z", - "token_issued_at": "2026-05-07T16:05:34.000Z" - }, - "result_signature": "200" - }, - "correlation_id": "00000003-0003-0003-0003-000000000003", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + "service_principal_id": "fa5cc200-f464-4068-88b9-c6b8335c8c70", + "sign_in_activity_id": "Rr_bbVOVu0WHOTzl0BhgAA==", + "time_generated": "2026-05-11T12:41:10.943Z", + "token_issued_at": "2026-05-11T12:35:58.000Z", + "wids": [ + "0997a1d0-0d1d-4acb-b408-d5ca73121e90" + ] + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -320,9 +304,9 @@ "region_iso_code": "GB-ENG", "region_name": "England" }, - "ip": "81.2.69.143", + "ip": "81.2.69.144", "user": { - "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + "id": "fa5cc200-f464-4068-88b9-c6b8335c8c70" } }, "cloud": { @@ -330,23 +314,29 @@ "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, "event": { - "action": "tenant-details-read", + "action": "groups-read", "category": [ "iam", "web" ], - "duration": 57000000, + "duration": 44000000, "id": "00000003-0003-0003-0003-000000000003", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000003-0003-0003-0003-000000000003\",\"durationMs\":57,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":57,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000003-0003-0003-0003-000000000003\",\"requestId\":\"00000003-0003-0003-0003-000000000003\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails\",\"responseSizeBytes\":13324,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"BBBBBBBBBBBBBBBBBBBBBB==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T16:12:53.7641499Z\",\"tokenIssuedAt\":\"2026-05-07T16:05:34Z\",\"userAgent\":\"azure-graph-test-probe/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T16:12:53.7641499Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639141000000000000,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000003-0003-0003-0003-000000000003\",\"actorType\":\"Application\",\"apiVersion\":\"AGSbeta-internal\",\"appId\":\"65d91a3d-ab74-42e6-8a2f-0add61688c74\",\"callerIpAddress\":\"81.2.69.144\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":44,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"GET\",\"httpStatusCode\":200,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/11/2026 12:35:58 PM\",\"location\":\"WestUS\",\"requestId\":\"00000003-0003-0003-0003-000000000003\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/groups\",\"responseSizeBytes\":695,\"roles\":\" Group.Read.All,\",\"scopes\":\"\",\"servicePrincipalId\":\"fa5cc200-f464-4068-88b9-c6b8335c8c70\",\"sessionId\":\"\",\"signInActivityId\":\"Rr_bbVOVu0WHOTzl0BhgAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-11T12:41:10.9430757Z\",\"userAgent\":\"\",\"userId\":\"\",\"wids\":\" 0997a1d0-0d1d-4acb-b408-d5ca73121e90,\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "success", "type": [ "access", @@ -359,24 +349,21 @@ "method": "GET" }, "response": { - "bytes": 13324, + "bytes": 695, "status_code": 200 } }, - "log": { - "level": "4" - }, "related": { "ip": [ - "81.2.69.143" + "81.2.69.144" ], "user": [ - "b37ec517-0a34-4266-b627-f7bb0d679d70", - "04b07795-8ddb-461a-bbee-02f9e1bf7b46" + "fa5cc200-f464-4068-88b9-c6b8335c8c70", + "65d91a3d-ab74-42e6-8a2f-0add61688c74" ] }, - "session": { - "id": "5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a" + "service": { + "id": "fa5cc200-f464-4068-88b9-c6b8335c8c70" }, "source": { "geo": { @@ -391,27 +378,151 @@ "region_iso_code": "GB-ENG", "region_name": "England" }, - "ip": "81.2.69.143" + "ip": "81.2.69.144" }, "tags": [ "preserve_original_event" ], - "token": { - "id": "BBBBBBBBBBBBBBBBBBBBBB==" - }, "url": { - "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails", - "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/tenantDetails" + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/groups", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/groups" }, "user": { - "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" + "id": "fa5cc200-f464-4068-88b9-c6b8335c8c70" + } + }, + { + "@timestamp": "2026-05-09T12:33:05.990Z", + "azure": { + "aadgraphactivitylogs": { + "category": "AzureADGraphActivityLogs", + "operation_name": "AAD Graph Activity", + "properties": { + "actor_type": "Application", + "api_version": "1.61-internal", + "app_id": "66244124-575c-4284-92bc-fdd00e669cea", + "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", + "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", + "service_principal_id": "4ed697af-4af9-43f7-8bf2-7b2172300068", + "sign_in_activity_id": "-TZsVeDWj0e-jamRtuUnAA==", + "time_generated": "2026-05-09T12:33:05.990Z", + "token_issued_at": "2026-05-09T12:28:05.000Z" + } + }, + "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, - "user_agent": { - "device": { - "name": "Other" + "client": { + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } }, - "name": "Other", - "original": "azure-graph-test-probe/1.0" + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "user": { + "id": "4ed697af-4af9-43f7-8bf2-7b2172300068" + } + }, + "cloud": { + "account": { + "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" + }, + "provider": "azure", + "region": "SouthCentralUS", + "service": { + "name": "Azure AD Graph" + } + }, + "destination": { + "geo": { + "region_name": "SouthCentralUS" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "batch-execute", + "category": [ + "iam", + "web" + ], + "duration": 152000000, + "id": "00000004-0004-0004-0004-000000000004", + "kind": "event", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"SouthCentralUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639141000000000001,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000004-0004-0004-0004-000000000004\",\"actorType\":\"Application\",\"apiVersion\":\"1.61-internal\",\"appId\":\"66244124-575c-4284-92bc-fdd00e669cea\",\"callerIpAddress\":\"89.160.20.156\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":152,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"POST\",\"httpStatusCode\":202,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/9/2026 12:28:05 PM\",\"location\":\"SouthCentralUS\",\"requestId\":\"00000004-0004-0004-0004-000000000004\",\"requestUri\":\"/ab30785b-417f-42a4-b5dc-8f9051718acb/$batch\",\"responseSizeBytes\":9580,\"roles\":\"\",\"scopes\":\"\",\"servicePrincipalId\":\"4ed697af-4af9-43f7-8bf2-7b2172300068\",\"sessionId\":\"\",\"signInActivityId\":\"-TZsVeDWj0e-jamRtuUnAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-09T12:33:05.9908603Z\",\"userAgent\":\"\",\"userId\":\"\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", + "outcome": "success", + "type": [ + "access", + "change" + ] + }, + "http": { + "request": { + "id": "00000004-0004-0004-0004-000000000004", + "method": "POST" + }, + "response": { + "bytes": 9580, + "status_code": 202 + } + }, + "related": { + "ip": [ + "89.160.20.156" + ], + "user": [ + "4ed697af-4af9-43f7-8bf2-7b2172300068", + "66244124-575c-4284-92bc-fdd00e669cea" + ] + }, + "service": { + "id": "4ed697af-4af9-43f7-8bf2-7b2172300068" + }, + "source": { + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/ab30785b-417f-42a4-b5dc-8f9051718acb/$batch", + "path": "/ab30785b-417f-42a4-b5dc-8f9051718acb/$batch" + }, + "user": { + "id": "4ed697af-4af9-43f7-8bf2-7b2172300068" } }, { @@ -420,14 +531,14 @@ "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { "actor_type": "User", "api_version": "1.6", "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000004-0004-0004-0004-000000000004", "scopes": [ "62e90394-69f5-4237-9190-012177145e10" ], @@ -435,13 +546,7 @@ "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", "time_generated": "2026-05-07T15:15:44.490Z", "token_issued_at": "2026-05-07T13:50:39.000Z" - }, - "result_signature": "200" - }, - "correlation_id": "00000004-0004-0004-0004-000000000004", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -468,42 +573,45 @@ "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, "event": { - "action": "applications-read", + "action": "applications-update", "category": [ "iam", "web" ], "duration": 71000000, - "id": "00000004-0004-0004-0004-000000000004", + "id": "00000005-0005-0005-0005-000000000005", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000004-0004-0004-0004-000000000004\",\"durationMs\":71,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":71,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000004-0004-0004-0004-000000000004\",\"requestId\":\"00000004-0004-0004-0004-000000000004\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications\",\"responseSizeBytes\":10026,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:15:44.4909654Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:15:44.4909654Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639142000000000000,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000005-0005-0005-0005-000000000005\",\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"callerIpAddress\":\"81.2.69.143\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":71,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"PATCH\",\"httpStatusCode\":204,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/7/2026 1:50:39 PM\",\"location\":\"WestUS\",\"requestId\":\"00000005-0005-0005-0005-000000000005\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications/fda2d70f-7df3-4556-8bcb-30acc0a45082\",\"responseSizeBytes\":0,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:15:44.4909654Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "success", "type": [ "access", - "info" + "change" ] }, "http": { "request": { - "id": "00000004-0004-0004-0004-000000000004", - "method": "GET" + "id": "00000005-0005-0005-0005-000000000005", + "method": "PATCH" }, "response": { - "bytes": 10026, - "status_code": 200 + "bytes": 0, + "status_code": 204 } }, - "log": { - "level": "4" - }, "related": { "ip": [ "81.2.69.143" @@ -534,12 +642,9 @@ "tags": [ "preserve_original_event" ], - "token": { - "id": "AAAAAAAAAAAAAAAAAAAAAA==" - }, "url": { - "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications", - "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications" + "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications/fda2d70f-7df3-4556-8bcb-30acc0a45082", + "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/applications/fda2d70f-7df3-4556-8bcb-30acc0a45082" }, "user": { "id": "b37ec517-0a34-4266-b627-f7bb0d679d70" diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml index 2d46cce9060..e3aec8fce7d 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -71,92 +71,73 @@ processors: } ctx.azure['aadgraphactivitylogs'] = keysToSnakeCase(ctx.azure.aadgraphactivitylogs); + # Drop Microsoft's internal UDI plumbing fields. They are present on every + # AAD Graph event in the diagnostic stream (Windows .NET ticks timestamp, + # region scope, internal tenant id, internal unique id) and carry no + # detection value. + - remove: + field: + - azure.aadgraphactivitylogs.properties.__udi_required_fields_event_time + - azure.aadgraphactivitylogs.properties.__udi_required_fields_region_scope + - azure.aadgraphactivitylogs.properties.__udi_required_fields_tenant_id + - azure.aadgraphactivitylogs.properties.__udi_required_fields_unique_id + ignore_missing: true + tag: remove-udi_required_fields - date: - field: azure.aadgraphactivitylogs.time + field: azure.aadgraphactivitylogs.properties.time_generated target_field: '@timestamp' - if: ctx.azure?.aadgraphactivitylogs?.time != null - tag: date-time - # Order is prioritized by observed frequency for this log - # category. - # We expect the first format to be used in 99% of cases. - # However, we include the other formats seen in other - # categories as a necessary fallback. - formats: + if: ctx.azure?.aadgraphactivitylogs?.properties?.time_generated != null + tag: date-timestamp_from_time_generated + # ISO8601 is the canonical event-hub shape. Other formats are kept as + # defensive fallbacks for tenant variants seen in sibling Azure + # categories. + formats: - ISO8601 - "M/d/yyyy h:mm:ss a XXX" - "M/d/yyyy h:mm:ss a" - "M/d/yyyy H:mm:ss" - yyyy-MM-dd'T'H:mm:ss.SSS'Z' - - remove: - field: azure.aadgraphactivitylogs.time + # In the live event-hub shape this field is named `issuedAt` and uses the + # M/d/yyyy h:mm:ss a format (e.g. "5/11/2026 12:35:58 PM"). Rename to + # token_issued_at so downstream consumers see a consistent name. + - rename: + field: azure.aadgraphactivitylogs.properties.issued_at + target_field: azure.aadgraphactivitylogs.properties.token_issued_at ignore_missing: true - tag: remove-time + tag: rename-properties_issued_at - date: field: azure.aadgraphactivitylogs.properties.token_issued_at target_field: azure.aadgraphactivitylogs.properties.token_issued_at if: ctx.azure?.aadgraphactivitylogs?.properties?.token_issued_at != null tag: date-token_issued_at - # Order is prioritized by observed frequency for this log - # category. - # We expect the first format to be used in 99% of cases. - # However, we include the other formats seen in other - # categories as a necessary fallback. - formats: - - ISO8601 - - "M/d/yyyy h:mm:ss a XXX" + formats: - "M/d/yyyy h:mm:ss a" - - "M/d/yyyy H:mm:ss" - - yyyy-MM-dd'T'H:mm:ss.SSS'Z' + - ISO8601 on_failure: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - rename: - field: azure.aadgraphactivitylogs.resource_id - target_field: azure.resource_id - ignore_missing: true - tag: rename-resource_id - - convert: - field: azure.aadgraphactivitylogs.caller_ip_address - target_field: source.ip - type: ip - ignore_missing: true - tag: convert-caller_ip_address - on_failure: - - rename: - field: azure.aadgraphactivitylogs.caller_ip_address - target_field: source.address - ignore_missing: true - tag: rename-caller_ip_address - on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + # In the live event-hub shape callerIpAddress is nested under properties. - convert: - field: azure.aadgraphactivitylogs.properties.ip_address - target_field: source.ip - type: ip - ignore_missing: true - tag: convert-properties_ip_address - if: ctx.source?.ip == null - on_failure: - - rename: - field: azure.aadgraphactivitylogs.properties.ip_address + field: azure.aadgraphactivitylogs.properties.caller_ip_address + target_field: source.ip + type: ip + ignore_missing: true + tag: convert-properties_caller_ip_address + on_failure: + - rename: + field: azure.aadgraphactivitylogs.properties.caller_ip_address target_field: source.address - ignore_missing: true - tag: rename-properties_ip_address - if: ctx.source?.address == null + ignore_missing: true + tag: rename-properties_caller_ip_address on_failure: - - append: - field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - remove: - field: - - azure.aadgraphactivitylogs.caller_ip_address - - azure.aadgraphactivitylogs.properties.ip_address + field: azure.aadgraphactivitylogs.properties.caller_ip_address if: 'ctx.source?.ip != null' - tag: remove-caller_ip_address + tag: remove-properties_caller_ip_address ignore_missing: true - set: field: client.ip @@ -184,27 +165,12 @@ processors: copy_from: azure.aadgraphactivitylogs.properties.session_id tag: set_session_id ignore_empty_value: true - - set: - field: token.id - copy_from: azure.aadgraphactivitylogs.properties.sign_in_activity_id - tag: set_token_id - ignore_empty_value: true - append: field: related.ip tag: append-related_ip value: '{{source.ip}}' allow_duplicates: false if: 'ctx.source?.ip != null' - - convert: - field: azure.aadgraphactivitylogs.level - target_field: log.level - type: string - tag: convert-level - ignore_missing: true - - remove: - field: azure.aadgraphactivitylogs.level - ignore_missing: true - tag: remove-level - convert: field: azure.aadgraphactivitylogs.duration_ms target_field: event.duration @@ -279,21 +245,16 @@ processors: tag: remove-tenant_id if: ctx.azure?.tenant_id != null ignore_missing: true - - rename: - field: azure.aadgraphactivitylogs.correlation_id - target_field: azure.correlation_id - ignore_missing: true - tag: rename-correlation_id - rename: field: azure.aadgraphactivitylogs.properties.request_id target_field: http.request.id ignore_missing: true tag: rename-properties_request_id - rename: - field: azure.aadgraphactivitylogs.properties.request_method + field: azure.aadgraphactivitylogs.properties.http_method target_field: http.request.method ignore_missing: true - tag: rename-properties_request_methods + tag: rename-properties_http_method - uri_parts: field: azure.aadgraphactivitylogs.properties.request_uri if: ctx.azure?.aadgraphactivitylogs?.properties?.request_uri != null @@ -314,14 +275,26 @@ processors: ignore_missing: true tag: rename-properties_response_bytes - rename: - field: azure.aadgraphactivitylogs.properties.response_status_code + field: azure.aadgraphactivitylogs.properties.http_status_code target_field: http.response.status_code ignore_missing: true - tag: rename-properties_response_statuscode + tag: rename-properties_http_status_code + # `roles` and `wids` arrive as comma-delimited strings with frequent leading + # whitespace and trailing commas (e.g. " Group.Read.All," or + # " 0997a1d0-...,"). Normalize first (strip surrounding whitespace/commas) + # then split on comma. `scopes` is space-separated per the MS Graph + # convention and is left as-is. + - gsub: + field: azure.aadgraphactivitylogs.properties.roles + pattern: '^[\s,]+|[\s,]+$' + replacement: '' + if: ctx.azure?.aadgraphactivitylogs?.properties?.roles != null && ctx.azure.aadgraphactivitylogs.properties.roles != '' + tag: gsub-properties_roles + ignore_missing: true - split: field: azure.aadgraphactivitylogs.properties.roles if: ctx.azure?.aadgraphactivitylogs?.properties?.roles != null && ctx.azure.aadgraphactivitylogs.properties.roles != '' - separator: ' ' + separator: '\s*,\s*' tag: split-properties_roles on_failure: - append: @@ -336,10 +309,17 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - gsub: + field: azure.aadgraphactivitylogs.properties.wids + pattern: '^[\s,]+|[\s,]+$' + replacement: '' + if: ctx.azure?.aadgraphactivitylogs?.properties?.wids != null && ctx.azure.aadgraphactivitylogs.properties.wids != '' + tag: gsub-properties_wids + ignore_missing: true - split: field: azure.aadgraphactivitylogs.properties.wids if: ctx.azure?.aadgraphactivitylogs?.properties?.wids != null && ctx.azure.aadgraphactivitylogs.properties.wids != '' - separator: ' ' + separator: '\s*,\s*' tag: split-properties_wids on_failure: - append: @@ -509,9 +489,16 @@ processors: String method = ctx.http?.request?.method; if (path == null || method == null) { return; } String[] parts = path.splitOnToken('/'); - // expected shape: ["", "v2", "", "", ...] - if (parts.length < 4) { return; } - String collection = parts[3]; + // Observed AAD Graph request URI shapes: + // /v2/{tid-or-myorganization}/{collection}/... -> collection at parts[3] + // /{tid-or-myorganization}/$batch (or other) -> collection at parts[2] + if (parts.length < 3) { return; } + String collection; + if (parts.length >= 4 && parts[1] == 'v2') { + collection = parts[3]; + } else { + collection = parts[2]; + } if (collection == null || collection.isEmpty()) { return; } if (collection.startsWith('$')) { collection = collection.substring(1); } // CamelCase -> hyphen-lower (servicePrincipals -> service-principals) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml index 4fe793d10ad..7db19b0af10 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/ecs-extended.yml @@ -4,9 +4,3 @@ - name: id type: keyword description: The unique identifier for the authentication session. -- name: token - type: group - fields: - - name: id - type: keyword - description: The unique token identifier of the API call. diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml index d26170c2fb6..29d2dd14920 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml @@ -5,18 +5,10 @@ type: keyword description: | Azure Event Category. For Azure AD Graph Activity Logs, this is `AzureADGraphActivityLogs`. - - name: operation_version - type: keyword - description: | - The legacy Azure AD Graph API version of the event (e.g. `1.6`). - name: operation_name type: keyword description: | - Operation name. - - name: result_signature - type: keyword - description: | - Result signature. + Operation name. For this category the value is always the literal string `AAD Graph Activity`; rely on the derived `event.action` (HTTP method + directory collection from `requestUri`) for detection. - name: properties type: group description: | @@ -30,10 +22,6 @@ type: keyword description: | The identifier for the application. - - name: at_content - type: keyword - description: | - Reserved for future use. - name: billed_size type: double description: | @@ -54,10 +42,14 @@ type: integer description: | Indicates how the client was authenticated. For a public client, the value is 0. If client ID and client secret are used, the value is 1. If a client certificate was used for authentication, the value is 2. - - name: client_request_id + - name: direct_access_source + type: keyword + description: | + The path through which the request reached the AAD Graph service (e.g. `Gateway`). + - name: env_cloud_role type: keyword description: | - The client request identifier when sent. If no client request identifier is sent, the value will be equal to the operation identifier. + The Microsoft cloud role identifier for the service handling the request (e.g. `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. - name: identity_provider type: keyword description: | @@ -66,10 +58,6 @@ type: boolean description: | Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account. - - name: operation_id - type: keyword - description: | - The identifier for the batch. For non-batched requests, this will be unique per request. For batched requests, this will be the same for all requests in the batch. - name: request_uri type: keyword description: | @@ -89,7 +77,7 @@ - name: sign_in_activity_id type: keyword description: | - The identifier representing the sign-in activitys. + Identifier of the Microsoft Entra ID sign-in event that established the authentication context for this AAD Graph request. Correlates this request with an entry in Entra ID Sign-In Logs. - name: source_system type: keyword description: | From 97285cc9e4b4a0a3b14e3515412629ee833d27bb Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:27:54 -0400 Subject: [PATCH 11/19] azure/aadgraphactivitylogs: fix vale linting warnings --- .../_dev/build/docs/aadgraphactivitylogs.md | 12 ++++++------ .../aadgraphactivitylogs/fields/fields.yml | 6 +++--- packages/azure/docs/aadgraphactivitylogs.md | 18 +++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/azure/_dev/build/docs/aadgraphactivitylogs.md b/packages/azure/_dev/build/docs/aadgraphactivitylogs.md index 83db4c02976..0f8590fbc89 100644 --- a/packages/azure/_dev/build/docs/aadgraphactivitylogs.md +++ b/packages/azure/_dev/build/docs/aadgraphactivitylogs.md @@ -26,7 +26,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m `eventhub` : _string_ -It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name; in this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (e.g., the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. +It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name. In this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (for example, the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. Default value `insights-operational-logs`. `consumer_group` : @@ -38,7 +38,7 @@ Default value: `$Default` _string_ The connection string required to communicate with Event Hubs, steps [here](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). -A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. +A Blob Storage account is required to store, retrieve, and update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. `storage_account` : _string_ @@ -54,7 +54,7 @@ The storage account container where the integration stores the checkpoint data f `resource_manager_endpoint` : _string_ -Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different Azure environment. +Optional. By default the integration uses the Azure public environment. To override, provide a specific resource manager endpoint to use a different Azure environment. Resource manager endpoints: @@ -76,14 +76,14 @@ https://management.usgovcloudapi.net/ ### aadgraphactivitylogs -The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`; the events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. +The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`. The events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. -Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled; only new events are routed to the dedicated dataset. +Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled. Only new events are routed to the dedicated dataset. {{event "aadgraphactivitylogs"}} **ECS Field Reference** -Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. +Refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. {{fields "aadgraphactivitylogs"}} diff --git a/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml index 29d2dd14920..9bd88f24f84 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/fields/fields.yml @@ -29,7 +29,7 @@ - name: actor_type type: keyword description: | - Type of identity that issued the request, e.g. `User`, `ServicePrincipal`. + Type of identity that issued the request, for example `User`, `ServicePrincipal`. - name: device_id type: keyword description: | @@ -45,11 +45,11 @@ - name: direct_access_source type: keyword description: | - The path through which the request reached the AAD Graph service (e.g. `Gateway`). + The path through which the request reached the AAD Graph service (for example, `Gateway`). - name: env_cloud_role type: keyword description: | - The Microsoft cloud role identifier for the service handling the request (e.g. `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. + The Microsoft cloud role identifier for the service handling the request (for example, `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. - name: identity_provider type: keyword description: | diff --git a/packages/azure/docs/aadgraphactivitylogs.md b/packages/azure/docs/aadgraphactivitylogs.md index 702e847414d..4f532b27027 100644 --- a/packages/azure/docs/aadgraphactivitylogs.md +++ b/packages/azure/docs/aadgraphactivitylogs.md @@ -26,7 +26,7 @@ Refer to the [Azure Logs](https://docs.elastic.co/integrations/azure) page for m `eventhub` : _string_ -It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name; in this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (e.g., the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. +It is a fully managed, real-time data ingestion service. Elastic recommends using only letters, numbers, and the hyphen (-) character for Event Hub names to maximize compatibility. You _can_ use existing Event Hubs having underscores (_) in the Event Hub name. In this case, the integration will replace underscores with hyphens (-) when it uses the Event Hub name to create dependent Azure resources behind the scenes (for example, the storage account container to store Event Hub consumer offsets). Elastic also recommends using a separate event hub for each log type as the field mappings of each log type differ. Default value `insights-operational-logs`. `consumer_group` : @@ -38,7 +38,7 @@ Default value: `$Default` _string_ The connection string required to communicate with Event Hubs, steps [here](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string). -A Blob Storage account is required in order to store/retrieve/update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. +A Blob Storage account is required to store, retrieve, and update the offset or state of the eventhub messages. This means that after stopping the filebeat azure module it can start back up at the spot that it stopped processing messages. `storage_account` : _string_ @@ -54,7 +54,7 @@ The storage account container where the integration stores the checkpoint data f `resource_manager_endpoint` : _string_ -Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different Azure environment. +Optional. By default the integration uses the Azure public environment. To override, provide a specific resource manager endpoint to use a different Azure environment. Resource manager endpoints: @@ -76,9 +76,9 @@ https://management.usgovcloudapi.net/ ### aadgraphactivitylogs -The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`; the events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. +The `aadgraphactivitylogs` data stream of the Azure Logs package collects Azure AD Graph activity events that have been streamed through an Azure event hub. The events ingest pipeline matches `category == "AzureADGraphActivityLogs"` and sets `event.dataset = azure.aadgraphactivitylogs`. The events data stream's routing rules then reroute the document from `logs-azure.events-*` directly to `logs-azure.aadgraphactivitylogs-*`, where this data stream's pipeline applies full ECS field extraction. -Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled; only new events are routed to the dedicated dataset. +Before this data stream existed, AAD Graph events had no specific override in the events router and fell through to the `azure.platformlogs` catch-all, landing in `logs-azure.platformlogs-default` with only generic platform-log parsing. Those previously-indexed events are not backfilled. Only new events are routed to the dedicated dataset. An example event for `aadgraphactivitylogs` looks as following: @@ -225,7 +225,7 @@ An example event for `aadgraphactivitylogs` looks as following: **ECS Field Reference** -Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. +Refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. **Exported fields** @@ -234,14 +234,14 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | @timestamp | Event timestamp. | date | | azure.aadgraphactivitylogs.category | Azure Event Category. For Azure AD Graph Activity Logs, this is `AzureADGraphActivityLogs`. | keyword | | azure.aadgraphactivitylogs.operation_name | Operation name. For this category the value is always the literal string `AAD Graph Activity`; rely on the derived `event.action` (HTTP method + directory collection from `requestUri`) for detection. | keyword | -| azure.aadgraphactivitylogs.properties.actor_type | Type of identity that issued the request, e.g. `User`, `ServicePrincipal`. | keyword | +| azure.aadgraphactivitylogs.properties.actor_type | Type of identity that issued the request, for example `User`, `ServicePrincipal`. | keyword | | azure.aadgraphactivitylogs.properties.api_version | The API version of the event. | keyword | | azure.aadgraphactivitylogs.properties.app_id | The identifier for the application. | keyword | | azure.aadgraphactivitylogs.properties.billed_size | The record size in bytes. | double | | azure.aadgraphactivitylogs.properties.client_auth_method | Indicates how the client was authenticated. For a public client, the value is 0. If client ID and client secret are used, the value is 1. If a client certificate was used for authentication, the value is 2. | integer | | azure.aadgraphactivitylogs.properties.device_id | The identifier of the device from which the authentication request originated. | keyword | -| azure.aadgraphactivitylogs.properties.direct_access_source | The path through which the request reached the AAD Graph service (e.g. `Gateway`). | keyword | -| azure.aadgraphactivitylogs.properties.env_cloud_role | The Microsoft cloud role identifier for the service handling the request (e.g. `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. | keyword | +| azure.aadgraphactivitylogs.properties.direct_access_source | The path through which the request reached the AAD Graph service (for example, `Gateway`). | keyword | +| azure.aadgraphactivitylogs.properties.env_cloud_role | The Microsoft cloud role identifier for the service handling the request (for example, `restdirectoryservice`). Useful for distinguishing first-party Microsoft service traffic from third-party callers. | keyword | | azure.aadgraphactivitylogs.properties.identity_provider | The identity provider that authenticated the subject of the token. | keyword | | azure.aadgraphactivitylogs.properties.is_billable | Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account. | boolean | | azure.aadgraphactivitylogs.properties.request_uri | The URI of the request. | keyword | From 7be1278fcf4bcd9e2723782c95c43f5a55f45a2d Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:34:03 -0400 Subject: [PATCH 12/19] azure/aadgraphactivitylogs: adds shared pipeline link --- .../ingest_pipeline/azure-shared-pipeline.yml | 101 ------------------ .../azure-shared-pipeline.yml.link | 1 + 2 files changed, 1 insertion(+), 101 deletions(-) delete mode 100644 packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml create mode 100644 packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml.link diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml deleted file mode 100644 index d7d26e8d31d..00000000000 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml +++ /dev/null @@ -1,101 +0,0 @@ ---- -description: Pipeline for parsing Azure shared log fields. -processors: - - set: - field: cloud.provider - value: azure - - grok: - field: azure.resource_id - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - GROUPID: .+ - PROVIDERNAME: .+ - NAMESPACE: .+ - RULE: .+ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - GROUPID: .+ - PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+. - NAME: ((?!AUTHORIZATIONRULES).)*$ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - GROUPID: .+ - PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+ - NAME: .+ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /providers/%{PROVIDER:azure.resource.provider} - - /PROVIDERS/%{PROVIDER:azure.resource.provider} - pattern_definitions: - PROVIDER: .+ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider} - - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group} - - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - GROUPID: .+ - ignore_failure: true - - grok: - field: azure.resource_id - if: 'ctx.azure?.subscription_id == null' - patterns: - - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id} - - /subscriptions/%{SUBID:azure.subscription_id} - pattern_definitions: - SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} - ignore_failure: true - - rename: - field: azure.resource_id - target_field: azure.resource.id - ignore_missing: true - - lowercase: - field: event.outcome - ignore_missing: true -on_failure: - - set: - field: event.kind - value: pipeline_error - - append: - field: tags - value: preserve_original_event - allow_duplicates: false - - append: - field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{/_ingest.on_failure_processor_tag}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml.link b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml.link new file mode 100644 index 00000000000..e7d39a1f92f --- /dev/null +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml.link @@ -0,0 +1 @@ +../../../graphactivitylogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml d9410a8b01f785a2e328560cc5d9a2286c8ca5b4e6ab5ec0edfaba385ddb0fb9 From a62a1acfa031743851f6da084009f8701383be18 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:38:54 -0400 Subject: [PATCH 13/19] azure/aadgraphactivitylogs: remove 8.14 workaround for dynamic field url.extension --- .../_dev/test/pipeline/test-common-config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml index f00b11ca2b6..4da22641654 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/_dev/test/pipeline/test-common-config.yml @@ -1,7 +1,3 @@ fields: tags: - preserve_original_event -dynamic_fields: - # This can be removed after ES 8.14 is the minimum version. - # Relates: https://github.com/elastic/elasticsearch/pull/105689 - url.extension: '^.*$' From 99758fd2ab81c636797e80c1b72efaf6890483fe Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:44:30 -0400 Subject: [PATCH 14/19] azure/aadgraphactivitylogs: adds aadgraphactivitylogs schema reference as comment in ingest pipeline --- .../elasticsearch/ingest_pipeline/default.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml index e3aec8fce7d..a1eeb106889 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -1,4 +1,16 @@ --- +# Canonical schema reference for Azure AD Graph activity logs: +# https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables/aadgraphactivitylogs +# +# Note: the event-hub envelope shape differs from the Log Analytics column +# projection documented at the URL above. Event-hub payloads use `httpMethod` +# / `httpStatusCode` / `issuedAt` (and nest `callerIpAddress` under +# `properties`), while the Log Analytics view exposes `RequestMethod` / +# `ResponseStatusCode` / `TokenIssuedAt` and adds projection-only columns +# like `AADTenantId`, `_SubscriptionId`, `SourceSystem`, `Type`, `Level`, +# `Location` (envelope-level), `ResultSignature`, etc. This pipeline targets +# the event-hub shape; projection-only columns are not handled because they +# never appear in events streamed via diagnostic settings. description: Pipeline for parsing azure AAD Graph activity logs. processors: - set: From 7f1b4b693985486e62b524ba0b62643b78cfd95f Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 11:47:11 -0400 Subject: [PATCH 15/19] azure/aadgraphactivitylogs: added Azure AD Graph Activity Logs right after Microsoft Graph Activity Logs in the supported-services list. --- packages/azure/data_stream/events/manifest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/azure/data_stream/events/manifest.yml b/packages/azure/data_stream/events/manifest.yml index c11e8d905d5..0e33d9065ac 100644 --- a/packages/azure/data_stream/events/manifest.yml +++ b/packages/azure/data_stream/events/manifest.yml @@ -19,6 +19,7 @@ streams: - Platform logs - Activity logs - Microsoft Graph Activity Logs + - Azure AD Graph Activity Logs - Spring Apps logs - Firewall logs - Application Gateway logs From c6ba91a4aec8445bbcb65ce1f2f300afb80554b1 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Mon, 11 May 2026 12:20:16 -0400 Subject: [PATCH 16/19] azure/aadgraphactivitylogs: sample event and documentation fix --- .../aadgraphactivitylogs/sample_event.json | 28 ++++++++----------- packages/azure/docs/aadgraphactivitylogs.md | 26 +++++++---------- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json b/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json index ea530b79392..7c00d6b3203 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json +++ b/packages/azure/data_stream/aadgraphactivitylogs/sample_event.json @@ -4,14 +4,14 @@ "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { "actor_type": "User", "api_version": "1.6", "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000001-0001-0001-0001-000000000001", "scopes": [ "62e90394-69f5-4237-9190-012177145e10" ], @@ -19,13 +19,7 @@ "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", "time_generated": "2026-05-07T15:19:33.536Z", "token_issued_at": "2026-05-07T13:50:39.000Z" - }, - "result_signature": "200" - }, - "correlation_id": "00000001-0001-0001-0001-000000000001", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -52,10 +46,16 @@ "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, @@ -68,7 +68,7 @@ "duration": 59000000, "id": "00000001-0001-0001-0001-000000000001", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639140000000000000,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000001-0001-0001-0001-000000000001\",\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"callerIpAddress\":\"81.2.69.143\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":59,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"GET\",\"httpStatusCode\":200,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/7/2026 1:50:39 PM\",\"location\":\"WestUS\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "success", "type": [ "access", @@ -85,9 +85,6 @@ "status_code": 200 } }, - "log": { - "level": "4" - }, "related": { "ip": [ "81.2.69.143" @@ -118,9 +115,6 @@ "tags": [ "preserve_original_event" ], - "token": { - "id": "AAAAAAAAAAAAAAAAAAAAAA==" - }, "url": { "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" @@ -135,4 +129,4 @@ "name": "Other", "original": "azure-graph-test-client/1.0" } -} \ No newline at end of file +} diff --git a/packages/azure/docs/aadgraphactivitylogs.md b/packages/azure/docs/aadgraphactivitylogs.md index 4f532b27027..2c8e8fe1de7 100644 --- a/packages/azure/docs/aadgraphactivitylogs.md +++ b/packages/azure/docs/aadgraphactivitylogs.md @@ -89,14 +89,14 @@ An example event for `aadgraphactivitylogs` looks as following: "aadgraphactivitylogs": { "category": "AzureADGraphActivityLogs", "operation_name": "AAD Graph Activity", - "operation_version": "1.6", "properties": { "actor_type": "User", "api_version": "1.6", "app_id": "04b07795-8ddb-461a-bbee-02f9e1bf7b46", "client_auth_method": 0, + "direct_access_source": "Gateway", + "env_cloud_role": "restdirectoryservice", "identity_provider": "https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/", - "operation_id": "00000001-0001-0001-0001-000000000001", "scopes": [ "62e90394-69f5-4237-9190-012177145e10" ], @@ -104,13 +104,7 @@ An example event for `aadgraphactivitylogs` looks as following: "sign_in_activity_id": "AAAAAAAAAAAAAAAAAAAAAA==", "time_generated": "2026-05-07T15:19:33.536Z", "token_issued_at": "2026-05-07T13:50:39.000Z" - }, - "result_signature": "200" - }, - "correlation_id": "00000001-0001-0001-0001-000000000001", - "resource": { - "id": "/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM", - "provider": "MICROSOFT.AADIAM" + } }, "tenant_id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, @@ -137,10 +131,16 @@ An example event for `aadgraphactivitylogs` looks as following: "id": "ab30785b-417f-42a4-b5dc-8f9051718acb" }, "provider": "azure", + "region": "WestUS", "service": { "name": "Azure AD Graph" } }, + "destination": { + "geo": { + "region_name": "WestUS" + } + }, "ecs": { "version": "8.11.0" }, @@ -153,7 +153,7 @@ An example event for `aadgraphactivitylogs` looks as following: "duration": 59000000, "id": "00000001-0001-0001-0001-000000000001", "kind": "event", - "original": "{\"Level\":4,\"callerIpAddress\":\"81.2.69.143\",\"category\":\"AzureADGraphActivityLogs\",\"correlationId\":\"00000001-0001-0001-0001-000000000001\",\"durationMs\":59,\"operationName\":\"AAD Graph Activity\",\"operationVersion\":\"1.6\",\"properties\":{\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"clientAuthMethod\":\"0\",\"durationMs\":59,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"ipAddress\":\"81.2.69.143\",\"operationId\":\"00000001-0001-0001-0001-000000000001\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestMethod\":\"GET\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"responseStatusCode\":200,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"tokenIssuedAt\":\"2026-05-07T13:50:39Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"resourceId\":\"/TENANTS/AB30785B-417F-42A4-B5DC-8F9051718ACB/PROVIDERS/MICROSOFT.AADIAM\",\"resultSignature\":\"200\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"time\":\"2026-05-07T15:19:33.5368860Z\"}", + "original": "{\"category\":\"AzureADGraphActivityLogs\",\"location\":\"WestUS\",\"operationName\":\"AAD Graph Activity\",\"properties\":{\"__UDI_RequiredFields_EventTime\":639140000000000000,\"__UDI_RequiredFields_RegionScope\":\"NA\",\"__UDI_RequiredFields_TenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"__UDI_RequiredFields_UniqueId\":\"00000001-0001-0001-0001-000000000001\",\"actorType\":\"User\",\"apiVersion\":\"1.6\",\"appId\":\"04b07795-8ddb-461a-bbee-02f9e1bf7b46\",\"callerIpAddress\":\"81.2.69.143\",\"clientAuthMethod\":0,\"deviceId\":\"\",\"directAccessSource\":\"Gateway\",\"durationMs\":59,\"env_cloud_role\":\"restdirectoryservice\",\"httpMethod\":\"GET\",\"httpStatusCode\":200,\"identityProvider\":\"https://sts.windows.net/ab30785b-417f-42a4-b5dc-8f9051718acb/\",\"issuedAt\":\"5/7/2026 1:50:39 PM\",\"location\":\"WestUS\",\"requestId\":\"00000001-0001-0001-0001-000000000001\",\"requestUri\":\"/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users\",\"responseSizeBytes\":54662,\"roles\":\"\",\"scopes\":\"62e90394-69f5-4237-9190-012177145e10\",\"servicePrincipalId\":\"\",\"sessionId\":\"5a5a5a5a-5a5a-5a5a-5a5a-5a5a5a5a5a5a\",\"signInActivityId\":\"AAAAAAAAAAAAAAAAAAAAAA==\",\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\",\"timeGenerated\":\"2026-05-07T15:19:33.5368860Z\",\"userAgent\":\"azure-graph-test-client/1.0\",\"userId\":\"b37ec517-0a34-4266-b627-f7bb0d679d70\",\"wids\":\"\"},\"tenantId\":\"ab30785b-417f-42a4-b5dc-8f9051718acb\"}", "outcome": "success", "type": [ "access", @@ -170,9 +170,6 @@ An example event for `aadgraphactivitylogs` looks as following: "status_code": 200 } }, - "log": { - "level": "4" - }, "related": { "ip": [ "81.2.69.143" @@ -203,9 +200,6 @@ An example event for `aadgraphactivitylogs` looks as following: "tags": [ "preserve_original_event" ], - "token": { - "id": "AAAAAAAAAAAAAAAAAAAAAA==" - }, "url": { "original": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users", "path": "/v2/ab30785b-417f-42a4-b5dc-8f9051718acb/users" From 65754633c2ab5bf399ecd5420141368a3ce68f90 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Tue, 12 May 2026 08:46:06 -0400 Subject: [PATCH 17/19] Update packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Maurizio Branca --- .../elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml index a1eeb106889..b3583a03325 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -358,7 +358,7 @@ processors: - set: field: client.user.id value: '{{user.id}}' - tag: set-client_ip + tag: set-client_user_id ignore_empty_value: true - append: field: related.user From 08110a7fa3046c2bdac0dd0765a41b64f0317514 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus <99630311+terrancedejesus@users.noreply.github.com> Date: Tue, 12 May 2026 08:46:22 -0400 Subject: [PATCH 18/19] Update packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Maurizio Branca --- .../elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml index b3583a03325..8817e4c1628 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -556,7 +556,7 @@ processors: - set: field: cloud.service.name value: 'Azure AD Graph' - tag: set-cloud_region + tag: set-cloud_service_name ignore_empty_value: true - geoip: field: client.ip From 8fae9c52502f6fa6d3153d78ea16fa265dc37851 Mon Sep 17 00:00:00 2001 From: Terrance DeJesus Date: Tue, 12 May 2026 10:27:12 -0400 Subject: [PATCH 19/19] azure/aadgraphactivitylogs: url.original keep original from fingerprint --- .../elasticsearch/ingest_pipeline/default.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml index 8817e4c1628..9d956cdaa31 100644 --- a/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/aadgraphactivitylogs/elasticsearch/ingest_pipeline/default.yml @@ -534,9 +534,12 @@ processors: - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' tag: pipeline-azure-shared-pipeline + # url.original is populated by the uri_parts processor above + # (keep_original: true) and carries the same value that + # properties.request_uri held before it was removed. - fingerprint: - fields: - - azure.aadgraphactivitylogs.properties.request_uri + fields: + - url.original - http.request.id - azure.tenant_id - azure.aadgraphactivitylogs.properties.time_generated