Skip to content

Commit ace8fb4

Browse files
ali786XIali786XI
authored
[O11y][AWS] Rally benchmark aws.cloudtrail (elastic#9448)
1 parent d4e4aa4 commit ace8fb4

File tree

4 files changed

+434
-0
lines changed

4 files changed

+434
-0
lines changed

‎packages/aws/_dev/benchmark/rally/cloudtrail-benchmark.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
description: Benchmark of 20000 aws.cloudtrail events ingested
3+
data_stream:
4+
name: cloudtrail
5+
corpora:
6+
generator:
7+
total_events: 20000
8+
template:
9+
type: gotext
10+
path: ./cloudtrail-benchmark/template.ndjson
11+
config:
12+
path: ./cloudtrail-benchmark/config.yml
13+
fields:
14+
path: ./cloudtrail-benchmark/fields.yml

‎packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/config.yml‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
fields:
2+
- name: timestamp
3+
period: -24h
4+
- name: digest_previous_s3_bucket
5+
value: "alice-bucket"
6+
- name: digest_s3_object
7+
value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz"
8+
- name: digest_public_Key_fingerprint
9+
value: "47aaa19f7eec22e9bd0b5e58cfade8cb"
10+
- name: digest_signature_algorithm
11+
value: "SHA256withRSA"
12+
- name: previous_digest_s3_object
13+
value: "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz"
14+
- name: previous_digest_hash_value
15+
value: "531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860"
16+
- name: digest_previous_hash_algorithm
17+
value: "SHA-256"
18+
- name: previous_digest_signature
19+
value: "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987"
20+
- name: eventCategory
21+
value: "Insight"
22+
- name: eventId
23+
value: "11ea990b-4678-4bcd-8fbe-625EXAMPLE"
24+
- name: userIdentity_accountId
25+
range:
26+
min: 1000000000000
27+
max: 2000000000000
28+
- name: userIdentity_type
29+
enum: ["Root", "IAMUser", "AssumedRole", "Role", "FederatedUser", "AWSAccount"]
30+
- name: userIdentity_arn
31+
value: "arn:aws:iam::123456789012:user/Alice"
32+
- name: userIdentity_eventSource
33+
value: "iam.amazonaws.com"
34+
- name: userIdentity_eventName
35+
value: "UpdateUser"
36+
- name: userIdentity_eventTime
37+
value: "2014-07-08T17:35:27Z"
38+
- name: newestEventTime
39+
value: "2020-09-11T19:26:24.000Z"
40+
- name: oldestEventTime
41+
value: "2020-09-11T18:32:04.000Z"
42+
- name: userIdentity_awsregion
43+
enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"]
44+
cardinality: 25
45+
- name: userIdentity_sourceIPAddress
46+
value: 127.0.0.1
47+
- name: userIdentity_useragent
48+
value: "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46"
49+
- name: requestID
50+
value: "3a6b3260-739d-465e-9406-bcEXAMPLE"
51+
- name: event_id
52+
value: "9150d546-3564-4262-8e62-110EXAMPLE"
53+
- name: logFiles
54+
value: "https://elastic-package-aws-bucket-64547.s3.us-east-1.amazonaws.com/cloudtrail-digest.log"
55+
- name: eventType
56+
value: "AwsApiCall"
57+
- name: userIdentity_requestparameters_username
58+
value: "username"
59+
- name: userIdentity_requestparameters_newusername
60+
value: "newusername"
61+
- name: recipientAccountId
62+
value: "123456789012"
63+
- name: digest_start_time
64+
value: "2020-09-11T18:36:49.000Z"
65+
- name: userIdentity_sessionContext_mfaauthenticated
66+
enum: ["true", "false"]
67+
- name: userIdentity_sessionContext_creationDate
68+
value: "2020-09-11T18:12:52.000Z"
69+
- name: invokedBy
70+
value: "signin.amazonaws.com"
71+
- name: sharedEventId
72+
value: "123456789012"
73+
- name: vpcEndpointId
74+
value: "123456789012"
75+
- name: userIdentity_sessionContext_sessionIssuer_type
76+
enum: ["Root", "IAMUser", "Role"]
77+
- name: userIdentity_sessionContext_sessionIssuer_principalId
78+
value: "AROAIDPPEZS35WEXAMPLE"
79+
- name: userIdentity_sessionContext_sessionIssuer_arn
80+
value: "arn:aws:iam::123456789012:role/RoleToBeAssumed"
81+
- name: userIdentity_sessionContext_sessionIssuer_accountId
82+
range:
83+
min: 1000000000000
84+
max: 2000000000000
85+
- name: userIdentity_sessionContext_sessionIssuer_userName
86+
value: "RoleToBeAssumed"
87+
- name: errorMessage
88+
value: "Failed authentication"
89+
- name: errorCode
90+
value: "AccessDeniedException"
91+
- name: apiVersion
92+
value: "api1.1"
93+
- name: responseElements_ConsoleLogin
94+
enum: ["Failure", "Success"]
95+
- name: additionalEventData_MobileVersion
96+
enum: ["Yes", "No"]
97+
- name: additionalEventData_LoginTo
98+
value: "https://console.aws.amazon.com/sns"
99+
- name: additionalEventData_MFAUsed
100+
enum: ["Yes", "No"]
101+
- name: aws_Account_Id
102+
value: "hdbcskndcl123y2873y"
103+
- name: digest_start_time
104+
value: "2020-09-11T18:36:49Z"
105+
- name: digest_end_time
106+
value: "2020-09-12T19:13:56Z"
107+
- name: digest_s3_bucket
108+
value: "alice-bucket"
109+
- name: resources_type
110+
value: "AWS::IAM::Role"
111+
- name: resources_ARN
112+
value: "arn:aws:iam::111122223333:role/JohnRole2"
113+
- name: resources_accountId
114+
value: "111111100000011111"
115+
- name: readOnly
116+
enum: ["true", "false"]
117+
- name: managementEvent
118+
value: "cloudtrail event is genearted"
119+
- name: insightDetails_state
120+
enum: ["End", "start"]
121+
- name: insightDetails_eventSource
122+
value: "iam.amazonaws.com"
123+
- name: insightDetails_eventName
124+
value: "AttachUserPolicy"
125+
- name: insightDetails_insightType
126+
value: "ApiCallRateInsight"
127+
- name: insightDetails_insffightContext_statistics_baseline_average
128+
range:
129+
min: 1
130+
max: 1000
131+
- name: insightDetails_insffightContext_statistics_insight_average
132+
range:
133+
min: 1
134+
max: 1000
135+
- name: insightDetails_insffightContext_statistics_insightDuration
136+
range:
137+
min: 1
138+
max: 100
139+
- name: insightDetails_insffightContext_statistics_baselineDuration
140+
range:
141+
min: 1
142+
max: 100000
143+
- name: insightDetails_insffightContext_attributions_attribute
144+
value: "userIdentityArn"
145+
- name: insightDetails_insffightContext_attributions_insight_value
146+
value: "arn:aws:iam::123456789012:user/Alice"
147+
- name: insightDetails_insffightContext_attributions_insight_average
148+
range:
149+
min: 1
150+
max: 100
151+
- name: insightDetails_insffightContext_attributions_baseline
152+
value: []
153+
- name: bucket_num
154+
range:
155+
min: 63461
156+
max: 63471

‎packages/aws/_dev/benchmark/rally/cloudtrail-benchmark/fields.yml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
- name: timestamp
2+
type: date
3+
- name: aws_Account_Id
4+
type: keyword
5+
- name: file_hash_sha256
6+
type: keyword
7+
- name: file_path
8+
type: keyword
9+
- name: digest_start_time
10+
type: date
11+
- name: digest_end_time
12+
type: date
13+
- name: digest_previous_s3_bucket
14+
type: keyword
15+
- name: digest_s3_bucket
16+
type: keyword
17+
- name: digest_s3_object
18+
type: keyword
19+
- name: digest_public_Key_fingerprint
20+
type: keyword
21+
- name: digest_signature_algorithm
22+
type: keyword
23+
- name: newestEventTime
24+
type: date
25+
- name: oldestEventTime
26+
type: date
27+
- name: previous_digest_s3_object
28+
type: keyword
29+
- name: previous_digest_hash_value
30+
type: keyword
31+
- name: digest_previous_hash_algorithm
32+
type: keyword
33+
- name: logFiles
34+
type: keyword
35+
- name: previous_digest_signature
36+
type: keyword
37+
- name: eventCategory
38+
type: keyword
39+
- name: eventId
40+
type: keyword
41+
- name: userIdentity_type
42+
type: keyword
43+
- name: userIdentity_principalId
44+
type: keyword
45+
- name: userIdentity_arn
46+
type: keyword
47+
- name: userIdentity_accountId
48+
type: long
49+
- name: userIdentity_accesskeyId
50+
type: keyword
51+
example: example_key
52+
- name: userIdentity_userName
53+
type: keyword
54+
- name: userIdentity_eventTime
55+
type: date
56+
- name: userIdentity_eventSource
57+
type: keyword
58+
- name: userIdentity_eventName
59+
type: keyword
60+
- name: userIdentity_awsregion
61+
type: keyword
62+
- name: userIdentity_sourceIPAddress
63+
type: ip
64+
- name: userIdentity_useragent
65+
type: keyword
66+
- name: userIdentity_requestparameters_username
67+
type: keyword
68+
- name: userIdentity_requestparameters_newusername
69+
type: keyword
70+
- name: requestID
71+
type: keyword
72+
- name: eventType
73+
type: keyword
74+
- name: recipientAccountId
75+
type: keyword
76+
- name: userIdentity_sessionContext_mfaauthenticated
77+
type: boolean
78+
- name: userIdentity_sessionContext_creationDate
79+
type: date
80+
- name: sharedEventId
81+
type: keyword
82+
- name: vpcEndpointId
83+
type: keyword
84+
- name: invokedBy
85+
type: keyword
86+
- name: userIdentity_sessionContext_sessionIssuer_type
87+
type: keyword
88+
- name: userIdentity_sessionContext_sessionIssuer_principalId
89+
type: keyword
90+
- name: userIdentity_sessionContext_sessionIssuer_arn
91+
type: keyword
92+
- name: userIdentity_sessionContext_sessionIssuer_accountId
93+
type: keyword
94+
- name: userIdentity_sessionContext_sessionIssuer_userName
95+
type: keyword
96+
- name: errorMessage
97+
type: keyword
98+
- name: errorCode
99+
type: keyword
100+
- name: apiVersion
101+
type: keyword
102+
- name: responseElements_ConsoleLogin
103+
type: keyword
104+
- name: additionalEventData_MobileVersion
105+
type: boolean
106+
- name: additionalEventData_LoginTo
107+
type: keyword
108+
- name: additionalEventData_MFAUsed
109+
type: boolean
110+
- name: resources_type
111+
type: keyword
112+
- name: resources_ARN
113+
type: keyword
114+
- name: resources_accountId
115+
type: keyword
116+
- name: readOnly
117+
type: boolean
118+
- name: managementEvent
119+
type: keyword
120+
- name: insightDetails_state
121+
type: boolean
122+
- name: insightDetails_eventSource
123+
type: "iam.amazonaws.com"
124+
- name: insightDetails_eventName
125+
type: "AttachUserPolicy"
126+
- name: insightDetails_insightType
127+
type: "ApiCallRateInsight"
128+
- name: insightDetails_insffightContext_statistics_baseline_average
129+
type: float
130+
- name: insightDetails_insffightContext_statistics_insight_average
131+
type: float
132+
- name: insightDetails_insffightContext_statistics_insightDuration
133+
type: long
134+
- name: insightDetails_insffightContext_statistics_baselineDuration
135+
type: long
136+
- name: insightDetails_insffightContext_attributions_attribute
137+
type: keyword
138+
- name: insightDetails_insffightContext_attributions_insight_value
139+
type: keyword
140+
- name: insightDetails_insffightContext_attributions_insight_average
141+
type: float
142+
- name: insightDetails_insffightContext_attributions_baseline
143+
type: keyword
144+
- name: file_name
145+
type: keyword
146+
example: extra-samples
147+
- name: bucket_num
148+
type: long

0 commit comments

Comments
 (0)