diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 794a51de0819..fd4fb472739a 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -120,7 +120,7 @@ characters. This only applies to non-JSON logs. See <<_encoding_3>>. ==== `decoding` The file decoding option is used to specify a codec that will be used to -decode the file contents. This can apply to any file stream data. +decode the file contents. This can apply to any file stream data. An example config is shown below: [source,yaml] @@ -131,17 +131,17 @@ An example config is shown below: Currently supported codecs are given below:- 1. <>: This codec decodes parquet compressed data streams. - + [id="attrib-decoding-parquet"] [float] ==== `the parquet codec` The `parquet` codec is used to decode parquet compressed data streams. Only enabling the codec will use the default codec options. The parquet codec supports -two sub attributes which can make parquet decoding more efficient. The `batch_size` attribute and +two sub attributes which can make parquet decoding more efficient. The `batch_size` attribute and the `process_parallel` attribute. The `batch_size` attribute can be used to specify the number of -records to read from the parquet stream at a time. By default the `batch size` is set to `1` and -`process_parallel` is set to `false`. If the `process_parallel` attribute is set to `true` then functions -which read multiple columns will read those columns in parallel from the parquet stream with a +records to read from the parquet stream at a time. By default the `batch size` is set to `1` and +`process_parallel` is set to `false`. If the `process_parallel` attribute is set to `true` then functions +which read multiple columns will read those columns in parallel from the parquet stream with a number of readers equal to the number of columns. Setting `process_parallel` to `true` will greatly increase the rate of processing at the cost of increased memory usage. Having a larger `batch_size` also helps to increase the rate of processing. An example config is shown below: @@ -265,8 +265,9 @@ The default is `10 MiB`. [float] ==== `max_number_of_messages` -The maximum number of SQS messages that can be inflight at any time. Defaults -to 5. +The maximum number of SQS messages to receive during a single `ReceiveMessage` call. +As many goroutine workers will be started to process the SQS messages received. +Defaults to 5. [id="input-{type}-parsers"] [float] diff --git a/x-pack/filebeat/input/awss3/benchmarks-TO-BE-DELETED.md b/x-pack/filebeat/input/awss3/benchmarks-TO-BE-DELETED.md new file mode 100644 index 000000000000..92cbe2bab9ab --- /dev/null +++ b/x-pack/filebeat/input/awss3/benchmarks-TO-BE-DELETED.md @@ -0,0 +1,237 @@ +## DELETION ON ACKER + +### DYNAMIC NUMBER OF MESSAGES (124.20s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 534 | 49517.80415430267 | 386 kB | 36 MB | 8 | 741.839762611276 | 2 | 0.19998500028504163 | 10.000750042 | 0.010784 | 9.989966042 | 99,89 | 12 | +| 2 | 1096 | 66849.9877126551 | 806 kB | 49 MB | 16 | 975.9122293818261 | 3 | 0.29995790216825385 | 10.001403458 | 0.016394917 | 9.985008541000001 | 99,84 | 12 | +| 4 | 1569 | 94558.18546638881 | 1.2 MB | 70 MB | 21 | 1265.5971286132344 | 5 | 0.49993063257501463 | 10.001387541 | 0.016592958 | 9.984794583 | 99,83 | 12 | +| 8 | 3394 | 125587.03334972182 | 2.5 MB | 91 MB | 41 | 1517.1091241421905 | 9 | 0.8999142269252461 | 10.000953125 | 0.027025083 | 9.973928042 | 99,73 | 12 | +| 16 | 5956 | 272461.62721531175 | 4.3 MB | 199 MB | 72 | 3293.6932772838227 | 18 | 1.7997335643835817 | 10.001480417 | 0.021859959 | 9.979620458 | 99,78 | 12 | +| 32 | 12800 | 266282.4986867904 | 9.3 MB | 194 MB | 161 | 3349.3345537947857 | 36 | 3.5995028334290944 | 10.001381209 | 0.04806925 | 9.953311959 | 99,52 | 12 | +| 64 | 27343 | 186554.56881472556 | 20 MB | 135 MB | 344 | 2347.027453910163 | 71 | 7.099137336967879 | 10.001215166 | 0.146568375 | 9.854646791 | 98,53 | 12 | +| 128 | 55831 | 129989.33761940138 | 41 MB | 94 MB | 681 | 1585.5481527970544 | 141 | 14.098866039483537 | 10.000804292 | 0.429504458 | 9.571299834 | 95,71 | 12 | +| 256 | 109313 | 83960.9816045163 | 79 MB | 61 MB | 1357 | 1042.2827297515266 | 282 | 28.19473134641538 | 10.001868666 | 1.30195 | 8.699918666 | 86,98 | 12 | +| 512 | 219611 | 46694.95705637275 | 160 MB | 34 MB | 2712 | 576.6410768899686 | 564 | 56.389861570925405 | 10.001797917 | 4.703098875 | 5.298699041999999 | 52,98 | 12 | +| 1024 | 448582 | 23887.37921413506 | 326 MB | 17 MB | 5565 | 296.34105988796165 | 1127 | 56.34529821869664 | 20.001668917 | 18.779037917 | 1.2226309999999998 | 6,11 | 12 | + +### 71 MESSAGES (111.50s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 26140 | 73290.30170185407 | 19 MB | 53 MB | 329 | 922.4372325902827 | 71 | 7.098849690328135 | 10.001620417 | 0.356663834 | 9.644956582999999 | 96,43 | 12 | +| 2 | 26140 | 141016.09063185065 | 19 MB | 102 MB | 329 | 1774.8390902019457 | 71 | 7.0995195101991655 | 10.000676792 | 0.185368917 | 9.815307875 | 98,15 | 12 | +| 4 | 26140 | 253372.72881857055 | 19 MB | 184 MB | 329 | 3188.9681630187342 | 71 | 7.099005488252842 | 10.001400917 | 0.103168167 | 9.89823275 | 98,97 | 12 | +| 8 | 26140 | 379671.6865344571 | 19 MB | 276 MB | 329 | 4778.576314836892 | 71 | 7.098718711087268 | 10.001804958 | 0.068848958 | 9.932955999999999 | 99,31 | 12 | +| 16 | 26140 | 294557.1993402324 | 19 MB | 214 MB | 329 | 3707.3189970518924 | 71 | 7.0989061472990285 | 10.001540875 | 0.088743375 | 9.9127975 | 99,11 | 12 | +| 32 | 26140 | 227728.69946491945 | 19 MB | 165 MB | 329 | 2866.2104867619933 | 71 | 7.098394106876501 | 10.002262333 | 0.114785708 | 9.887476625 | 98,85 | 12 | +| 64 | 26140 | 192476.25771489978 | 19 MB | 140 MB | 329 | 2422.5206116374147 | 71 | 7.098927944038247 | 10.001510166 | 0.135808958 | 9.865701207999999 | 98,64 | 12 | +| 128 | 26140 | 177702.24337185588 | 19 MB | 129 MB | 329 | 2236.5737593473827 | 71 | 7.099228284778609 | 10.001087041 | 0.1471 | 9.853987041 | 98,53 | 12 | +| 256 | 26140 | 136009.97141443985 | 19 MB | 99 MB | 329 | 1711.8316983684283 | 71 | 7.099239198672492 | 10.001071666 | 0.192191791 | 9.808879874999999 | 98,08 | 12 | +| 512 | 26140 | 116130.85213918249 | 19 MB | 84 MB | 329 | 1461.6316126163367 | 71 | 7.0980791412464574 | 10.002706167 | 0.225090917 | 9.77761525 | 97,75 | 12 | +| 1024 | 26140 | 87955.27538177604 | 19 MB | 64 MB | 329 | 1107.0116909182982 | 71 | 7.098742930466147 | 10.001770834 | 0.2971965 | 9.704574334 | 97,03 | 12 | + +### 1 MESSAGE : 1 OBJECT : 1 EVENT (110.12s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 2 | 357.9311365548448 | 1.2 kB | 212 kB | 2 | 357.9311365548448 | 2 | 0.1999996550005951 | 10.00001725 | 0.005587667 | 9.994429583 | 99,94 | 12 | +| 2 | 3 | 528.5334974840924 | 2.6 kB | 457 kB | 3 | 528.5334974840924 | 3 | 0.2999928589399865 | 10.000238041 | 0.005676083 | 9.994561957999998 | 99,94 | 12 | +| 4 | 5 | 889.6797153024911 | 3.8 kB | 678 kB | 5 | 889.6797153024911 | 5 | 0.5000020625085079 | 9.99995875 | 0.00562 | 9.994338749999999 | 99,94 | 12 | +| 8 | 9 | 1387.5238480661387 | 7.4 kB | 1.1 MB | 9 | 1387.5238480661387 | 9 | 0.9000190428729149 | 9.999788417 | 0.006486375 | 9.993302042 | 99,94 | 12 | +| 16 | 18 | 3181.7579212514916 | 16 kB | 2.8 MB | 18 | 3181.7579212514916 | 18 | 1.7999130942560961 | 10.000482833 | 0.00565725 | 9.994825582999999 | 99,94 | 12 | +| 32 | 36 | 6485.805184788696 | 28 kB | 5.1 MB | 36 | 6485.805184788696 | 36 | 3.5998180144401157 | 10.000505541 | 0.005550583 | 9.994954958000001 | 99,94 | 12 | +| 64 | 71 | 6472.688990211288 | 54 kB | 4.9 MB | 71 | 6472.688990211288 | 71 | 7.099864422408963 | 10.000190958 | 0.010969166 | 9.989221791999999 | 99,89 | 12 | +| 128 | 141 | 17653.503788554593 | 108 kB | 14 MB | 141 | 17653.503788554593 | 141 | 14.099971330528293 | 10.000020333 | 0.007987083 | 9.99203325 | 99,92 | 12 | +| 256 | 282 | 17596.772577189615 | 216 kB | 14 MB | 282 | 17596.772577189615 | 282 | 28.19464947881269 | 10.001897708 | 0.016025666 | 9.985872041999999 | 99,84 | 12 | +| 512 | 564 | 29950.81216353462 | 426 kB | 23 MB | 564 | 29950.81216353462 | 564 | 56.39437794859186 | 10.000996917 | 0.018830875 | 9.982166042 | 99,81 | 12 | +| 1024 | 1127 | 52982.10257993222 | 843 kB | 40 MB | 1127 | 52982.10257993222 | 1127 | 112.65294204978225 | 10.00417725 | 0.021271334 | 9.982905916 | 99,79 | 12 | + + +## UNCAPPED DELETION + +### DYNAMIC NUMBER OF MESSAGES (236.53s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|---------------------:|------------------:|-----:| +| 1 | 534 | 33476.651853352705 | 386 kB | 24 MB | 8 | 501.5228742075312 | 2 | 0.19999470432022484 | 10.000264791 | 0.015951416 | 9.984313375 | 99,84 | 10 | +| 2 | 1096 | 104412.43521618043 | 806 kB | 77 MB | 16 | 1524.2691272435097 | 3 | 0.29997075660086586 | 10.000974875 | 0.010496834 | 9.990478041000001 | 99,90 | 10 | +| 4 | 1569 | 30350.754911676788 | 1.2 MB | 22 MB | 21 | 406.224253119957 | 5 | 0.49994750756149087 | 10.001049959 | 0.051695584 | 9.949354374999999 | 99,48 | 10 | +| 8 | 3394 | 10774.6003355815 | 2.5 MB | 7.8 MB | 41 | 130.15869586294681 | 9 | 0.8999113850059893 | 10.000984708 | 0.315000083 | 9.685984625000001 | 96,85 | 10 | +| 16 | 5956 | 52104.57380964997 | 4.3 MB | 38 MB | 72 | 629.8739614329747 | 18 | 1.79256181708282 | 10.041494708 | 0.114308583 | 9.927186125 | 98,86 | 10 | +| 32 | 12800 | 41893.11600502637 | 9.3 MB | 31 MB | 161 | 526.9368497507223 | 36 | 3.580617015831019 | 10.054133084 | 0.305539459 | 9.748593625 | 96,96 | 10 | +| 64 | 27343 | 24680.776188829113 | 20 MB | 18 MB | 344 | 310.50678451366764 | 71 | 7.069355492709372 | 10.043348375 | 1.107866292 | 8.935482083 | 88,97 | 10 | +| 128 | 55831 | 41824.388657950345 | 41 MB | 30 MB | 681 | 510.1540125748095 | 141 | 14.051082386142353 | 10.034814125 | 1.334891 | 8.699923125 | 86,70 | 10 | +| 256 | 109313 | 22552.537371923707 | 79 MB | 16 MB | 1357 | 279.9648094343808 | 282 | 28.10006119662218 | 10.035565333 | 4.84703775 | 5.188527582999999 | 51,70 | 10 | +| 512 | 219611 | 63324.72009374242 | 160 MB | 46 MB | 2712 | 782.0038199098836 | 564 | 56.15162754927949 | 10.044232458 | 3.46801375 | 6.576218708 | 65,47 | 10 | +| 1024 | 448582 | 55551.67442589024 | 326 MB | 40 MB | 5565 | 689.1606622202389 | 1127 | 138.58965301839567 | 8.131920208 | 8.075040125 | 0.056880083000001136 | 0,70 | 10 | + +### 71 MESSAGES (111.73s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 26140 | 91748.08364349176 | 19 MB | 67 MB | 329 | 1154.7482600883238 | 71 | 7.0997052436673105 | 10.000415167 | 0.284910583 | 9.715504584 | 97,15 | 10 | +| 2 | 26140 | 61259.79814029408 | 19 MB | 44 MB | 329 | 771.0204127068382 | 71 | 7.086361323258707 | 10.019246375 | 0.42670725 | 9.592539125 | 95,74 | 10 | +| 4 | 26140 | 28801.399292295628 | 19 MB | 21 MB | 329 | 362.49657104687304 | 71 | 7.099345381431265 | 10.000922083 | 0.907594792 | 9.093327291000001 | 90,92 | 10 | +| 8 | 26140 | 51415.166866842264 | 19 MB | 37 MB | 329 | 647.1151453401341 | 71 | 7.096252912352794 | 10.005280375 | 0.508410292 | 9.496870083 | 94,92 | 10 | +| 16 | 26140 | 43575.091184295634 | 19 MB | 32 MB | 329 | 548.4393649438891 | 71 | 7.083790399028503 | 10.022882666 | 0.599884 | 9.422998666 | 94,01 | 10 | +| 32 | 26140 | 32072.511182303715 | 19 MB | 23 MB | 329 | 403.6670305653375 | 71 | 7.099737930923625 | 10.000369125 | 0.815028167 | 9.185340958000001 | 91,85 | 10 | +| 64 | 26140 | 27670.26601536393 | 19 MB | 20 MB | 329 | 348.260042810051 | 71 | 7.099000579271748 | 10.001407833 | 0.944696375 | 9.056711458 | 90,55 | 10 | +| 128 | 26140 | 21984.360450447064 | 19 MB | 16 MB | 329 | 276.6968090358487 | 71 | 7.099141299227072 | 10.001209584 | 1.1890270840000001 | 8.812182499999999 | 88,11 | 10 | +| 256 | 26140 | 23343.357775469092 | 19 MB | 17 MB | 329 | 293.8012512673807 | 71 | 7.095628413136305 | 10.006160958 | 1.119804625 | 8.886356333 | 88,81 | 10 | +| 512 | 26140 | 20567.43918692126 | 19 MB | 15 MB | 329 | 258.86333177112067 | 71 | 7.057589123978741 | 10.060092583 | 1.270940916 | 8.789151666999999 | 87,37 | 10 | +| 1024 | 26140 | 47420.73058881012 | 19 MB | 34 MB | 329 | 596.8408708385052 | 71 | 7.075070928859076 | 10.035235083 | 0.551235708 | 9.483999375 | 94,51 | 10 | + +### 1 MESSAGE : 1 OBJECT : 1 EVENT (112.92s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 2 | 395.3024627145776 | 1.2 kB | 234 kB | 2 | 395.3024627145776 | 2 | 0.2000003366605667 | 9.999983167 | 0.005059417 | 9.99492375 | 99,95 | 10 | +| 2 | 3 | 572.5828985520524 | 2.6 kB | 495 kB | 3 | 572.5828985520524 | 3 | 0.29999898252345086 | 10.000033916 | 0.005239416 | 9.9947945 | 99,95 | 10 | +| 4 | 5 | 436.0227712020215 | 3.8 kB | 332 kB | 5 | 436.0227712020215 | 5 | 0.4999889960921773 | 10.000220083 | 0.011467291 | 9.988752792 | 99,89 | 10 | +| 8 | 9 | 609.3467288507261 | 7.4 kB | 498 kB | 9 | 609.3467288507261 | 9 | 0.8996953406652672 | 10.00338625 | 0.014769916 | 9.988616334 | 99,85 | 10 | +| 16 | 18 | 96.57043097741648 | 16 kB | 84 kB | 18 | 96.57043097741648 | 18 | 1.7894709097457604 | 10.058839125 | 0.186392458 | 9.872446667 | 98,15 | 10 | +| 32 | 36 | 94.81145422867225 | 28 kB | 75 kB | 36 | 94.81145422867225 | 36 | 3.5984909278482706 | 10.004193625 | 0.379700958 | 9.624492666999998 | 96,20 | 10 | +| 64 | 71 | 32.996492205653425 | 54 kB | 25 kB | 71 | 32.996492205653425 | 71 | 6.894487619016878 | 10.298082167 | 2.151743875 | 8.146338292 | 79,11 | 10 | +| 128 | 141 | 90.3410929260777 | 108 kB | 70 kB | 141 | 90.3410929260777 | 141 | 13.614254133733517 | 10.356792125 | 1.5607515410000001 | 8.796040584 | 84,93 | 10 | +| 256 | 282 | 84.55657598731302 | 216 kB | 65 kB | 282 | 84.55657598731302 | 282 | 26.45563037584542 | 10.659356666 | 3.335045166 | 7.3243115 | 68,71 | 10 | +| 512 | 564 | 117.41881487011658 | 426 kB | 89 kB | 564 | 117.41881487011658 | 564 | 54.02426795979435 | 10.43975275 | 4.803318792 | 5.6364339580000005 | 53,99 | 10 | +| 1024 | 1127 | 173.7517371943818 | 843 kB | 130 kB | 1127 | 173.7517371943818 | 1127 | 102.50634920938927 | 10.994440917 | 6.486266084 | 4.508174833 | 41,00 | 10 | + +## CAPPED DELETION + +### DYNAMIC NUMBER OF MESSAGES (129.09s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 534 | 34795.169179455625 | 386 kB | 25 MB | 8 | 521.2759427633802 | 2 | 0.19999374353572283 | 10.000312833 | 0.015346958 | 9.984965875 | 99,85 | 10 | +| 2 | 1096 | 69790.21164263744 | 806 kB | 51 MB | 16 | 1018.8352064618604 | 3 | 0.29998010882894977 | 10.000663083 | 0.015704208 | 9.984958874999998 | 99,84 | 10 | +| 4 | 1569 | 79595.34517898757 | 1.2 MB | 59 MB | 21 | 1065.3296677875965 | 5 | 0.499988644007923 | 10.000227125 | 0.019712208 | 9.980514917 | 99,80 | 10 | +| 8 | 3394 | 30285.38667630374 | 2.5 MB | 22 MB | 41 | 365.85175419223725 | 9 | 0.8999114412449165 | 10.000984083 | 0.11206725 | 9.888916833 | 98,88 | 10 | +| 16 | 5956 | 20495.25031176799 | 4.3 MB | 15 MB | 72 | 247.75990974602007 | 18 | 1.7947778397347496 | 10.029096416 | 0.290603916 | 9.7384925 | 97,10 | 10 | +| 32 | 12800 | 17914.975587444715 | 9.3 MB | 13 MB | 161 | 225.33680231082806 | 36 | 3.5967396156040063 | 10.009064833 | 0.714486042 | 9.294578791 | 92,86 | 10 | +| 64 | 27343 | 23536.6331404344 | 20 MB | 17 MB | 344 | 296.11241635187923 | 71 | 7.096650351937619 | 10.004720041 | 1.161720958 | 8.842999083 | 88,39 | 10 | +| 128 | 55831 | 30150.36774297852 | 41 MB | 22 MB | 681 | 367.7598544351413 | 141 | 13.982909330758774 | 10.083738417 | 1.8517518750000002 | 8.231986542 | 81,64 | 10 | +| 256 | 109313 | 42706.806371763625 | 79 MB | 31 MB | 1357 | 530.1577694005584 | 282 | 28.126771012235494 | 10.026035334 | 2.559615417 | 7.466419917 | 74,47 | 10 | +| 512 | 219611 | 27256.798510843055 | 160 MB | 20 MB | 2712 | 336.5971538830312 | 564 | 56.21467240679431 | 10.032967833 | 8.057109125 | 1.9758587080000005 | 19,69 | 10 | +| 1024 | 448582 | 40054.32150041598 | 326 MB | 29 MB | 5565 | 496.9042430365349 | 1127 | 99.99467493309047 | 11.270600167 | 11.199340875 | 0.07125929199999881 | 0,63 | 10 | + +### 71 MESSAGES : 329 OBJECTS : 26140 EVENTS (105.90s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 26140 | 109625.84677526618 | 19 MB | 80 MB | 329 | 1379.75912735511 | 71 | 7.099808630338217 | 10.000269542 | 0.238447417 | 9.761822125 | 97,62 | 10 | +| 2 | 26140 | 72137.42531654994 | 19 MB | 52 MB | 329 | 907.927043961168 | 71 | 7.099324055608355 | 10.000952125 | 0.362363917 | 9.638588208 | 96,38 | 10 | +| 4 | 26140 | 47008.762487278 | 19 MB | 34 MB | 329 | 591.6558094228945 | 71 | 7.099316424569769 | 10.000962875 | 0.556066542 | 9.444896333 | 94,44 | 10 | +| 8 | 26140 | 48051.90120040138 | 19 MB | 35 MB | 329 | 604.784831481716 | 71 | 7.099304356914327 | 10.000979875 | 0.543995125 | 9.45698475 | 94,56 | 10 | +| 16 | 26140 | 79621.05373773864 | 19 MB | 58 MB | 329 | 1002.1165523992354 | 71 | 7.099496231816533 | 10.000709583 | 0.328305125 | 9.672404458 | 96,72 | 10 | +| 32 | 26140 | 43767.73080754615 | 19 MB | 32 MB | 329 | 550.8639416864071 | 71 | 7.091037578748608 | 10.012639083 | 0.597243666 | 9.415395417 | 94,04 | 10 | +| 64 | 26140 | 28775.26650170689 | 19 MB | 21 MB | 329 | 362.1676617850638 | 71 | 7.096071378585638 | 10.005536333 | 0.908419041 | 9.097117292 | 90,92 | 10 | +| 128 | 26140 | 26747.535715972164 | 19 MB | 19 MB | 329 | 336.64649007478357 | 71 | 7.08464309100885 | 10.021676334 | 0.977286292 | 9.044390042 | 90,25 | 10 | +| 256 | 26140 | 48014.57333039672 | 19 MB | 35 MB | 329 | 604.315020110961 | 71 | 7.068498969274716 | 10.044565375 | 0.544418042 | 9.500147333 | 94,58 | 10 | +| 512 | 26140 | 26388.236329374038 | 19 MB | 19 MB | 329 | 332.1243210544781 | 71 | 68.50246301522354 | 1.036459083 | 0.990592917 | 0.04586616600000004 | 4,43 | 10 | +| 1024 | 26140 | 24969.348500847864 | 19 MB | 18 MB | 329 | 314.2660924552007 | 71 | 7.077267345233379 | 10.032120667 | 1.046883542 | 8.985237125 | 89,56 | 10 | + +### 1 MESSAGE : 1 OBJECT : 1 EVENT (123.52s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 2 | 395.83055839618953 | 1.2 kB | 234 kB | 2 | 395.83055839618953 | 2 | 0.2000018583572673 | 9.999907083 | 0.005052667 | 9.994854416 | 99,95 | 10 | +| 2 | 3 | 536.1371639023137 | 2.6 kB | 464 kB | 3 | 536.1371639023137 | 3 | 0.29999378511875185 | 10.000207167 | 0.005595583 | 9.994611584 | 99,94 | 10 | +| 4 | 5 | 491.4427530623027 | 3.8 kB | 375 kB | 5 | 491.4427530623027 | 5 | 0.5000005354505734 | 9.999989291 | 0.010174125 | 9.989815166 | 99,90 | 10 | +| 8 | 9 | 127.73076126724754 | 7.4 kB | 104 kB | 9 | 127.73076126724754 | 9 | 0.8959297054285792 | 10.045430959 | 0.070460709 | 9.97497025 | 99,30 | 10 | +| 16 | 18 | 90.40842003751949 | 16 kB | 79 kB | 18 | 90.40842003751949 | 18 | 1.795549797160854 | 10.024784625 | 0.1990965 | 9.825688125000001 | 98,01 | 10 | +| 32 | 36 | 41.16429983834505 | 28 kB | 32 kB | 36 | 41.16429983834505 | 36 | 3.5983315587274998 | 10.004636708 | 0.874544208 | 9.1300925 | 91,26 | 10 | +| 64 | 71 | 52.876211464847614 | 54 kB | 40 kB | 71 | 52.876211464847614 | 71 | 6.92309304923333 | 10.255531667 | 1.342758833 | 8.912772834 | 86,91 | 10 | +| 128 | 141 | 55.888284984832495 | 108 kB | 43 kB | 141 | 55.888284984832495 | 141 | 13.692525170393644 | 10.29758925 | 2.522890084 | 7.7746991659999996 | 75,50 | 10 | +| 256 | 282 | 52.68909681601697 | 216 kB | 40 kB | 282 | 52.68909681601697 | 282 | 26.954578632512252 | 10.462044458 | 5.352150958 | 5.109893499999999 | 48,84 | 10 | +| 512 | 564 | 91.11824318695749 | 426 kB | 69 kB | 564 | 91.11824318695749 | 564 | 46.264973271827536 | 12.190647916 | 6.189759375 | 6.000888540999999 | 49,23 | 10 | +| 1024 | 1127 | 78.80667488058174 | 843 kB | 59 kB | 1127 | 78.80667488058174 | 1127 | 55.92493299201936 | 20.152013417 | 14.300819083 | 5.851194333999999 | 29,04 | 10 | + + +## CURRENT IMPLEMENTATION (acker with mutex) + +### DYNAMIC NUMBER OF MESSAGES (420.15s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 534 | 53.377307971948426 | 386 kB | 39 kB | 8 | 0.799660044523572 | 2 | 0.09999887376268425 | 20.00022525 | 10.00425125 | 9.995974 | 49,98 | 10 | +| 2 | 1096 | 109.54472373240463 | 806 kB | 80 kB | 16 | 1.5991930471883886 | 3 | 0.1499952010910351 | 20.000639875 | 10.005046 | 9.995593875 | 49,98 | 10 | +| 4 | 1569 | 156.58283429763634 | 1.2 MB | 115 kB | 21 | 2.0957549523584214 | 5 | 0.2499882995101315 | 20.000936083 | 10.020255458 | 9.980680625 | 49,90 | 10 | +| 8 | 3394 | 327.7503948980639 | 2.5 MB | 238 kB | 41 | 3.9592711228110256 | 9 | 0.4499758991108533 | 20.001071208 | 10.355441375 | 9.645629833 | 48,23 | 10 | +| 16 | 5956 | 6995.8233255091345 | 4.3 MB | 5.1 MB | 72 | 84.5700603486665 | 18 | 1.68781488955836 | 10.664676625 | 0.851365125 | 9.813311500000001 | 92,02 | 10 | +| 32 | 12800 | 9577.580684203449 | 9.3 MB | 7.0 MB | 161 | 120.4680070434965 | 36 | 3.5207708469266525 | 10.225033541 | 1.336454416 | 8.888579125 | 86,93 | 10 | +| 64 | 27343 | 5969.565759122024 | 20 MB | 4.3 MB | 344 | 75.10260838744748 | 71 | 7.09735159344403 | 10.003731542 | 4.580400167 | 5.423331375000001 | 54,21 | 10 | +| 128 | 55831 | 6528.538457378312 | 41 MB | 4.7 MB | 681 | 79.63200891036576 | 141 | 13.948629932462296 | 10.108519667 | 8.5518375 | 1.556682167 | 15,40 | 10 | +| 256 | 109313 | 2418.262154430809 | 79 MB | 1.8 MB | 1357 | 30.0200501638653 | 282 | 5.63602730525321 | 50.03524375 | 45.203122333 | 4.832121416999996 | 9,66 | 10 | +| 512 | 219611 | 3242.754954766131 | 160 MB | 2.4 MB | 2712 | 40.04513178905313 | 564 | 7.980402861454806 | 70.673123875 | 67.723587833 | 2.9495360420000054 | 4,17 | 10 | +| 1024 | 448582 | 2735.7629115360114 | 326 MB | 2.0 MB | 5565 | 33.939214241092834 | 1127 | 6.612725338373049 | 170.428974792 | 163.969618167 | 6.459356624999998 | 3,79 | 10 | + +### 71 MESSAGES : 329 OBJECTS : 26140 EVENTS (113.31s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|---------------------:|------------------:|-----:| +| 1 | 26140 | 9877.562063511137 | 19 MB | 7.2 MB | 329 | 124.31973675957016 | 71 | 7.098897157221147 | 10.001553541 | 2.646402 | 7.355151541 | 73,54 | 10 | +| 2 | 26140 | 12573.780468960163 | 19 MB | 9.1 MB | 329 | 158.25454377535937 | 71 | 18.855849203387365 | 3.765409833 | 2.07892925 | 1.6864805830000003 | 44,79 | 10 | +| 4 | 26140 | 7938.4983023008135 | 19 MB | 5.8 MB | 329 | 99.91453486828492 | 71 | 6.85461611092427 | 10.357983416 | 3.2928142080000002 | 7.0651692079999995 | 68,21 | 10 | +| 8 | 26140 | 5360.695072382419 | 19 MB | 3.9 MB | 329 | 67.47011013059739 | 71 | 7.098512566319234 | 10.002095416 | 4.876233333 | 5.1258620829999995 | 51,25 | 10 | +| 16 | 26140 | 2351.529242916884 | 19 MB | 1.7 MB | 329 | 29.5965233710656 | 71 | 3.5123938673326833 | 20.214133916 | 11.1161705 | 9.097963416 | 45,01 | 10 | +| 32 | 26140 | 9500.726928457989 | 19 MB | 6.9 MB | 329 | 119.57686149436412 | 71 | 6.861171234604384 | 10.348087458 | 2.751368416 | 7.596719042 | 73,41 | 10 | +| 64 | 26140 | 6361.638976257994 | 19 MB | 4.6 MB | 329 | 80.06806515642234 | 71 | 6.87722113211347 | 10.323937334 | 4.109004 | 6.214933334 | 60,20 | 10 | +| 128 | 26140 | 6578.852782344765 | 19 MB | 4.8 MB | 329 | 82.80193440671108 | 71 | 17.592678026519042 | 4.035769875 | 3.973337125 | 0.062432749999999704 | 1,55 | 10 | +| 256 | 26140 | 7012.815402100417 | 19 MB | 5.1 MB | 329 | 88.26382047785145 | 71 | 7.000925642568938 | 10.141516083 | 3.727461583 | 6.414054500000001 | 63,25 | 10 | +| 512 | 26140 | 6679.284014737735 | 19 MB | 4.8 MB | 329 | 84.0659694280304 | 71 | 6.996431130334545 | 10.148031 | 3.9135931250000002 | 6.234437874999999 | 61,43 | 10 | +| 1024 | 26140 | 3193.499741765352 | 19 MB | 2.3 MB | 329 | 40.19362720125481 | 71 | 6.531022966383471 | 10.871191292 | 8.185377208 | 2.6858140840000004 | 24,71 | 10 | + +### 1 MESSAGE : 1 OBJECT : 1 EVENT (900.27s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|--------------------:|---------------:|-----------------:|-----------------:|--------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 2 | 0.19998794072717416 | 1.2 kB | 118 B | 2 | 0.19998794072717416 | 2 | 0.09999999729500009 | 20.000000541 | 10.000603 | 9.999397540999999 | 50,00 | 10 | +| 2 | 3 | 0.2999579983812767 | 2.6 kB | 259 B | 3 | 0.2999579983812767 | 3 | 0.14999602135553355 | 20.0005305 | 10.00140025 | 9.99913025 | 49,99 | 10 | +| 4 | 5 | 0.49973281994669333 | 3.8 kB | 380 B | 5 | 0.49973281994669333 | 5 | 0.2500007073020011 | 19.999943416 | 10.005346458 | 9.994596958 | 49,97 | 10 | +| 8 | 9 | 0.8990750353797166 | 7.4 kB | 734 B | 9 | 0.8990750353797166 | 9 | 0.4499962050245043 | 20.000168667 | 10.010287958 | 9.989880709000001 | 49,95 | 10 | +| 16 | 18 | 1.7925517681481675 | 16 kB | 1.6 kB | 18 | 1.7925517681481675 | 18 | 0.8999971687439067 | 20.000062917 | 10.041551 | 9.958511917000001 | 49,79 | 10 | +| 32 | 36 | 3.521597946166357 | 28 kB | 2.8 kB | 36 | 3.521597946166357 | 36 | 1.7986028827815255 | 20.015535583 | 10.222632041 | 9.792903541999998 | 48,93 | 10 | +| 64 | 71 | 6.530635221320813 | 54 kB | 5.0 kB | 71 | 6.530635221320813 | 71 | 3.5422948887154386 | 20.0435035 | 10.87183675 | 9.17166675 | 45,76 | 10 | +| 128 | 141 | 11.55643631263828 | 108 kB | 8.9 kB | 141 | 11.55643631263828 | 141 | 6.994094252240433 | 20.159865583 | 12.200993125 | 7.958872457999998 | 39,48 | 10 | +| 256 | 282 | 14.330504553562285 | 216 kB | 11 kB | 282 | 14.330504553562285 | 282 | 5.574490878106893 | 50.587579416 | 19.67830225 | 30.909277165999995 | 61,10 | 10 | +| 512 | 564 | 14.132980354791805 | 426 kB | 11 kB | 564 | 14.132980354791805 | 564 | 3.96003592248513 | 142.422950458 | 39.906657042 | 102.516293416 | 71,98 | 10 | +| 1024 | 1127 | 10.888147574358264 | 843 kB | 8.1 kB | 1127 | 10.888147574358264 | 1127 | 2.060683835679075 | 546.905828292 | 103.50704675 | 443.39878154200005 | 81,07 | 10 | + + +## CURRENT IMPLEMENTATION (acker with atomic) + +### DYNAMIC NUMBER OF MESSAGES (164.32s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 534 | 53.36880126588998 | 386 kB | 39 kB | 8 | 0.7995326032343067 | 2 | 0.0999987339710285 | 20.000253209 | 10.005845875 | 9.994407334 | 49,97 | 10 | +| 2 | 1096 | 109.5183622748013 | 806 kB | 80 kB | 16 | 1.5988082083912598 | 3 | 0.14999513391036398 | 20.000648833 | 10.00745425 | 9.993194583 | 49,96 | 10 | +| 4 | 1569 | 156.72384371614336 | 1.2 MB | 115 kB | 21 | 2.097642267711288 | 5 | 0.24983862454793027 | 20.012918375 | 10.011239916 | 10.001678459000003 | 49,98 | 10 | +| 8 | 3394 | 338.5693441187711 | 2.5 MB | 246 kB | 41 | 4.089965559478378 | 9 | 0.449976796204042 | 20.001031333 | 10.024534291 | 9.976497042 | 49,88 | 10 | +| 16 | 5956 | 10475.731016772343 | 4.3 MB | 7.6 MB | 72 | 126.63744681121703 | 18 | 1.799447509692922 | 10.003070333 | 0.568552208 | 9.434518125 | 94,32 | 10 | +| 32 | 12800 | 17234.142316989062 | 9.3 MB | 13 MB | 161 | 216.77319633087802 | 36 | 3.5888302145609554 | 10.03112375 | 0.74271175 | 9.288412000000001 | 92,60 | 10 | +| 64 | 27343 | 19069.442066866573 | 20 MB | 14 MB | 344 | 239.9110584428227 | 71 | 7.096113340218469 | 10.005477167 | 1.433864709 | 8.571612458 | 85,67 | 10 | +| 128 | 55831 | 35664.64780353814 | 41 MB | 26 MB | 681 | 435.0204215258453 | 141 | 14.081119155366537 | 10.013408625 | 1.56544375 | 8.447964875 | 84,37 | 10 | +| 256 | 109313 | 25977.965417155614 | 79 MB | 19 MB | 1357 | 322.4877102547745 | 282 | 28.14029976872242 | 10.021215208 | 4.207912292 | 5.813302916 | 58,01 | 10 | +| 512 | 219611 | 29818.010625950345 | 160 MB | 22 MB | 2712 | 368.22583940502676 | 564 | 56.35987951526856 | 10.007118625 | 7.365045333 | 2.642073292 | 26,40 | 10 | +| 1024 | 448582 | 37269.46254020562 | 326 MB | 27 MB | 5565 | 462.35595506784546 | 1127 | 92.49486483283363 | 12.184460208 | 12.036181083 | 0.14827912500000018 | 1,22 | 10 | + +### 71 MESSAGES : 329 OBJECTS : 26140 EVENTS (113.02s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|-------------------:|---------------:|-----------------:|-----------------:|-------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 26140 | 66013.45586060404 | 19 MB | 48 MB | 329 | 830.8503052080614 | 71 | 7.097534700421701 | 10.003473459 | 0.395979875 | 9.607493584 | 96,04 | 10 | +| 2 | 26140 | 81726.08569439051 | 19 MB | 59 MB | 329 | 1028.610642442788 | 71 | 7.099410541622258 | 10.000830292 | 0.319848917 | 9.680981375 | 96,80 | 10 | +| 4 | 26140 | 67420.14255630635 | 19 MB | 49 MB | 329 | 848.5549694347663 | 71 | 7.099278210704895 | 10.001016708 | 0.387717958 | 9.61329875 | 96,12 | 10 | +| 8 | 26140 | 49733.531297140966 | 19 MB | 36 MB | 329 | 625.9499539693717 | 71 | 7.099305332973168 | 10.0009785 | 0.525601125 | 9.475377375 | 94,74 | 10 | +| 16 | 26140 | 37976.62825897801 | 19 MB | 28 MB | 329 | 477.97669078820826 | 71 | 7.0829980599942655 | 10.024003875 | 0.688318084 | 9.335685791 | 93,13 | 10 | +| 32 | 26140 | 30417.46339417117 | 19 MB | 22 MB | 329 | 382.8364750069746 | 71 | 7.096205718410493 | 10.005346916 | 0.85937475 | 9.145972166 | 91,41 | 10 | +| 64 | 26140 | 23190.059236007757 | 19 MB | 17 MB | 329 | 291.8718243552621 | 71 | 7.091070009187293 | 10.012593291 | 1.127207125 | 8.885386166 | 88,74 | 10 | +| 128 | 26140 | 22119.799277562448 | 19 MB | 16 MB | 329 | 278.40145226924426 | 71 | 7.080700459054835 | 10.027256542 | 1.181746709 | 8.845509833 | 88,21 | 10 | +| 256 | 26140 | 28731.619614081766 | 19 MB | 21 MB | 329 | 361.61831878473225 | 71 | 7.087304628867566 | 10.017912834 | 0.909799042 | 9.108113792000001 | 90,92 | 10 | +| 512 | 26140 | 27872.444273062203 | 19 MB | 20 MB | 329 | 350.80467352094354 | 71 | 7.081151420182479 | 10.026617959 | 0.937843834 | 9.088774124999999 | 90,65 | 10 | +| 1024 | 26140 | 19964.594159382417 | 19 MB | 14 MB | 329 | 251.27587905267083 | 71 | 7.0626779549516066 | 10.052844042 | 1.309317875 | 8.743526167 | 86,98 | 10 | + +### 1 MESSAGE : 1 OBJECT : 1 EVENT (220.35s) +| MAX MSGS INFLIGHT | EVENTS TOTAL | EVENTS PER SEC | S3 BYTES TOTAL | S3 BYTES PER SEC | S3 OBJECTS TOTAL | S3 OBJECTS PER SEC | SQS MESSAGES TOTAL | SQS MESSAGES PER SEC | FULL TIME (SEC) | PROCESSING TIME (SEC) | FLUSHING TIME (SEC) | FLUSHING TIME (%) | CPUS | +|------------------:|-------------:|--------------------:|---------------:|-----------------:|-----------------:|--------------------:|-------------------:|---------------------:|----------------:|----------------------:|--------------------:|------------------:|-----:| +| 1 | 2 | 0.19990798735112197 | 1.2 kB | 118 B | 2 | 0.19990798735112197 | 2 | 0.10000075792574448 | 19.999848416 | 10.00460275 | 9.995245665999999 | 49,98 | 10 | +| 2 | 3 | 0.2999677996865716 | 2.6 kB | 259 B | 3 | 0.2999677996865716 | 3 | 0.14999982531020342 | 20.000023292 | 10.001073459 | 9.998949833000001 | 49,99 | 10 | +| 4 | 5 | 0.4997554779905357 | 3.8 kB | 380 B | 5 | 0.4997554779905357 | 5 | 0.24999950312598754 | 20.00003975 | 10.004892833 | 9.995146917 | 49,98 | 10 | +| 8 | 9 | 0.894460453636782 | 7.4 kB | 730 B | 9 | 0.894460453636782 | 9 | 0.4499972034698792 | 20.000124291 | 10.061931708 | 9.938192583 | 49,69 | 10 | +| 16 | 18 | 1.7925670684868178 | 16 kB | 1.6 kB | 18 | 1.7925670684868178 | 18 | 0.9000027825386028 | 19.999938166 | 10.041465291 | 9.958472875 | 49,79 | 10 | +| 32 | 36 | 3.5774289276666726 | 28 kB | 2.8 kB | 36 | 3.5774289276666726 | 36 | 1.7957864302908053 | 20.046927292 | 10.063093 | 9.983834292 | 49,80 | 10 | +| 64 | 71 | 6.927311449043817 | 54 kB | 5.3 kB | 71 | 6.927311449043817 | 71 | 3.5460116087271363 | 20.022495083 | 10.249286541 | 9.773208541999999 | 48,81 | 10 | +| 128 | 141 | 13.838002220126679 | 108 kB | 11 kB | 141 | 13.838002220126679 | 141 | 7.048862151877301 | 20.003228459 | 10.189332084 | 9.813896374999999 | 49,06 | 10 | +| 256 | 282 | 27.005488400390938 | 216 kB | 21 kB | 282 | 27.005488400390938 | 282 | 14.057648405968855 | 20.060254166 | 10.442321791 | 9.617932375 | 47,95 | 10 | +| 512 | 564 | 50.91963451508912 | 426 kB | 38 kB | 564 | 50.91963451508912 | 564 | 28.157323059061508 | 20.030313209 | 11.076277459 | 8.95403575 | 44,70 | 10 | +| 1024 | 1127 | 79.02205520334414 | 843 kB | 59 kB | 1127 | 79.02205520334414 | 1127 | 56.14676718350413 | 20.072393417 | 14.261841167 | 5.810552250000001 | 28,95 | 10 | diff --git a/x-pack/filebeat/input/awss3/decoding_test.go b/x-pack/filebeat/input/awss3/decoding_test.go index 81f2bbc450c2..0df1f9fef81d 100644 --- a/x-pack/filebeat/input/awss3/decoding_test.go +++ b/x-pack/filebeat/input/awss3/decoding_test.go @@ -21,7 +21,7 @@ func TestParquetDecoding(t *testing.T) { name string file string contentType string - numEvents int + numEvents uint64 assertAgainst string config *readerConfig }{ @@ -121,7 +121,7 @@ func readJSONFromFile(t *testing.T, filepath string) []string { var rawMessages []json.RawMessage err = json.Unmarshal(fileBytes, &rawMessages) assert.NoError(t, err) - var data []string + data := make([]string, 0, len(rawMessages)) for _, rawMsg := range rawMessages { data = append(data, string(rawMsg)) diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 0b33ae042f9a..08c66c5c1ddf 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -157,7 +157,7 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { // Create client for publishing events and receive notification of their ACKs. client, err := pipeline.ConnectWith(beat.ClientConfig{ CloseRef: inputContext.Cancelation, - EventListener: awscommon.NewEventACKHandler(), + EventListener: NewEventACKHandler(), Processing: beat.ProcessingConfig{ // This input only produces events with basic types so normalization // is not required. @@ -227,8 +227,8 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, pipeline beat.Pipeline) (*s } in.metrics = newInputMetrics(ctx.ID, nil, in.config.MaxNumberOfMessages) s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), in.metrics, s3API, fileSelectors, in.config.BackupConfig, in.config.MaxNumberOfMessages) - sqsMessageHandler := newSQSS3EventProcessor(log.Named("sqs_s3_event"), in.metrics, sqsAPI, script, in.config.VisibilityTimeout, in.config.SQSMaxReceiveCount, pipeline, s3EventHandlerFactory, in.config.MaxNumberOfMessages) - sqsReader := newSQSReader(log.Named("sqs"), in.metrics, sqsAPI, in.config.MaxNumberOfMessages, sqsMessageHandler) + sqsMessageHandler := newSQSS3EventProcessor(log.Named("sqs_s3_event"), in.metrics, sqsAPI, script, in.config.VisibilityTimeout, in.config.SQSMaxReceiveCount, s3EventHandlerFactory) + sqsReader := newSQSReader(log.Named("sqs"), in.metrics, sqsAPI, in.config.MaxNumberOfMessages, sqsMessageHandler, pipeline) return sqsReader, nil } diff --git a/x-pack/filebeat/input/awss3/input_benchmark_test.go b/x-pack/filebeat/input/awss3/input_benchmark_test.go index e05e5b461ca6..2a7c21ace0b3 100644 --- a/x-pack/filebeat/input/awss3/input_benchmark_test.go +++ b/x-pack/filebeat/input/awss3/input_benchmark_test.go @@ -6,16 +6,24 @@ package awss3 import ( "context" + "encoding/json" "errors" "fmt" - "io/ioutil" + "math" + "math/rand" "os" "path/filepath" "runtime" + "strconv" + "strings" "sync" "testing" "time" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + + "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" @@ -28,35 +36,90 @@ import ( "github.com/dustin/go-humanize" "github.com/olekukonko/tablewriter" - pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/monitoring" ) const ( - cloudtrailTestFile = "testdata/aws-cloudtrail.json.gz" + cloudtrailTestFileGz = "testdata/aws-cloudtrail.json.gz" + cloudtrailTestFile = "testdata/aws-cloudtrail.json" totalListingObjects = 10000 totalListingObjectsForInputS3 = totalListingObjects / 5 ) type constantSQS struct { - msgs []sqsTypes.Message + s3API *constantS3 + receiveCallN *atomic.Uint64 + msgs [][]sqsTypes.Message } var _ sqsAPI = (*constantSQS)(nil) -func newConstantSQS() *constantSQS { - return &constantSQS{ - msgs: []sqsTypes.Message{ - newSQSMessage(newS3Event(filepath.Base(cloudtrailTestFile))), - }, +func newConstantSQS(t testing.TB, maxMessages int, totalSqsMessages uint64, s3API *constantS3) *constantSQS { + customRand := rand.New(rand.NewSource(1)) + + var s3ObjN int + var generatedMessages uint64 + + msgs := make([][]sqsTypes.Message, 0) + + c := &constantSQS{s3API: s3API, receiveCallN: atomic.NewUint64(0)} + for { + if generatedMessages == totalSqsMessages { + break + } + + currentMessages := uint64(customRand.Intn(maxMessages)) + 1 + if totalSqsMessages < generatedMessages+currentMessages { + currentMessages = totalSqsMessages - generatedMessages + } + + generatedMessages += currentMessages + + currentMsgs := make([]sqsTypes.Message, 0, currentMessages) + for ; currentMessages > 0; currentMessages-- { + totS3Events := customRand.Intn(9) + 1 + s3Events := make([]s3EventV2, 0, totS3Events) + for ; totS3Events > 0; totS3Events-- { + totRecordsInS3Events := customRand.Intn(160) + 1 + recordsInS3Events := make([]map[string]any, 0, totRecordsInS3Events) + for ; totRecordsInS3Events > 0; totRecordsInS3Events-- { + recordN := customRand.Intn(len(c.s3API.records)-1) + 1 + recordsInS3Events = append(recordsInS3Events, c.s3API.records[recordN]) + } + + s3ObjKey := fmt.Sprintf("%d", s3ObjN) + s3Events = append(s3Events, newS3Event(s3ObjKey)) + + data, err := json.Marshal(struct{ Records []map[string]any }{Records: recordsInS3Events}) + if err != nil { + t.Fatal(err) + } + + c.s3API.objects = append(c.s3API.objects, data) + s3ObjN++ + } + + currentMsgs = append(currentMsgs, newSQSMessage(s3Events...)) + } + + msgs = append(msgs, currentMsgs) } + + c.msgs = msgs + + return c } func (c *constantSQS) ReceiveMessage(ctx context.Context, maxMessages int) ([]sqsTypes.Message, error) { - return c.msgs, nil + receiveCallN := c.receiveCallN.Add(1) + var msgs []sqsTypes.Message + if receiveCallN <= uint64(len(c.msgs)) { + msgs = c.msgs[receiveCallN-1] + } + + return msgs, nil } func (*constantSQS) DeleteMessage(ctx context.Context, msg *sqsTypes.Message) error { @@ -72,6 +135,7 @@ func (c *constantSQS) GetQueueAttributes(ctx context.Context, attr []sqsTypes.Qu } type s3PagerConstant struct { + listPrefix string mutex *sync.Mutex objects []s3Types.Object currentIndex int @@ -100,13 +164,13 @@ func (c *s3PagerConstant) NextPage(ctx context.Context, optFns ...func(*s3.Optio ret.Contents = c.objects[c.currentIndex : c.currentIndex+pageSize] c.currentIndex = c.currentIndex + pageSize - return ret, nil } func newS3PagerConstant(listPrefix string) *s3PagerConstant { lastModified := time.Now() ret := &s3PagerConstant{ + listPrefix: listPrefix, mutex: new(sync.Mutex), currentIndex: 0, } @@ -125,6 +189,8 @@ func newS3PagerConstant(listPrefix string) *s3PagerConstant { type constantS3 struct { filename string data []byte + records []map[string]any + objects [][]byte contentType string pagerConstant s3Pager } @@ -132,57 +198,134 @@ type constantS3 struct { var _ s3API = (*constantS3)(nil) func newConstantS3(t testing.TB) *constantS3 { - data, err := ioutil.ReadFile(cloudtrailTestFile) + dataGz, err := os.ReadFile(cloudtrailTestFileGz) + if err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(cloudtrailTestFile) + if err != nil { + t.Fatal(err) + } + + var records struct{ Records []map[string]any } + err = json.Unmarshal(data, &records) if err != nil { t.Fatal(err) } return &constantS3{ - filename: filepath.Base(cloudtrailTestFile), - data: data, + filename: filepath.Base(cloudtrailTestFileGz), + data: dataGz, + records: records.Records, contentType: contentTypeJSON, + objects: make([][]byte, 0), } } -func (c constantS3) GetObject(ctx context.Context, bucket, key string) (*s3.GetObjectOutput, error) { - return newS3GetObjectResponse(c.filename, c.data, c.contentType), nil +func (c *constantS3) GetObject(ctx context.Context, bucket, key string) (*s3.GetObjectOutput, error) { + // direct listing uses gz content + if strings.HasSuffix(key, ".json.gz") { + return newS3GetObjectResponse(c.filename, c.data, c.contentType), nil + } + + // this is s3 sqs notification + keyN, err := strconv.Atoi(key) + if err != nil { + return nil, err + } + + return newS3GetObjectResponse(key, c.objects[keyN], c.contentType), nil } -func (c constantS3) CopyObject(ctx context.Context, from_bucket, to_bucket, from_key, to_key string) (*s3.CopyObjectOutput, error) { +func (c *constantS3) CopyObject(ctx context.Context, from_bucket, to_bucket, from_key, to_key string) (*s3.CopyObjectOutput, error) { return nil, nil } -func (c constantS3) DeleteObject(ctx context.Context, bucket, key string) (*s3.DeleteObjectOutput, error) { +func (c *constantS3) DeleteObject(ctx context.Context, bucket, key string) (*s3.DeleteObjectOutput, error) { return nil, nil } -func (c constantS3) ListObjectsPaginator(bucket, prefix string) s3Pager { +func (c *constantS3) ListObjectsPaginator(bucket, prefix string) s3Pager { return c.pagerConstant } var _ beat.Pipeline = (*fakePipeline)(nil) +func newFakePipeline() *fakePipeline { + fp := &fakePipeline{ + mutex: new(sync.Mutex), + flush: time.NewTicker(10 * time.Second), + pendingEvents: atomic.NewUint64(0), + clients: make([]*ackClient, 0), + } + + go func() { + for { + <-fp.flush.C + fp.mutex.Lock() + fp.ackEvents() + fp.mutex.Unlock() + } + }() + + return fp +} + // fakePipeline returns new ackClients. -type fakePipeline struct{} +type fakePipeline struct { + flush *time.Ticker + mutex *sync.Mutex + pendingEvents *atomic.Uint64 + clients []*ackClient +} -func (c *fakePipeline) ConnectWith(clientConfig beat.ClientConfig) (beat.Client, error) { - return &ackClient{}, nil +func (fp *fakePipeline) ackEvents() { + for _, client := range fp.clients { + for _, acker := range client.ackers { + addedEvents := acker.eventsToBeAcked.Load() + for addedEvents > 0 && acker.eventsAcked.Load() != addedEvents { + fp.pendingEvents.Dec() + client.eventListener.AddEvent(beat.Event{Private: acker}, true) + client.eventListener.ACKEvents(1) + } + } + } } -func (c *fakePipeline) Connect() (beat.Client, error) { +func (fp *fakePipeline) ConnectWith(clientConfig beat.ClientConfig) (beat.Client, error) { + fp.mutex.Lock() + client := &ackClient{fp: fp, ackers: make(map[uint64]*EventACKTracker), eventListener: NewEventACKHandler()} + fp.clients = append(fp.clients, client) + fp.mutex.Unlock() + return client, nil +} + +func (fp *fakePipeline) Connect() (beat.Client, error) { panic("Connect() is not implemented.") } var _ beat.Client = (*ackClient)(nil) // ackClient is a fake beat.Client that ACKs the published messages. -type ackClient struct{} +type ackClient struct { + fp *fakePipeline + ackers map[uint64]*EventACKTracker + eventListener beat.EventListener +} func (c *ackClient) Close() error { return nil } func (c *ackClient) Publish(event beat.Event) { - // Fake the ACK handling. - event.Private.(*awscommon.EventACKTracker).ACK() + c.fp.mutex.Lock() + c.fp.pendingEvents.Inc() + acker := event.Private.(*EventACKTracker) + c.ackers[acker.ID] = acker + if c.fp.pendingEvents.Load() > 3200 { + c.fp.ackEvents() + } + + c.fp.mutex.Unlock() } func (c *ackClient) PublishAll(event []beat.Event) { @@ -198,6 +341,9 @@ file_selectors: - regex: '.json.gz$' expand_event_list_from_field: Records +- + regex: '^[\d]+$' + expand_event_list_from_field: Records `) inputConfig := defaultConfig() @@ -212,27 +358,37 @@ func benchmarkInputSQS(t *testing.T, maxMessagesInflight int) testing.BenchmarkR log := logp.NewLogger(inputName) metricRegistry := monitoring.NewRegistry() metrics := newInputMetrics("test_id", metricRegistry, maxMessagesInflight) - sqsAPI := newConstantSQS() + totalSqsMessages := uint64(math.Ceil(float64(maxMessagesInflight) * 1.1)) s3API := newConstantS3(t) - pipeline := &fakePipeline{} + sqsAPI := newConstantSQS(t, maxMessagesInflight, totalSqsMessages, s3API) + + logSqs := log.Named("sqs") + pipeline := newFakePipeline() + conf := makeBenchmarkConfig(t) s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), metrics, s3API, conf.FileSelectors, backupConfig{}, maxMessagesInflight) - sqsMessageHandler := newSQSS3EventProcessor(log.Named("sqs_s3_event"), metrics, sqsAPI, nil, time.Minute, 5, pipeline, s3EventHandlerFactory, maxMessagesInflight) - sqsReader := newSQSReader(log.Named("sqs"), metrics, sqsAPI, maxMessagesInflight, sqsMessageHandler) + sqsMessageHandler := newSQSS3EventProcessor(log.Named("sqs_s3_event"), metrics, sqsAPI, nil, time.Minute, 5, s3EventHandlerFactory) + sqsReader := newSQSReader(logSqs, metrics, sqsAPI, maxMessagesInflight, sqsMessageHandler, pipeline) ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) + cancelChan := make(chan time.Duration) + start := time.Now() go func() { - for metrics.sqsMessagesReceivedTotal.Get() < uint64(b.N) { + for { + if metrics.sqsMessagesProcessedTotal.Get() == totalSqsMessages { + break + } time.Sleep(5 * time.Millisecond) } + cancel() + cancelChan <- time.Since(start) }() b.ResetTimer() - start := time.Now() if err := sqsReader.Receive(ctx); err != nil { if !errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) @@ -240,15 +396,23 @@ func benchmarkInputSQS(t *testing.T, maxMessagesInflight int) testing.BenchmarkR } b.StopTimer() elapsed := time.Since(start) + cancelElapsed := <-cancelChan + deltaElapsedCanceled := elapsed.Seconds() - cancelElapsed.Seconds() b.ReportMetric(float64(maxMessagesInflight), "max_messages_inflight") b.ReportMetric(elapsed.Seconds(), "sec") + b.ReportMetric(cancelElapsed.Seconds(), "cancel_sec") + b.ReportMetric(deltaElapsedCanceled, "delta_sec_from_cancel") + b.ReportMetric(100.*(deltaElapsedCanceled/elapsed.Seconds()), "flushing_time_percentage") b.ReportMetric(float64(metrics.s3EventsCreatedTotal.Get()), "events") - b.ReportMetric(float64(metrics.s3EventsCreatedTotal.Get())/elapsed.Seconds(), "events_per_sec") + b.ReportMetric(float64(metrics.s3EventsCreatedTotal.Get())/cancelElapsed.Seconds(), "events_per_sec") b.ReportMetric(float64(metrics.s3BytesProcessedTotal.Get()), "s3_bytes") - b.ReportMetric(float64(metrics.s3BytesProcessedTotal.Get())/elapsed.Seconds(), "s3_bytes_per_sec") + b.ReportMetric(float64(metrics.s3BytesProcessedTotal.Get())/cancelElapsed.Seconds(), "s3_bytes_per_sec") + + b.ReportMetric(float64(metrics.s3ObjectsRequestedTotal.Get()), "s3_objects") + b.ReportMetric(float64(metrics.s3ObjectsRequestedTotal.Get())/cancelElapsed.Seconds(), "s3_objects_per_sec") b.ReportMetric(float64(metrics.sqsMessagesDeletedTotal.Get()), "sqs_messages") b.ReportMetric(float64(metrics.sqsMessagesDeletedTotal.Get())/elapsed.Seconds(), "sqs_messages_per_sec") @@ -271,21 +435,38 @@ func TestBenchmarkInputSQS(t *testing.T) { benchmarkInputSQS(t, 512), benchmarkInputSQS(t, 1024), } - headers := []string{ "Max Msgs Inflight", + "Events total", "Events per sec", + "S3 Bytes total", "S3 Bytes per sec", - "Time (sec)", + "S3 Objects total", + "S3 Objects per sec", + "SQS Messages total", + "SQS Messages per sec", + "Full Time (sec)", + "Processing Time (sec)", + "Flushing Time (sec)", + "Flushing time (%)", "CPUs", } data := make([][]string, 0) for _, r := range results { data = append(data, []string{ fmt.Sprintf("%v", r.Extra["max_messages_inflight"]), + fmt.Sprintf("%v", r.Extra["events"]), fmt.Sprintf("%v", r.Extra["events_per_sec"]), + fmt.Sprintf("%v", humanize.Bytes(uint64(r.Extra["s3_bytes"]))), fmt.Sprintf("%v", humanize.Bytes(uint64(r.Extra["s3_bytes_per_sec"]))), + fmt.Sprintf("%v", r.Extra["s3_objects"]), + fmt.Sprintf("%v", r.Extra["s3_objects_per_sec"]), + fmt.Sprintf("%v", r.Extra["sqs_messages"]), + fmt.Sprintf("%v", r.Extra["sqs_messages_per_sec"]), fmt.Sprintf("%v", r.Extra["sec"]), + fmt.Sprintf("%v", r.Extra["cancel_sec"]), + fmt.Sprintf("%v", r.Extra["delta_sec_from_cancel"]), + fmt.Sprintf("%v", humanize.FormatFloat("#,##", r.Extra["flushing_time_percentage"])), fmt.Sprintf("%v", runtime.GOMAXPROCS(0)), }) } @@ -312,7 +493,7 @@ func benchmarkInputS3(t *testing.T, numberOfWorkers int) testing.BenchmarkResult _ = client.Close() }() - config := makeBenchmarkConfig(t) + conf := makeBenchmarkConfig(t) b.ResetTimer() start := time.Now() @@ -348,7 +529,7 @@ func benchmarkInputS3(t *testing.T, numberOfWorkers int) testing.BenchmarkResult return } - s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), metrics, s3API, config.FileSelectors, backupConfig{}, numberOfWorkers) + s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), metrics, s3API, conf.FileSelectors, backupConfig{}, numberOfWorkers) s3Poller := newS3Poller(logp.NewLogger(inputName), metrics, s3API, client, s3EventHandlerFactory, newStates(inputCtx), store, "bucket", listPrefix, "region", "provider", numberOfWorkers, time.Second) if err := s3Poller.Poll(ctx); err != nil { diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 62cbc8350113..58f03e77687c 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -12,7 +12,6 @@ import ( "bytes" "context" "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -64,7 +63,7 @@ func getTerraformOutputs(t *testing.T, isLocalStack bool) terraformOutputData { outputFile = terraformOutputYML } - ymlData, err := ioutil.ReadFile(path.Join(path.Dir(filename), outputFile)) + ymlData, err := os.ReadFile(path.Join(path.Dir(filename), outputFile)) if os.IsNotExist(err) { t.Skipf("Run 'terraform apply' in %v to setup S3 and SQS for the test.", filepath.Dir(outputFile)) } @@ -365,7 +364,7 @@ func uploadS3TestFiles(t *testing.T, region, bucket string, s3Client *s3.Client, _, basefile, _, _ := runtime.Caller(0) basedir := path.Dir(basefile) for _, filename := range filenames { - data, err := ioutil.ReadFile(path.Join(basedir, filename)) + data, err := os.ReadFile(path.Join(basedir, filename)) if err != nil { t.Fatalf("Failed to open file %q, %v", filename, err) } diff --git a/x-pack/filebeat/input/awss3/interfaces.go b/x-pack/filebeat/input/awss3/interfaces.go index 1f1390c4f2fa..762af280cfaf 100644 --- a/x-pack/filebeat/input/awss3/interfaces.go +++ b/x-pack/filebeat/input/awss3/interfaces.go @@ -11,18 +11,19 @@ import ( "net/url" "time" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/aws/smithy-go/middleware" - "github.com/elastic/beats/v7/libbeat/beat" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/elastic-agent-libs/logp" ) @@ -65,7 +66,8 @@ type sqsProcessor interface { // given message and is responsible for updating the message's visibility // timeout while it is being processed and for deleting it when processing // completes successfully. - ProcessSQS(ctx context.Context, msg *types.Message) error + ProcessSQS(ctx context.Context, msg *types.Message, client beat.Client, acker *EventACKTracker, start time.Time, metrics *inputMetrics) (uint64, error) + DeleteSQS(msg *types.Message, receiveCount int, processingErr error, handles []s3ObjectHandler) error } // ------ @@ -100,16 +102,16 @@ type s3ObjectHandlerFactory interface { // Create returns a new s3ObjectHandler that can be used to process the // specified S3 object. If the handler is not configured to process the // given S3 object (based on key name) then it will return nil. - Create(ctx context.Context, log *logp.Logger, client beat.Client, acker *awscommon.EventACKTracker, obj s3EventV2) s3ObjectHandler + Create(ctx context.Context, log *logp.Logger, client beat.Client, acker *EventACKTracker, obj s3EventV2) s3ObjectHandler + CreateForS3Polling(ctx context.Context, log *logp.Logger, client beat.Client, acker *awscommon.EventACKTracker, obj s3EventV2) s3ObjectHandler } type s3ObjectHandler interface { // ProcessS3Object downloads the S3 object, parses it, creates events, and // publishes them. It returns when processing finishes or when it encounters // an unrecoverable error. It does not wait for the events to be ACKed by - // the publisher before returning (use eventACKTracker's Wait() method to - // determine this). - ProcessS3Object() error + // the publisher before returning. + ProcessS3Object() (uint64, error) // FinalizeS3Object finalizes processing of an S3 object after the current // batch is finished. @@ -117,7 +119,7 @@ type s3ObjectHandler interface { // Wait waits for every event published by ProcessS3Object() to be ACKed // by the publisher before returning. Internally it uses the - // s3ObjectHandler eventACKTracker's Wait() method + // s3ObjectHandler ackerForPolling's Wait() method Wait() } diff --git a/x-pack/filebeat/input/awss3/metrics.go b/x-pack/filebeat/input/awss3/metrics.go index bef57210ca65..c697a0339fa8 100644 --- a/x-pack/filebeat/input/awss3/metrics.go +++ b/x-pack/filebeat/input/awss3/metrics.go @@ -54,6 +54,7 @@ type inputMetrics struct { sqsWorkerStartTimes map[uint64]time.Time // Map of SQS worker ID to the time at which the worker started. sqsMessagesReceivedTotal *monitoring.Uint // Number of SQS messages received (not necessarily processed fully). + sqsMessagesProcessedTotal *monitoring.Uint // Number of SQS messages processed fully. sqsVisibilityTimeoutExtensionsTotal *monitoring.Uint // Number of SQS visibility timeout extensions. sqsMessagesInflight *monitoring.Uint // Number of SQS messages inflight (gauge). sqsMessagesReturnedTotal *monitoring.Uint // Number of SQS message returned to queue (happens on errors implicitly after visibility timeout passes). @@ -83,8 +84,6 @@ func (m *inputMetrics) Close() { // must be used to call endSQSWorker when the worker finishes. It also // increments the sqsMessagesInflight counter. func (m *inputMetrics) beginSQSWorker() (id uint64) { - m.sqsMessagesInflight.Inc() - m.sqsWorkerUtilizationMutex.Lock() defer m.sqsWorkerUtilizationMutex.Unlock() m.sqsWorkerIDCounter++ @@ -97,8 +96,6 @@ func (m *inputMetrics) beginSQSWorker() (id uint64) { // It also decrements the sqsMessagesInflight counter and // sqsMessageProcessingTime histogram. func (m *inputMetrics) endSQSWorker(id uint64) { - m.sqsMessagesInflight.Dec() - m.sqsWorkerUtilizationMutex.Lock() defer m.sqsWorkerUtilizationMutex.Unlock() now := currentTime() @@ -158,6 +155,7 @@ func newInputMetrics(id string, optionalParent *monitoring.Registry, maxWorkers sqsWorkerStartTimes: map[uint64]time.Time{}, sqsWorkerUtilizationLastUpdate: currentTime(), sqsMessagesReceivedTotal: monitoring.NewUint(reg, "sqs_messages_received_total"), + sqsMessagesProcessedTotal: monitoring.NewUint(reg, "sqs_messages_processed_total"), sqsVisibilityTimeoutExtensionsTotal: monitoring.NewUint(reg, "sqs_visibility_timeout_extensions_total"), sqsMessagesInflight: monitoring.NewUint(reg, "sqs_messages_inflight_gauge"), sqsMessagesReturnedTotal: monitoring.NewUint(reg, "sqs_messages_returned_total"), diff --git a/x-pack/filebeat/input/awss3/mock_interfaces_test.go b/x-pack/filebeat/input/awss3/mock_interfaces_test.go index b976cf00ebc4..d47a557567f6 100644 --- a/x-pack/filebeat/input/awss3/mock_interfaces_test.go +++ b/x-pack/filebeat/input/awss3/mock_interfaces_test.go @@ -276,18 +276,33 @@ func (m *MockSQSProcessor) EXPECT() *MockSQSProcessorMockRecorder { return m.recorder } -// ProcessSQS mocks base method. -func (m *MockSQSProcessor) ProcessSQS(ctx context.Context, msg *types.Message) error { +// DeleteSQS mocks base method. +func (m *MockSQSProcessor) DeleteSQS(msg *types.Message, receiveCount int, processingErr error, handles []s3ObjectHandler) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessSQS", ctx, msg) + ret := m.ctrl.Call(m, "DeleteSQS", msg, receiveCount, processingErr, handles) ret0, _ := ret[0].(error) return ret0 } +// DeleteSQS indicates an expected call of DeleteSQS. +func (mr *MockSQSProcessorMockRecorder) DeleteSQS(msg, receiveCount, processingErr, handles interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteSQS", reflect.TypeOf((*MockSQSProcessor)(nil).DeleteSQS), msg, receiveCount, processingErr, handles) +} + +// ProcessSQS mocks base method. +func (m *MockSQSProcessor) ProcessSQS(ctx context.Context, msg *types.Message, client beat.Client, acker *EventACKTracker, start time.Time, metrics *inputMetrics) (uint64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProcessSQS", ctx, msg, client, acker, start, metrics) + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + // ProcessSQS indicates an expected call of ProcessSQS. -func (mr *MockSQSProcessorMockRecorder) ProcessSQS(ctx, msg interface{}) *gomock.Call { +func (mr *MockSQSProcessorMockRecorder) ProcessSQS(ctx, msg, client, acker, start, metrics interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSQS", reflect.TypeOf((*MockSQSProcessor)(nil).ProcessSQS), ctx, msg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSQS", reflect.TypeOf((*MockSQSProcessor)(nil).ProcessSQS), ctx, msg, client, acker, start, metrics) } // MockS3API is a mock of s3API interface. @@ -581,7 +596,7 @@ func (m *MockS3ObjectHandlerFactory) EXPECT() *MockS3ObjectHandlerFactoryMockRec } // Create mocks base method. -func (m *MockS3ObjectHandlerFactory) Create(ctx context.Context, log *logp.Logger, client beat.Client, acker *aws.EventACKTracker, obj s3EventV2) s3ObjectHandler { +func (m *MockS3ObjectHandlerFactory) Create(ctx context.Context, log *logp.Logger, client beat.Client, acker *EventACKTracker, obj s3EventV2) s3ObjectHandler { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", ctx, log, client, acker, obj) ret0, _ := ret[0].(s3ObjectHandler) @@ -594,6 +609,20 @@ func (mr *MockS3ObjectHandlerFactoryMockRecorder) Create(ctx, log, client, acker return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockS3ObjectHandlerFactory)(nil).Create), ctx, log, client, acker, obj) } +// CreateForS3Polling mocks base method. +func (m *MockS3ObjectHandlerFactory) CreateForS3Polling(ctx context.Context, log *logp.Logger, client beat.Client, acker *aws.EventACKTracker, obj s3EventV2) s3ObjectHandler { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateForS3Polling", ctx, log, client, acker, obj) + ret0, _ := ret[0].(s3ObjectHandler) + return ret0 +} + +// CreateForS3Polling indicates an expected call of CreateForS3Polling. +func (mr *MockS3ObjectHandlerFactoryMockRecorder) CreateForS3Polling(ctx, log, client, acker, obj interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateForS3Polling", reflect.TypeOf((*MockS3ObjectHandlerFactory)(nil).CreateForS3Polling), ctx, log, client, acker, obj) +} + // MockS3ObjectHandler is a mock of s3ObjectHandler interface. type MockS3ObjectHandler struct { ctrl *gomock.Controller @@ -632,11 +661,12 @@ func (mr *MockS3ObjectHandlerMockRecorder) FinalizeS3Object() *gomock.Call { } // ProcessS3Object mocks base method. -func (m *MockS3ObjectHandler) ProcessS3Object() error { +func (m *MockS3ObjectHandler) ProcessS3Object() (uint64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ProcessS3Object") - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 } // ProcessS3Object indicates an expected call of ProcessS3Object. diff --git a/x-pack/filebeat/input/awss3/s3.go b/x-pack/filebeat/input/awss3/s3.go index 5aa8d31e95de..b04f50bc0713 100644 --- a/x-pack/filebeat/input/awss3/s3.go +++ b/x-pack/filebeat/input/awss3/s3.go @@ -129,7 +129,7 @@ func (p *s3Poller) createS3ObjectProcessor(ctx context.Context, state state) (s3 acker := awscommon.NewEventACKTracker(ctx) - return p.s3ObjectHandler.Create(ctx, p.log, p.client, acker, event), event + return p.s3ObjectHandler.CreateForS3Polling(ctx, p.log, p.client, acker, event), event } func (p *s3Poller) ProcessObject(s3ObjectPayloadChan <-chan *s3ObjectPayload) error { @@ -137,7 +137,7 @@ func (p *s3Poller) ProcessObject(s3ObjectPayloadChan <-chan *s3ObjectPayload) er for s3ObjectPayload := range s3ObjectPayloadChan { // Process S3 object (download, parse, create events). - err := s3ObjectPayload.s3ObjectHandler.ProcessS3Object() + _, err := s3ObjectPayload.s3ObjectHandler.ProcessS3Object() // Wait for all events to be ACKed before proceeding. s3ObjectPayload.s3ObjectHandler.Wait() diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index 32911778336b..47cf17947ce4 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -19,13 +19,14 @@ import ( "strings" "time" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/reader" "github.com/elastic/beats/v7/libbeat/reader/readfile" "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -73,7 +74,7 @@ func (f *s3ObjectProcessorFactory) findReaderConfig(key string) *readerConfig { // Create returns a new s3ObjectProcessor. It returns nil when no file selectors // match the S3 object key. -func (f *s3ObjectProcessorFactory) Create(ctx context.Context, log *logp.Logger, client beat.Client, ack *awscommon.EventACKTracker, obj s3EventV2) s3ObjectHandler { +func (f *s3ObjectProcessorFactory) Create(ctx context.Context, log *logp.Logger, client beat.Client, ack *EventACKTracker, obj s3EventV2) s3ObjectHandler { log = log.With( "bucket_arn", obj.S3.Bucket.Name, "object_key", obj.S3.Object.Key) @@ -96,28 +97,56 @@ func (f *s3ObjectProcessorFactory) Create(ctx context.Context, log *logp.Logger, } } +// CreateForS3Polling returns a new s3ObjectProcessor. It returns nil when no file selectors +// match the S3 object key. +func (f *s3ObjectProcessorFactory) CreateForS3Polling(ctx context.Context, log *logp.Logger, client beat.Client, ack *awscommon.EventACKTracker, obj s3EventV2) s3ObjectHandler { + log = log.With( + "bucket_arn", obj.S3.Bucket.Name, + "object_key", obj.S3.Object.Key) + + readerConfig := f.findReaderConfig(obj.S3.Object.Key) + if readerConfig == nil { + log.Debug("Skipping S3 object processing. No file_selectors are a match.") + return nil + } + + return &s3ObjectProcessor{ + s3ObjectProcessorFactory: f, + log: log, + ctx: ctx, + publisher: client, + ackerForPolling: ack, + readerConfig: readerConfig, + s3Obj: obj, + s3ObjHash: s3ObjectHash(obj), + } +} + type s3ObjectProcessor struct { *s3ObjectProcessorFactory - log *logp.Logger - ctx context.Context - publisher beat.Client - acker *awscommon.EventACKTracker // ACKer tied to the SQS message (multiple S3 readers share an ACKer when the S3 notification event contains more than one S3 object). - readerConfig *readerConfig // Config about how to process the object. - s3Obj s3EventV2 // S3 object information. - s3ObjHash string - s3RequestURL string + log *logp.Logger + ctx context.Context + publisher beat.Client + acker *EventACKTracker // ACKer tied to the SQS message (multiple S3 readers share an ACKer when the S3 notification event contains more than one S3 object). + ackerForPolling *awscommon.EventACKTracker // ACKer tied to the S3 object (multiple S3 readers share an ACKer when the S3 notification event contains more than one S3 object). + readerConfig *readerConfig // Config about how to process the object. + s3Obj s3EventV2 // S3 object information. + s3ObjHash string + s3RequestURL string s3Metadata map[string]interface{} // S3 object metadata. + + eventsPublishedTotal uint64 } func (p *s3ObjectProcessor) Wait() { - p.acker.Wait() + p.ackerForPolling.Wait() } -func (p *s3ObjectProcessor) ProcessS3Object() error { +func (p *s3ObjectProcessor) ProcessS3Object() (uint64, error) { if p == nil { - return nil + return 0, nil } // Metrics and Logging @@ -135,7 +164,7 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { // Request object (download). contentType, meta, body, err := p.download() if err != nil { - return fmt.Errorf("failed to get s3 object (elapsed_time_ns=%d): %w", + return 0, fmt.Errorf("failed to get s3 object (elapsed_time_ns=%d): %w", time.Since(start).Nanoseconds(), err) } defer body.Close() @@ -143,7 +172,7 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { reader, err := p.addGzipDecoderIfNeeded(newMonitoredReader(body, p.metrics.s3BytesProcessedTotal)) if err != nil { - return fmt.Errorf("failed checking for gzip content: %w", err) + return 0, fmt.Errorf("failed checking for gzip content: %w", err) } // Overwrite with user configured Content-Type. @@ -152,19 +181,19 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { } // try to create a decoder from the using the codec config - decoder, err := newDecoder(p.readerConfig.Decoding, reader) - if err != nil { - return err + decoder, decoderErr := newDecoder(p.readerConfig.Decoding, reader) + if decoderErr != nil { + return 0, err } if decoder != nil { defer decoder.close() var evtOffset int64 for decoder.next() { - data, err := decoder.decode() - if err != nil { + data, decoderErr := decoder.decode() + if decoderErr != nil { if errors.Is(err, io.EOF) { - return nil + return p.eventsPublishedTotal, nil } break } @@ -184,11 +213,11 @@ func (p *s3ObjectProcessor) ProcessS3Object() error { } } if err != nil { - return fmt.Errorf("failed reading s3 object (elapsed_time_ns=%d): %w", + return 0, fmt.Errorf("failed reading s3 object (elapsed_time_ns=%d): %w", time.Since(start).Nanoseconds(), err) } - return nil + return p.eventsPublishedTotal, nil } // download requests the S3 object from AWS and returns the object's @@ -250,7 +279,7 @@ func (p *s3ObjectProcessor) readJSON(r io.Reader) error { data, _ := item.MarshalJSON() evt := p.createEvent(string(data), offset) - p.publish(p.acker, &evt) + p.publish(&evt) } return nil @@ -285,11 +314,11 @@ func (p *s3ObjectProcessor) readJSONSlice(r io.Reader, evtOffset int64) (int64, data, _ := item.MarshalJSON() evt := p.createEvent(string(data), evtOffset) - p.publish(p.acker, &evt) + p.publish(&evt) evtOffset++ } - return evtOffset, nil + return evtOffset, p.ctx.Err() } func (p *s3ObjectProcessor) splitEventList(key string, raw json.RawMessage, offset int64, objHash string) error { @@ -330,7 +359,7 @@ func (p *s3ObjectProcessor) splitEventList(key string, raw json.RawMessage, offs data, _ := item.MarshalJSON() p.s3ObjHash = objHash evt := p.createEvent(string(data), offset+arrayOffset) - p.publish(p.acker, &evt) + p.publish(&evt) } return nil @@ -370,7 +399,7 @@ func (p *s3ObjectProcessor) readFile(r io.Reader) error { event := p.createEvent(string(message.Content), offset) event.Fields.DeepUpdate(message.Fields) offset += int64(message.Bytes) - p.publish(p.acker, &event) + p.publish(&event) } if errors.Is(err, io.EOF) { @@ -385,9 +414,15 @@ func (p *s3ObjectProcessor) readFile(r io.Reader) error { return nil } -func (p *s3ObjectProcessor) publish(ack *awscommon.EventACKTracker, event *beat.Event) { - ack.Add() - event.Private = ack +func (p *s3ObjectProcessor) publish(event *beat.Event) { + if p.acker != nil { + event.Private = p.acker + } else if p.ackerForPolling != nil { + p.ackerForPolling.Add() + event.Private = p.ackerForPolling + } + + p.eventsPublishedTotal++ p.metrics.s3EventsCreatedTotal.Inc() p.publisher.Publish(*event) } diff --git a/x-pack/filebeat/input/awss3/s3_objects_test.go b/x-pack/filebeat/input/awss3/s3_objects_test.go index 6732c12e0579..fa08b0a5b974 100644 --- a/x-pack/filebeat/input/awss3/s3_objects_test.go +++ b/x-pack/filebeat/input/awss3/s3_objects_test.go @@ -8,10 +8,13 @@ import ( "bytes" "context" "errors" - "io/ioutil" + "io" + "os" "path/filepath" "strings" + "sync" "testing" + "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -21,13 +24,12 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) func newS3Object(t testing.TB, filename, contentType string) (s3EventV2, *s3.GetObjectOutput) { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { t.Fatal(err) } @@ -39,7 +41,7 @@ func newS3GetObjectResponse(filename string, data []byte, contentType string) *s r := bytes.NewReader(data) getObjectOutput := s3.GetObjectOutput{} getObjectOutput.ContentLength = int64(r.Len()) - getObjectOutput.Body = ioutil.NopCloser(r) + getObjectOutput.Body = io.NopCloser(r) if contentType != "" { getObjectOutput.ContentType = &contentType } @@ -154,8 +156,8 @@ func TestS3ObjectProcessor(t *testing.T) { Return(nil, errFakeConnectivityFailure) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupConfig{}, 1) - ack := awscommon.NewEventACKTracker(ctx) - err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() + ack := NewEventACKTracker(ctx, nil) + _, err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() require.Error(t, err) assert.True(t, errors.Is(err, errFakeConnectivityFailure), "expected errFakeConnectivityFailure error") }) @@ -176,8 +178,8 @@ func TestS3ObjectProcessor(t *testing.T) { Return(nil, nil) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupConfig{}, 1) - ack := awscommon.NewEventACKTracker(ctx) - err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() + ack := NewEventACKTracker(ctx, nil) + _, err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() require.Error(t, err) }) @@ -203,8 +205,8 @@ func TestS3ObjectProcessor(t *testing.T) { ) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupConfig{}, 1) - ack := awscommon.NewEventACKTracker(ctx) - err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() + ack := NewEventACKTracker(ctx, nil) + _, err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() require.NoError(t, err) }) @@ -229,7 +231,7 @@ func TestS3ObjectProcessor(t *testing.T) { ) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupCfg, 1) - ack := awscommon.NewEventACKTracker(ctx) + ack := NewEventACKTracker(ctx, nil) err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).FinalizeS3Object() require.NoError(t, err) }) @@ -259,7 +261,7 @@ func TestS3ObjectProcessor(t *testing.T) { ) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupCfg, 1) - ack := awscommon.NewEventACKTracker(ctx) + ack := NewEventACKTracker(ctx, nil) err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).FinalizeS3Object() require.NoError(t, err) }) @@ -286,7 +288,7 @@ func TestS3ObjectProcessor(t *testing.T) { ) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, nil, backupCfg, 1) - ack := awscommon.NewEventACKTracker(ctx) + ack := NewEventACKTracker(ctx, nil) err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).FinalizeS3Object() require.NoError(t, err) }) @@ -300,15 +302,15 @@ func TestS3ObjectProcessor(t *testing.T) { }) } -func testProcessS3Object(t testing.TB, file, contentType string, numEvents int, selectors ...fileSelectorConfig) []beat.Event { +func testProcessS3Object(t testing.TB, file, contentType string, numEvents uint64, selectors ...fileSelectorConfig) []beat.Event { return _testProcessS3Object(t, file, contentType, numEvents, false, selectors) } -func testProcessS3ObjectError(t testing.TB, file, contentType string, numEvents int, selectors ...fileSelectorConfig) []beat.Event { +func testProcessS3ObjectError(t testing.TB, file, contentType string, numEvents uint64, selectors ...fileSelectorConfig) []beat.Event { return _testProcessS3Object(t, file, contentType, numEvents, true, selectors) } -func _testProcessS3Object(t testing.TB, file, contentType string, numEvents int, expectErr bool, selectors []fileSelectorConfig) []beat.Event { +func _testProcessS3Object(t testing.TB, file, contentType string, numEvents uint64, expectErr bool, selectors []fileSelectorConfig) []beat.Event { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), testTimeout) @@ -320,6 +322,14 @@ func _testProcessS3Object(t testing.TB, file, contentType string, numEvents int, mockPublisher := NewMockBeatClient(ctrl) s3Event, s3Resp := newS3Object(t, file, contentType) + acker := NewEventACKTracker(ctx, nil) + _, keepaliveCancel := context.WithCancel(ctx) + log := log.Named("sqs_s3_event") + mockMsgHandler := NewMockSQSProcessor(ctrl) + msg := newSQSMessage(newS3Event("log.json")) + acker.MarkSQSProcessedWithData(&msg, numEvents, -1, time.Now(), nil, nil, keepaliveCancel, new(sync.WaitGroup), mockMsgHandler, log, newInputMetrics("", nil, 0)) + acker.eventsToBeAcked.Add(numEvents) + var events []beat.Event gomock.InOrder( mockS3API.EXPECT(). @@ -327,18 +337,20 @@ func _testProcessS3Object(t testing.TB, file, contentType string, numEvents int, Return(s3Resp, nil), mockPublisher.EXPECT(). Publish(gomock.Any()). - Do(func(event beat.Event) { events = append(events, event) }). - Times(numEvents), + Do(func(event beat.Event) { + events = append(events, event) + acker.ACK() + }). + Times(int(numEvents)), ) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockS3API, selectors, backupConfig{}, 1) - ack := awscommon.NewEventACKTracker(ctx) - err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, ack, s3Event).ProcessS3Object() + s3EventsCreatedTotal, err := s3ObjProc.Create(ctx, logp.NewLogger(inputName), mockPublisher, acker, s3Event).ProcessS3Object() if !expectErr { require.NoError(t, err) - assert.Equal(t, numEvents, len(events)) - assert.EqualValues(t, numEvents, ack.PendingACKs) + assert.EqualValues(t, numEvents, len(events)) + assert.EqualValues(t, numEvents, s3EventsCreatedTotal) } else { require.Error(t, err) } diff --git a/x-pack/filebeat/input/awss3/sqs.go b/x-pack/filebeat/input/awss3/sqs.go index dd454a3bfb92..18f9917fcd37 100644 --- a/x-pack/filebeat/input/awss3/sqs.go +++ b/x-pack/filebeat/input/awss3/sqs.go @@ -11,11 +11,14 @@ import ( "sync" "time" + "github.com/elastic/beats/v7/libbeat/common/atomic" + + "github.com/elastic/go-concert/timed" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/go-concert/timed" ) const ( @@ -25,45 +28,125 @@ const ( type sqsReader struct { maxMessagesInflight int - workerSem *awscommon.Sem sqs sqsAPI + pipeline beat.Pipeline msgHandler sqsProcessor log *logp.Logger metrics *inputMetrics } -func newSQSReader(log *logp.Logger, metrics *inputMetrics, sqs sqsAPI, maxMessagesInflight int, msgHandler sqsProcessor) *sqsReader { +func newSQSReader(log *logp.Logger, metrics *inputMetrics, sqs sqsAPI, maxMessagesInflight int, msgHandler sqsProcessor, pipeline beat.Pipeline) *sqsReader { if metrics == nil { // Metrics are optional. Initialize a stub. metrics = newInputMetrics("", nil, 0) } return &sqsReader{ maxMessagesInflight: maxMessagesInflight, - workerSem: awscommon.NewSem(maxMessagesInflight), sqs: sqs, + pipeline: pipeline, msgHandler: msgHandler, log: log, metrics: metrics, } } +type processingData struct { + msg types.Message + start time.Time +} + func (r *sqsReader) Receive(ctx context.Context) error { - // This loop tries to keep the workers busy as much as possible while - // honoring the max message cap as opposed to a simpler loop that receives - // N messages, waits for them all to finish, then requests N more messages. - var workerWg sync.WaitGroup - for ctx.Err() == nil { - // Determine how many SQS workers are available. - workers, err := r.workerSem.AcquireContext(r.maxMessagesInflight, ctx) + workersWg := new(sync.WaitGroup) + workersWg.Add(r.maxMessagesInflight) + workersChan := make(chan processingData, r.maxMessagesInflight) + + deletionWg := new(sync.WaitGroup) + deletionWaiter := atomic.MakeBool(true) + clients := make(map[uint64]beat.Client, r.maxMessagesInflight) + + // Start a fixed amount of goroutines that will process all the SQS messages sent to the workersChan asynchronously. + for i := 0; i < r.maxMessagesInflight; i++ { + id := r.metrics.beginSQSWorker() + + // Create a pipeline client scoped to this goroutine. + client, err := r.pipeline.ConnectWith(beat.ClientConfig{ + EventListener: NewEventACKHandler(), + Processing: beat.ProcessingConfig{ + // This input only produces events with basic types so normalization + // is not required. + EventNormalization: boolPtr(false), + }, + }) + + clients[id] = client + if err != nil { - break + r.log.Warnw("Failed setting up worker.", + "worker_id", id, + "error", err) + + r.metrics.endSQSWorker(id) + workersWg.Done() } + go func(id uint64, client beat.Client) { + defer func() { + // Mark processing wait group as done. + r.metrics.endSQSWorker(id) + workersWg.Done() + }() + for { + incomingData, ok := <-workersChan + if !ok { + return + } + + msg := incomingData.msg + start := incomingData.start + + r.log.Debugw("Going to process SQS message.", + "worker_id", id, + "message_id", *msg.MessageId, + "elapsed_time_ns", time.Since(start)) + + acker := NewEventACKTracker(ctx, deletionWg) + + deletionWg.Add(1) + deletionWaiter.Swap(false) + + r.metrics.sqsMessagesInflight.Inc() + + eventsCreatedTotal, err := r.msgHandler.ProcessSQS(ctx, &msg, client, acker, start, r.metrics) + // No Track will be invoked by the client event listener, deletionWg.Done() has to be called here + if eventsCreatedTotal == 0 { + deletionWg.Done() + } + + if err != nil { + r.log.Warnw("Failed processing SQS message.", + "worker_id", id, + "error", err, + "message_id", *msg.MessageId, + "elapsed_time_ns", time.Since(start)) + + return + } + + r.log.Debugw("Success processing SQS message.", + "worker_id", id, + "message_id", msg.MessageId, + "elapsed_time_ns", time.Since(start)) + } + }(id, client) + } + + // The loop tries to keep a fixed amount of goroutines that process SQS message busy as much as possible while + // honoring the max message cap as opposed to a simpler loop that receives N messages, waits for them all to finish + // sending events to the queue, then requests N more messages. + for ctx.Err() == nil { // Receive (at most) as many SQS messages as there are workers. - msgs, err := r.sqs.ReceiveMessage(ctx, workers) + msgs, err := r.sqs.ReceiveMessage(ctx, r.maxMessagesInflight) if err != nil { - r.workerSem.Release(workers) - if ctx.Err() == nil { r.log.Warnw("SQS ReceiveMessage returned an error. Will retry after a short delay.", "error", err) @@ -73,43 +156,45 @@ func (r *sqsReader) Receive(ctx context.Context) error { continue } - // Release unused workers. - r.workerSem.Release(workers - len(msgs)) - - // Process each SQS message asynchronously with a goroutine. r.log.Debugf("Received %v SQS messages.", len(msgs)) r.metrics.sqsMessagesReceivedTotal.Add(uint64(len(msgs))) - workerWg.Add(len(msgs)) for _, msg := range msgs { - go func(msg types.Message, start time.Time) { - id := r.metrics.beginSQSWorker() - defer func() { - r.metrics.endSQSWorker(id) - workerWg.Done() - r.workerSem.Release(1) - }() - - if err := r.msgHandler.ProcessSQS(ctx, &msg); err != nil { - r.log.Warnw("Failed processing SQS message.", - "error", err, - "message_id", *msg.MessageId, - "elapsed_time_ns", time.Since(start)) - } - }(msg, time.Now()) + workersChan <- processingData{msg: msg, start: time.Now()} + } + } + + // Let's stop the workers + close(workersChan) + + // Wait for all processing to happen. + workersWg.Wait() + + // Wait for all deletion to happen. + if r.metrics.sqsMessagesReceivedTotal.Get() > 0 { + for deletionWaiter.Load() { + time.Sleep(500 * time.Millisecond) } } - // Wait for all workers to finish. - workerWg.Wait() + deletionWg.Wait() + + closeClients(clients) if errors.Is(ctx.Err(), context.Canceled) { // A canceled context is a normal shutdown. return nil } + return ctx.Err() } +func closeClients(clients map[uint64]beat.Client) { + for _, client := range clients { + client.Close() + } +} + func (r *sqsReader) GetApproximateMessageCount(ctx context.Context) (int, error) { attributes, err := r.sqs.GetQueueAttributes(ctx, []types.QueueAttributeName{sqsApproximateNumberOfMessages}) if err == nil { diff --git a/x-pack/filebeat/input/awss3/sqs_acker.go b/x-pack/filebeat/input/awss3/sqs_acker.go new file mode 100644 index 000000000000..bd53905a9c06 --- /dev/null +++ b/x-pack/filebeat/input/awss3/sqs_acker.go @@ -0,0 +1,158 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awss3 + +import ( + "context" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/common/atomic" +) + +var ackerIDCounter *atomic.Uint64 + +func init() { + ackerIDCounter = atomic.NewUint64(0) +} + +// EventACKTracker tracks the publishing state of S3 objects. Specifically +// it tracks the number of message acknowledgements that are pending from the +// output. It can be used to wait until all ACKs have been received for one or +// more S3 objects. +type EventACKTracker struct { + ID uint64 + + eventsAcked atomic.Uint64 + eventsToBeAcked atomic.Uint64 + + ctx context.Context + cancel context.CancelFunc + deletionWg *sync.WaitGroup + + msg *types.Message + ReceiveCount int + start time.Time + processingErr error + Handles []s3ObjectHandler + keepaliveCancel context.CancelFunc + keepaliveWg *sync.WaitGroup + msgHandler sqsProcessor + log *logp.Logger + metrics *inputMetrics +} + +func NewEventACKTracker(ctx context.Context, deletionWg *sync.WaitGroup) *EventACKTracker { + ctx, cancel := context.WithCancel(ctx) + acker := &EventACKTracker{ + ID: ackerIDCounter.Inc(), + ctx: ctx, + cancel: cancel, + deletionWg: deletionWg, + } + + go func() { + t := time.NewTicker(500 * time.Microsecond) + defer t.Stop() + + for { + <-t.C + if !acker.FullyTracked() { + continue + } + + acker.cancelAndFlush() + return + } + }() + + return acker +} + +func (a *EventACKTracker) cancelAndFlush() { + a.deleteSQS() + a.cancel() +} + +// MarkSQSProcessedWithData Every call after the first one is a no-op +func (a *EventACKTracker) MarkSQSProcessedWithData(msg *types.Message, publishedEvent uint64, receiveCount int, start time.Time, processingErr error, handles []s3ObjectHandler, keepaliveCancel context.CancelFunc, keepaliveWg *sync.WaitGroup, msgHandler sqsProcessor, log *logp.Logger, metrics *inputMetrics) { + // We want to execute the logic of this call only once, when the ack mutex was locked on init + if a.eventsToBeAcked.Load() > 0 { + return + } + + a.msg = msg + a.eventsToBeAcked.Store(publishedEvent) + a.ReceiveCount = receiveCount + a.start = start + a.processingErr = processingErr + a.Handles = handles + a.keepaliveCancel = keepaliveCancel + a.keepaliveWg = keepaliveWg + a.msgHandler = msgHandler + a.log = log + a.metrics = metrics +} + +func (a *EventACKTracker) FullyTracked() bool { + eventsToBeTracked := a.eventsToBeAcked.Load() + if eventsToBeTracked == 0 { + return false + } + + // This is eating its own tail: we should check for dropped+published, but then we won't wait for acked. + // Acked might not be equal to published? + return a.eventsAcked.Load() == eventsToBeTracked +} + +// deleteSQS delete related SQS message +func (a *EventACKTracker) deleteSQS() { + // Stop keepalive visibility routine before deleting. + a.keepaliveCancel() + a.keepaliveWg.Wait() + + a.metrics.sqsMessagesInflight.Dec() + + err := a.msgHandler.DeleteSQS(a.msg, a.ReceiveCount, a.processingErr, a.Handles) + if err != nil { + a.log.Warnw("Failed deleting SQS message.", + "error", err, + "message_id", *a.msg.MessageId, + "elapsed_time_ns", time.Since(a.start)) + } else { + a.log.Debugw("Success deleting SQS message.", + "message_id", *a.msg.MessageId, + "elapsed_time_ns", time.Since(a.start)) + } + + a.deletionWg.Done() + +} + +// ACK increments the number of eventsAcked. +func (a *EventACKTracker) ACK() { + a.eventsAcked.Inc() +} + +// NewEventACKHandler returns a beat ACKer that can receive callbacks when +// an event has been ACKed an output. If the event contains a private metadata +// pointing to an eventACKTracker then it will invoke the trackers ACK() method +// to decrement the number of pending ACKs. +func NewEventACKHandler() beat.EventListener { + return acker.ConnectionOnly( + acker.EventPrivateReporter(func(_ int, privates []interface{}) { + for _, private := range privates { + if acker, ok := private.(*EventACKTracker); ok { + acker.ACK() + } + } + })) +} diff --git a/x-pack/filebeat/input/awss3/sqs_acker_test.go b/x-pack/filebeat/input/awss3/sqs_acker_test.go new file mode 100644 index 000000000000..c14138f26b2a --- /dev/null +++ b/x-pack/filebeat/input/awss3/sqs_acker_test.go @@ -0,0 +1,109 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awss3 + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/golang/mock/gomock" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" +) + +func TestEventACKTracker(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + deletionWg := new(sync.WaitGroup) + deletionWg.Add(1) + + acker := NewEventACKTracker(ctx, deletionWg) + _, keepaliveCancel := context.WithCancel(ctx) + log := log.Named("sqs_s3_event") + ctrl, _ := gomock.WithContext(ctx, t) + defer ctrl.Finish() + mockMsgHandler := NewMockSQSProcessor(ctrl) + msg := newSQSMessage(newS3Event("log.json")) + mockMsgHandler.EXPECT().DeleteSQS(gomock.Eq(&msg), gomock.Eq(-1), gomock.Nil(), gomock.Nil()).Return(nil) + acker.MarkSQSProcessedWithData(&msg, 1, -1, time.Now(), nil, nil, keepaliveCancel, new(sync.WaitGroup), mockMsgHandler, log, newInputMetrics("", nil, 0)) + acker.ACK() + <-acker.ctx.Done() + + assert.EqualValues(t, true, acker.FullyTracked()) + assert.ErrorIs(t, acker.ctx.Err(), context.Canceled) +} + +func TestEventACKTrackerCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + deletionWg := new(sync.WaitGroup) + acker := NewEventACKTracker(ctx, deletionWg) + cancel() + <-acker.ctx.Done() + + assert.EqualValues(t, false, acker.FullyTracked()) + assert.ErrorIs(t, acker.ctx.Err(), context.Canceled) +} + +func TestEventACKHandler(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + // Create acker. Add one ACK to event listener. + deletionWg := new(sync.WaitGroup) + deletionWg.Add(1) + acker := NewEventACKTracker(ctx, deletionWg) + _, keepaliveCancel := context.WithCancel(ctx) + log := log.Named("sqs_s3_event") + ctrl, _ := gomock.WithContext(ctx, t) + defer ctrl.Finish() + mockMsgHandler := NewMockSQSProcessor(ctrl) + msg := newSQSMessage(newS3Event("log.json")) + mockMsgHandler.EXPECT().DeleteSQS(gomock.Eq(&msg), gomock.Eq(-1), gomock.Nil(), gomock.Nil()).Return(nil) + acker.MarkSQSProcessedWithData(&msg, 1, -1, time.Now(), nil, nil, keepaliveCancel, new(sync.WaitGroup), mockMsgHandler, log, newInputMetrics("", nil, 0)) + + // Create an ACK handler and simulate one ACKed event. + ackHandler := NewEventACKHandler() + ackHandler.AddEvent(beat.Event{Private: acker}, true) + ackHandler.ACKEvents(1) + <-acker.ctx.Done() + + assert.EqualValues(t, true, acker.FullyTracked()) + assert.ErrorIs(t, acker.ctx.Err(), context.Canceled) +} + +func TestEventACKHandlerFullyAcked(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + // Create acker. Add one Track. + deletionWg := new(sync.WaitGroup) + deletionWg.Add(1) + + acker := NewEventACKTracker(ctx, deletionWg) + _, keepaliveCancel := context.WithCancel(ctx) + log := log.Named("sqs_s3_event") + ctrl, _ := gomock.WithContext(ctx, t) + defer ctrl.Finish() + mockMsgHandler := NewMockSQSProcessor(ctrl) + msg := newSQSMessage(newS3Event("log.json")) + mockMsgHandler.EXPECT().DeleteSQS(gomock.Eq(&msg), gomock.Eq(-1), gomock.Nil(), gomock.Nil()).Return(nil) + acker.MarkSQSProcessedWithData(&msg, 1, -1, time.Now(), nil, nil, keepaliveCancel, new(sync.WaitGroup), mockMsgHandler, log, newInputMetrics("", nil, 0)) + acker.ACK() + <-acker.ctx.Done() + assert.EqualValues(t, true, acker.FullyTracked()) + + assert.ErrorIs(t, acker.ctx.Err(), context.Canceled) + + acker.eventsToBeAcked.Inc() + + assert.EqualValues(t, false, acker.FullyTracked()) +} diff --git a/x-pack/filebeat/input/awss3/sqs_s3_event.go b/x-pack/filebeat/input/awss3/sqs_s3_event.go index 7f95cf564c09..87dc4d88b16f 100644 --- a/x-pack/filebeat/input/awss3/sqs_s3_event.go +++ b/x-pack/filebeat/input/awss3/sqs_s3_event.go @@ -20,7 +20,6 @@ import ( "go.uber.org/multierr" "github.com/elastic/beats/v7/libbeat/beat" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" ) @@ -44,7 +43,7 @@ func (e *nonRetryableError) Error() string { } func (e *nonRetryableError) Is(err error) bool { - _, ok := err.(*nonRetryableError) //nolint:errorlint // This is not used directly to detected wrapped errors (errors.Is handles unwrapping). + _, ok := err.(*nonRetryableError) return ok } @@ -88,7 +87,6 @@ type sqsS3EventProcessor struct { sqsVisibilityTimeout time.Duration maxReceiveCount int sqs sqsAPI - pipeline beat.Pipeline // Pipeline creates clients for publishing events. log *logp.Logger warnOnce sync.Once metrics *inputMetrics @@ -102,9 +100,7 @@ func newSQSS3EventProcessor( script *script, sqsVisibilityTimeout time.Duration, maxReceiveCount int, - pipeline beat.Pipeline, s3 s3ObjectHandlerFactory, - maxWorkers int, ) *sqsS3EventProcessor { if metrics == nil { // Metrics are optional. Initialize a stub. @@ -115,54 +111,25 @@ func newSQSS3EventProcessor( sqsVisibilityTimeout: sqsVisibilityTimeout, maxReceiveCount: maxReceiveCount, sqs: sqs, - pipeline: pipeline, log: log, metrics: metrics, script: script, } } -func (p *sqsS3EventProcessor) ProcessSQS(ctx context.Context, msg *types.Message) error { - log := p.log.With( - "message_id", *msg.MessageId, - "message_receipt_time", time.Now().UTC()) - - keepaliveCtx, keepaliveCancel := context.WithCancel(ctx) - defer keepaliveCancel() - - // Start SQS keepalive worker. - var keepaliveWg sync.WaitGroup - keepaliveWg.Add(1) - go p.keepalive(keepaliveCtx, log, &keepaliveWg, msg) - - receiveCount := getSQSReceiveCount(msg.Attributes) - if receiveCount == 1 { - // Only contribute to the sqs_lag_time histogram on the first message - // to avoid skewing the metric when processing retries. - if s, found := msg.Attributes[sqsSentTimestampAttribute]; found { - if sentTimeMillis, err := strconv.ParseInt(s, 10, 64); err == nil { - sentTime := time.UnixMilli(sentTimeMillis) - p.metrics.sqsLagTime.Update(time.Since(sentTime).Nanoseconds()) - } - } - } - - handles, processingErr := p.processS3Events(ctx, log, *msg.Body) - - // Stop keepalive routine before changing visibility. - keepaliveCancel() - keepaliveWg.Wait() - +func (p *sqsS3EventProcessor) DeleteSQS(msg *types.Message, receiveCount int, processingErr error, handles []s3ObjectHandler) error { // No error. Delete SQS. if processingErr == nil { if msgDelErr := p.sqs.DeleteMessage(context.Background(), msg); msgDelErr != nil { return fmt.Errorf("failed deleting message from SQS queue (it may be reprocessed): %w", msgDelErr) } + p.metrics.sqsMessagesDeletedTotal.Inc() // SQS message finished and deleted, finalize s3 objects if finalizeErr := p.finalizeS3Objects(handles); finalizeErr != nil { return fmt.Errorf("failed finalizing message from SQS queue (manual cleanup is required): %w", finalizeErr) } + return nil } @@ -180,12 +147,12 @@ func (p *sqsS3EventProcessor) ProcessSQS(ctx context.Context, msg *types.Message if errors.Is(processingErr, &nonRetryableError{}) { if msgDelErr := p.sqs.DeleteMessage(context.Background(), msg); msgDelErr != nil { return multierr.Combine( - fmt.Errorf("failed processing SQS message (attempted to delete message): %w", processingErr), + fmt.Errorf("failed deleting SQS message (attempted to delete message): %w", processingErr), fmt.Errorf("failed deleting message from SQS queue (it may be reprocessed): %w", msgDelErr), ) } p.metrics.sqsMessagesDeletedTotal.Inc() - return fmt.Errorf("failed processing SQS message (message was deleted): %w", processingErr) + return fmt.Errorf("failed deleting SQS message (message was deleted): %w", processingErr) } // An error that may be resolved by letting the visibility timeout @@ -193,7 +160,39 @@ func (p *sqsS3EventProcessor) ProcessSQS(ctx context.Context, msg *types.Message // queue is enabled then the message will eventually placed on the DLQ // after maximum receives is reached. p.metrics.sqsMessagesReturnedTotal.Inc() - return fmt.Errorf("failed processing SQS message (it will return to queue after visibility timeout): %w", processingErr) + return fmt.Errorf("failed deleting SQS message (it will return to queue after visibility timeout): %w", processingErr) +} + +func (p *sqsS3EventProcessor) ProcessSQS(ctx context.Context, msg *types.Message, client beat.Client, acker *EventACKTracker, start time.Time, metrics *inputMetrics) (uint64, error) { + keepaliveCtx, keepaliveCancel := context.WithCancel(ctx) + + // Start SQS keepalive worker. + var keepaliveWg sync.WaitGroup + keepaliveWg.Add(1) + + log := p.log.With( + "message_id", *msg.MessageId, + "message_receipt_time", time.Now().UTC()) + + go p.keepalive(keepaliveCtx, log, &keepaliveWg, msg) + + receiveCount := getSQSReceiveCount(msg.Attributes) + if receiveCount == 1 { + // Only contribute to the sqs_lag_time histogram on the first message + // to avoid skewing the metric when processing retries. + if s, found := msg.Attributes[sqsSentTimestampAttribute]; found { + if sentTimeMillis, err := strconv.ParseInt(s, 10, 64); err == nil { + sentTime := time.UnixMilli(sentTimeMillis) + p.metrics.sqsLagTime.Update(time.Since(sentTime).Nanoseconds()) + } + } + } + + eventsPublishedTotal, handles, processingErr := p.processS3Events(ctx, log, *msg.Body, client, acker) + p.metrics.sqsMessagesProcessedTotal.Inc() + acker.MarkSQSProcessedWithData(msg, eventsPublishedTotal, receiveCount, start, processingErr, handles, keepaliveCancel, &keepaliveWg, p, p.log, metrics) + + return eventsPublishedTotal, processingErr } func (p *sqsS3EventProcessor) keepalive(ctx context.Context, log *logp.Logger, wg *sync.WaitGroup, msg *types.Message) { @@ -291,42 +290,25 @@ func (*sqsS3EventProcessor) isObjectCreatedEvents(event s3EventV2) bool { return event.EventSource == "aws:s3" && strings.HasPrefix(event.EventName, "ObjectCreated:") } -func (p *sqsS3EventProcessor) processS3Events(ctx context.Context, log *logp.Logger, body string) ([]s3ObjectHandler, error) { +func (p *sqsS3EventProcessor) processS3Events(ctx context.Context, log *logp.Logger, body string, client beat.Client, acker *EventACKTracker) (uint64, []s3ObjectHandler, error) { s3Events, err := p.getS3Notifications(body) if err != nil { if errors.Is(err, context.Canceled) { // Messages that are in-flight at shutdown should be returned to SQS. - return nil, err + return 0, nil, err } - return nil, &nonRetryableError{err} + return 0, nil, &nonRetryableError{err} } log.Debugf("SQS message contained %d S3 event notifications.", len(s3Events)) defer log.Debug("End processing SQS S3 event notifications.") if len(s3Events) == 0 { - return nil, nil + return 0, nil, nil } - // Create a pipeline client scoped to this goroutine. - client, err := p.pipeline.ConnectWith(beat.ClientConfig{ - EventListener: awscommon.NewEventACKHandler(), - Processing: beat.ProcessingConfig{ - // This input only produces events with basic types so normalization - // is not required. - EventNormalization: boolPtr(false), - }, - }) - if err != nil { - return nil, err - } - defer client.Close() - - // Wait for all events to be ACKed before proceeding. - acker := awscommon.NewEventACKTracker(ctx) - defer acker.Wait() - var errs []error var handles []s3ObjectHandler + var eventsPublishedTotal uint64 for i, event := range s3Events { s3Processor := p.s3ObjectHandler.Create(ctx, log, client, acker, event) if s3Processor == nil { @@ -334,21 +316,22 @@ func (p *sqsS3EventProcessor) processS3Events(ctx context.Context, log *logp.Log } // Process S3 object (download, parse, create events). - if err := s3Processor.ProcessS3Object(); err != nil { + if eventsPublished, err := s3Processor.ProcessS3Object(); err != nil { errs = append(errs, fmt.Errorf( "failed processing S3 event for object key %q in bucket %q (object record %d of %d in SQS notification): %w", event.S3.Object.Key, event.S3.Bucket.Name, i+1, len(s3Events), err)) } else { + eventsPublishedTotal += eventsPublished handles = append(handles, s3Processor) } } // Make sure all s3 events were processed successfully if len(handles) == len(s3Events) { - return handles, multierr.Combine(errs...) + return eventsPublishedTotal, handles, multierr.Combine(errs...) } - return nil, multierr.Combine(errs...) + return 0, nil, multierr.Combine(errs...) } func (p *sqsS3EventProcessor) finalizeS3Objects(handles []s3ObjectHandler) error { diff --git a/x-pack/filebeat/input/awss3/sqs_s3_event_test.go b/x-pack/filebeat/input/awss3/sqs_s3_event_test.go index 5ecd72fc4c91..953a50cb7f78 100644 --- a/x-pack/filebeat/input/awss3/sqs_s3_event_test.go +++ b/x-pack/filebeat/input/awss3/sqs_s3_event_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" - awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/timed" ) @@ -39,19 +38,27 @@ func TestSQSS3EventProcessor(t *testing.T) { ctrl, ctx := gomock.WithContext(ctx, t) defer ctrl.Finish() mockAPI := NewMockSQSAPI(ctrl) + mockS3Handler := NewMockS3ObjectHandler(ctrl) mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) mockClient := NewMockBeatClient(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) + expectedEventsPublishedTotal := uint64(10) gomock.InOrder( - mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil), - mockS3HandlerFactory.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil), - mockClient.EXPECT().Close(), - mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&msg)).Return(nil), + mockS3HandlerFactory.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(mockS3Handler), + mockS3Handler.EXPECT().ProcessS3Object().Return(expectedEventsPublishedTotal, nil), + mockS3Handler.EXPECT().FinalizeS3Object().Return(nil), ) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - require.NoError(t, p.ProcessSQS(ctx, &msg)) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockS3HandlerFactory) + + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &msg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + require.NoError(t, processingErr) + require.Equal(t, expectedEventsPublishedTotal, eventsPublishedTotal) + + mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&msg)).Return(nil) + + require.NoError(t, p.DeleteSQS(&msg, acker.ReceiveCount, processingErr, acker.Handles)) }) t.Run("invalid SQS JSON body does not retry", func(t *testing.T) { @@ -62,21 +69,24 @@ func TestSQSS3EventProcessor(t *testing.T) { defer ctrl.Finish() mockAPI := NewMockSQSAPI(ctrl) mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) invalidBodyMsg := newSQSMessage(newS3Event("log.json")) body := *invalidBodyMsg.Body body = body[10:] invalidBodyMsg.Body = &body - gomock.InOrder( - mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&invalidBodyMsg)).Return(nil), - ) + mockClient := NewMockBeatClient(ctrl) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - err := p.ProcessSQS(ctx, &invalidBodyMsg) - require.Error(t, err) - t.Log(err) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockS3HandlerFactory) + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &invalidBodyMsg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + t.Log(processingErr) + require.Error(t, processingErr) + require.Equal(t, uint64(0), eventsPublishedTotal) + + mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&invalidBodyMsg)).Return(nil) + + require.Error(t, p.DeleteSQS(&invalidBodyMsg, acker.ReceiveCount, processingErr, acker.Handles)) }) t.Run("zero S3 events in body", func(t *testing.T) { @@ -87,16 +97,20 @@ func TestSQSS3EventProcessor(t *testing.T) { defer ctrl.Finish() mockAPI := NewMockSQSAPI(ctrl) mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) emptyRecordsMsg := newSQSMessage([]s3EventV2{}...) - gomock.InOrder( - mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&emptyRecordsMsg)).Return(nil), - ) + mockClient := NewMockBeatClient(ctrl) + + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockS3HandlerFactory) + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &emptyRecordsMsg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + require.NoError(t, processingErr) + require.Equal(t, uint64(0), eventsPublishedTotal) + + mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&emptyRecordsMsg)).Return(nil) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - require.NoError(t, p.ProcessSQS(ctx, &emptyRecordsMsg)) + require.NoError(t, p.DeleteSQS(&emptyRecordsMsg, acker.ReceiveCount, processingErr, acker.Handles)) }) t.Run("visibility is extended after half expires", func(t *testing.T) { @@ -111,24 +125,29 @@ func TestSQSS3EventProcessor(t *testing.T) { mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) mockS3Handler := NewMockS3ObjectHandler(ctrl) mockClient := NewMockBeatClient(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) mockAPI.EXPECT().ChangeMessageVisibility(gomock.Any(), gomock.Eq(&msg), gomock.Eq(visibilityTimeout)).AnyTimes().Return(nil) gomock.InOrder( - mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil), mockS3HandlerFactory.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Do(func(ctx context.Context, _ *logp.Logger, _ beat.Client, _ *awscommon.EventACKTracker, _ s3EventV2) { + Do(func(ctx context.Context, _ *logp.Logger, _ beat.Client, _ *EventACKTracker, _ s3EventV2) { require.NoError(t, timed.Wait(ctx, 5*visibilityTimeout)) }).Return(mockS3Handler), - mockS3Handler.EXPECT().ProcessS3Object().Return(nil), - mockClient.EXPECT().Close(), + mockS3Handler.EXPECT().ProcessS3Object().Return(uint64(1), nil), + ) + + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, visibilityTimeout, 5, mockS3HandlerFactory) + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &msg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + require.NoError(t, processingErr) + require.Equal(t, uint64(1), eventsPublishedTotal) + + gomock.InOrder( mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&msg)).Return(nil), mockS3Handler.EXPECT().FinalizeS3Object().Return(nil), ) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, visibilityTimeout, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - require.NoError(t, p.ProcessSQS(ctx, &msg)) + require.NoError(t, p.DeleteSQS(&msg, acker.ReceiveCount, processingErr, acker.Handles)) }) t.Run("message returns to queue on error", func(t *testing.T) { @@ -141,19 +160,20 @@ func TestSQSS3EventProcessor(t *testing.T) { mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) mockS3Handler := NewMockS3ObjectHandler(ctrl) mockClient := NewMockBeatClient(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) gomock.InOrder( - mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil), mockS3HandlerFactory.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(mockS3Handler), - mockS3Handler.EXPECT().ProcessS3Object().Return(errors.New("fake connectivity problem")), - mockClient.EXPECT().Close(), + mockS3Handler.EXPECT().ProcessS3Object().Return(uint64(0), errors.New("fake connectivity problem")), ) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - err := p.ProcessSQS(ctx, &msg) - t.Log(err) - require.Error(t, err) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockS3HandlerFactory) + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &msg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + t.Log(processingErr) + require.Error(t, processingErr) + require.Equal(t, uint64(0), eventsPublishedTotal) + + require.Error(t, p.DeleteSQS(&msg, acker.ReceiveCount, processingErr, acker.Handles)) }) t.Run("message is deleted after multiple receives", func(t *testing.T) { @@ -166,7 +186,6 @@ func TestSQSS3EventProcessor(t *testing.T) { mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) mockS3Handler := NewMockS3ObjectHandler(ctrl) mockClient := NewMockBeatClient(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) msg := msg msg.Attributes = map[string]string{ @@ -174,17 +193,20 @@ func TestSQSS3EventProcessor(t *testing.T) { } gomock.InOrder( - mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil), mockS3HandlerFactory.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(mockS3Handler), - mockS3Handler.EXPECT().ProcessS3Object().Return(errors.New("fake connectivity problem")), - mockClient.EXPECT().Close(), - mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&msg)).Return(nil), + mockS3Handler.EXPECT().ProcessS3Object().Return(uint64(0), errors.New("fake connectivity problem")), ) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockBeatPipeline, mockS3HandlerFactory, 5) - err := p.ProcessSQS(ctx, &msg) - t.Log(err) - require.Error(t, err) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, time.Minute, 5, mockS3HandlerFactory) + acker := NewEventACKTracker(ctx, new(sync.WaitGroup)) + eventsPublishedTotal, processingErr := p.ProcessSQS(ctx, &msg, mockClient, acker, time.Now(), newInputMetrics("", nil, 0)) + t.Log(processingErr) + require.Error(t, processingErr) + require.Equal(t, uint64(0), eventsPublishedTotal) + + mockAPI.EXPECT().DeleteMessage(gomock.Any(), gomock.Eq(&msg)).Return(nil) + + require.Error(t, p.DeleteSQS(&msg, acker.ReceiveCount, processingErr, acker.Handles)) }) } @@ -222,12 +244,11 @@ func TestSqsProcessor_keepalive(t *testing.T) { defer ctrl.Finish() mockAPI := NewMockSQSAPI(ctrl) mockS3HandlerFactory := NewMockS3ObjectHandlerFactory(ctrl) - mockBeatPipeline := NewMockBeatPipeline(ctrl) mockAPI.EXPECT().ChangeMessageVisibility(gomock.Any(), gomock.Eq(&msg), gomock.Eq(visibilityTimeout)). Times(1).Return(tc.Err) - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, visibilityTimeout, 5, mockBeatPipeline, mockS3HandlerFactory, 5) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, mockAPI, nil, visibilityTimeout, 5, mockS3HandlerFactory) var wg sync.WaitGroup wg.Add(1) p.keepalive(ctx, p.log, &wg, &msg) @@ -239,7 +260,7 @@ func TestSqsProcessor_keepalive(t *testing.T) { func TestSqsProcessor_getS3Notifications(t *testing.T) { logp.TestingSetup() - p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, nil, nil, time.Minute, 5, nil, nil, 5) + p := newSQSS3EventProcessor(logp.NewLogger(inputName), nil, nil, nil, time.Minute, 5, nil) t.Run("s3 key is url unescaped", func(t *testing.T) { msg := newSQSMessage(newS3Event("Happy+Face.jpg")) diff --git a/x-pack/filebeat/input/awss3/sqs_test.go b/x-pack/filebeat/input/awss3/sqs_test.go index 5eda5d1885e2..ab2be37f7e1c 100644 --- a/x-pack/filebeat/input/awss3/sqs_test.go +++ b/x-pack/filebeat/input/awss3/sqs_test.go @@ -9,9 +9,12 @@ import ( "crypto/sha256" "encoding/json" "errors" + "sync" "testing" "time" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/gofrs/uuid" "github.com/golang/mock/gomock" @@ -65,16 +68,37 @@ func TestSQSReceiver(t *testing.T) { }), ) + mockClient := NewMockBeatClient(ctrl) + mockBeatPipeline := NewMockBeatPipeline(ctrl) + + mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil).Times(maxMessages) + // Expect the one message returned to have been processed. mockMsgHandler.EXPECT(). - ProcessSQS(gomock.Any(), gomock.Eq(&msg)). + ProcessSQS(gomock.Any(), gomock.Eq(&msg), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(1). + DoAndReturn( + func(ctx context.Context, msg *types.Message, _ beat.Client, acker *EventACKTracker, _ time.Time, metrics *inputMetrics) (uint64, error) { + _, keepaliveCancel := context.WithCancel(ctx) + log := log.Named("sqs_s3_event") + acker.MarkSQSProcessedWithData(msg, 1, -1, time.Now(), nil, nil, keepaliveCancel, new(sync.WaitGroup), mockMsgHandler, log, metrics) + acker.ACK() + <-acker.ctx.Done() + return 1, nil + }) + + // Expect the client to be closed + mockClient.EXPECT().Close().Times(maxMessages) + + // Expect the one message returned to have been deleted. + mockMsgHandler.EXPECT(). + DeleteSQS(gomock.Eq(&msg), gomock.Any(), gomock.Any(), gomock.Any()). Times(1). Return(nil) // Execute sqsReader and verify calls/state. - receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler) + receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler, mockBeatPipeline) require.NoError(t, receiver.Receive(ctx)) - assert.Equal(t, maxMessages, receiver.workerSem.Available()) }) t.Run("retry after ReceiveMessage error", func(t *testing.T) { @@ -86,6 +110,11 @@ func TestSQSReceiver(t *testing.T) { mockAPI := NewMockSQSAPI(ctrl) mockMsgHandler := NewMockSQSProcessor(ctrl) + mockClient := NewMockBeatClient(ctrl) + mockBeatPipeline := NewMockBeatPipeline(ctrl) + + mockBeatPipeline.EXPECT().ConnectWith(gomock.Any()).Return(mockClient, nil).Times(maxMessages) + gomock.InOrder( // Initial ReceiveMessage gets an error. mockAPI.EXPECT(). @@ -104,10 +133,12 @@ func TestSQSReceiver(t *testing.T) { }), ) + // Expect the client to be closed + mockClient.EXPECT().Close().Times(maxMessages) + // Execute SQSReceiver and verify calls/state. - receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler) + receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler, mockBeatPipeline) require.NoError(t, receiver.Receive(ctx)) - assert.Equal(t, maxMessages, receiver.workerSem.Available()) }) } @@ -127,6 +158,7 @@ func TestGetApproximateMessageCount(t *testing.T) { defer ctrl.Finish() mockAPI := NewMockSQSAPI(ctrl) mockMsgHandler := NewMockSQSProcessor(ctrl) + mockBeatPipeline := NewMockBeatPipeline(ctrl) gomock.InOrder( mockAPI.EXPECT(). @@ -137,7 +169,7 @@ func TestGetApproximateMessageCount(t *testing.T) { }), ) - receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler) + receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler, mockBeatPipeline) receivedCount, err := receiver.GetApproximateMessageCount(ctx) assert.Equal(t, count, receivedCount) assert.Nil(t, err) @@ -152,6 +184,7 @@ func TestGetApproximateMessageCount(t *testing.T) { mockAPI := NewMockSQSAPI(ctrl) mockMsgHandler := NewMockSQSProcessor(ctrl) + mockBeatPipeline := NewMockBeatPipeline(ctrl) gomock.InOrder( mockAPI.EXPECT(). @@ -162,7 +195,7 @@ func TestGetApproximateMessageCount(t *testing.T) { }), ) - receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler) + receiver := newSQSReader(logp.NewLogger(inputName), nil, mockAPI, maxMessages, mockMsgHandler, mockBeatPipeline) receivedCount, err := receiver.GetApproximateMessageCount(ctx) assert.Equal(t, -1, receivedCount) assert.NotNil(t, err) diff --git a/x-pack/filebeat/input/awss3/testdata/aws-cloudtrail.json b/x-pack/filebeat/input/awss3/testdata/aws-cloudtrail.json new file mode 100644 index 000000000000..7c543cf58f21 --- /dev/null +++ b/x-pack/filebeat/input/awss3/testdata/aws-cloudtrail.json @@ -0,0 +1 @@ +{"Records":[{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}},{"eventVersion":"1.07","eventTime":"2019-11-14T00:51:00Z","awsRegion":"us-east-1","eventID":"EXAMPLE8-9621-4d00-b913-beca2EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"Start","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664}}}},"eventCategory":"Insight"},{"eventVersion":"1.07","eventTime":"2019-11-14T00:52:00Z","awsRegion":"us-east-1","eventID":"EXAMPLEc-28be-486c-8928-49ce6EXAMPLE","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"EXAMPLE2-1729-42f1-b735-5d8c0EXAMPLE","insightDetails":{"state":"End","eventSource":"ssm.amazonaws.com","eventName":"UpdateInstanceInformation","insightType":"ApiCallRateInsight","insightContext":{"statistics":{"baseline":{"average":85.4202380952},"insight":{"average":664},"insightDuration":1}}},"eventCategory":"Insight"},{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-d9cb6ff7d6a3","eventID":"b7d4398e-b2f0-4faa-9c76-e2d316a8d67f","eventType":"AwsApiCall","recipientAccountId":"123456789012"},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-25T20:17:37Z","eventSource":"iam.amazonaws.com","eventName":"CreateRole","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"assumeRolePolicyDocument":"{\n \"Version\": \"2012-10-17\",\n \"Statement\": [\n {\n \"Sid\": \"\",\n\"Effect\": \"Allow\",\n \"Principal\": {\n \"AWS\": \"arn:aws:iam::210987654321:root\"\n },\n \"Action\": \"sts:AssumeRole\"\n }\n ]\n}","roleName":"TestRole"},"responseElements":{"role":{"assumeRolePolicyDocument":"%7B%0A%20%20%22Version%22%3A%20%222012-10-17%22%2C%0A%20%20%22Statement%22%3A%20%5B%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%22Sid%22%3A%20%22%22%2C%0A%20%20%20%20%20%20%22Effect%22%3A%20%22Allow%22%2C%0A%20%20%20%20%20%20%22Principal%22%3A%20%7B%0A%20%20%20%20%20%20%20%20%22AWS%22%3A%20%22arn%3Aaws%3Aiam%3A%3A803981987763%3Aroot%22%0A%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%22Action%22%3A%20%22sts%3AAssumeRole%22%0A%20%20%20%20%7D%0A%20%20%5D%0A%7D","roleName":"TestRole","roleId":"AROAIUU2EOWSWPGX2UJUO","arn":"arn:aws:iam::123456789012:role/TestRole","createDate":"Mar 25, 2014 8:17:37 PM","path":"/"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-06T21:01:59Z","eventSource":"ec2.amazonaws.com","eventName":"StopInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]},"force":false},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":64,"name":"stopping"},"previousState":{"code":16,"name":"running"}}]}}},{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accessKeyId":"EXAMPLE_KEY_ID","accountId":"123456789012","userName":"Alice"},"eventTime":"2014-03-06T21:22:54Z","eventSource":"ec2.amazonaws.com","eventName":"StartInstances","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.176","userAgent":"ec2-api-tools 1.6.12.2","requestParameters":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2"}]}},"responseElements":{"instancesSet":{"items":[{"instanceId":"i-ebeaf9e2","currentState":{"code":0,"name":"pending"},"previousState":{"code":80,"name":"stopped"}}]}}}]}