From be0797ef074b7b65150e8e4258f2a75f037949c4 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Mon, 28 Nov 2022 16:49:47 +0530 Subject: [PATCH 1/7] initial commit for gcs input in akamai package --- packages/akamai/_dev/build/docs/README.md | 11 ++ packages/akamai/changelog.yml | 5 + .../data_stream/siem/agent/stream/gcs.yml.hbs | 40 +++++++ packages/akamai/data_stream/siem/manifest.yml | 111 +++++++++++++++++- packages/akamai/docs/README.md | 11 ++ packages/akamai/manifest.yml | 5 +- 6 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs diff --git a/packages/akamai/_dev/build/docs/README.md b/packages/akamai/_dev/build/docs/README.md index c28b1986701..d7ea6a8e88e 100644 --- a/packages/akamai/_dev/build/docs/README.md +++ b/packages/akamai/_dev/build/docs/README.md @@ -12,6 +12,17 @@ Use this API to get security event data generated on the ​Akamai​ platform a See [Akamai API get started](https://techdocs.akamai.com/siem-integration/reference/api-get-started) to set up your Akamai account and get your credentials. +### To collect data from GCS Bucket [Beta], follow the below steps: +- Configure the [Data Forwarder](https://techdocs.akamai.com/datastream2/docs/stream-google-cloud/) to ingest data into a GCS bucket. +- Configure the GCS bucket names and credentials along with the required configs under the "Collect Akamai SIEM logs via Google Cloud Storage" section. +- Make sure the service account and authentication being used, has proper levels of access to the GCS bucket [Manage Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys/) + +**Note**: +- The GCS input currently does not support fetching of buckets using bucket prefixes, so the bucket names have to be configured manually for each data stream. +- The GCS input currently only accepts a service account json key or a service account json file for authentication. +- The GCS input currently only supports JSON data. +- This input is still in beta. + {{fields "siem"}} {{event "siem"}} \ No newline at end of file diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml index 60e9c18d519..dab2afbd594 100644 --- a/packages/akamai/changelog.yml +++ b/packages/akamai/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.3.0" + changes: + - description: Added support for GCS input. + type: enhancement + link: link to updates soon - version: "2.2.0" changes: - description: Update package to ECS 8.5.0. diff --git a/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs new file mode 100644 index 00000000000..d528536fbbb --- /dev/null +++ b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs @@ -0,0 +1,40 @@ +{{#if collect_gcs_logs}} + +{{#if project_id}} +project_id: {{project_id}} +{{/if}} +{{#if service_account_key}} +auth.credentials_json.account_key: {{service_account_key}} +{{/if}} +{{#if service_account_file}} +auth.credentials_file.path: {{service_account_file}} +{{/if}} +{{#if number_of_workers}} +max_workers: {{number_of_workers}} +{{/if}} +{{#if polling}} +poll: {{polling}} +{{/if}} +{{#if poll_interval}} +poll_interval: {{poll_interval}} +{{/if}} +{{#if bucket_timeout}} +bucket_timeout: {{bucket_timeout}} +{{/if}} +{{#if buckets}} +buckets: +{{buckets}} +{{/if}} +{{#if tags}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/akamai/data_stream/siem/manifest.yml b/packages/akamai/data_stream/siem/manifest.yml index 29152e44f53..6e981a11c4c 100644 --- a/packages/akamai/data_stream/siem/manifest.yml +++ b/packages/akamai/data_stream/siem/manifest.yml @@ -3,6 +3,9 @@ title: Akamai SIEM Logs release: experimental streams: - input: httpjson + template_path: httpjson.yml.hbs + title: Akamai SIEM logs + description: Collect Akamai logs via the SIEM API vars: - name: api_host type: text @@ -100,6 +103,108 @@ streams: required: false show_user: false description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" - template_path: httpjson.yml.hbs - title: Akamai SIEM logs - description: Collect Akamai logs via the SIEM API + - input: gcs + title: Collect Akamai SIEM logs via Google Cloud Storage [Beta] + description: Collecting SIEM logs from Akamai via Google Cloud Storage. + template_path: gcs.yml.hbs + vars: + - name: collect_gcs_logs + required: true + show_user: true + title: Collect logs via GCS Bucket + description: To Collect logs via GCS bucket enable the toggle switch. By default, it will be toggled off. + type: bool + multi: false + default: false + - name: project_id + type: text + title: "[GCS] Project Id" + description: It is a required parameter to collect logs via GCS. + multi: false + required: true + show_user: true + default: my-project-id + - name: service_account_key + type: password + title: "[GCS] Credentials json key" + description: It is an optional parameter for authentication. + multi: false + required: false + show_user: true + - name: service_account_file + type: text + title: "[GCS] Credentials file path" + description: It is an optional parameter for authentication. + multi: false + required: false + show_user: false + - name: max_workers + type: integer + title: '[GCS] Maximum number of workers' + multi: false + required: false + show_user: true + default: 3 + description: Determines how many workers are spawned per bucket. + - name: poll + type: bool + title: '[GCS] Polling' + multi: false + required: false + show_user: true + default: true + description: Determines if the bucket will be continuously polled for new documents. + - name: poll_interval + type: text + title: '[GCS] Polling interval' + multi: false + required: false + show_user: true + default: 15s + description: Determines the time interval between polling operations. + - name: bucket_timeout + type: text + title: '[GCS] Bucket Timeout' + multi: false + required: false + show_user: true + default: 120s + description: Defines the maximum time that the sdk will wait for a bucket api response before timing out. + - name: buckets + type: yaml + title: Buckets + description: "This attribute contains the details about a specific bucket like, name, max_workers, poll, + poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, + while the fields max_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. + If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific + attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. + Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n" + required: true + show_user: true + default: | + # You can define as many buckets as you want here. + - name: siem_gcs_bucket_1 + - name: siem_gcs_bucket_2 + # The config below is an example of how to override the global config. + #- name: siem_gcs_bucket_3 + # max_workers: 3 + # poll: true + # poll_interval: 10s + # bucket_timeout: 30s + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: | + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - akamai-siem \ No newline at end of file diff --git a/packages/akamai/docs/README.md b/packages/akamai/docs/README.md index cae85c4c043..fcac3234461 100644 --- a/packages/akamai/docs/README.md +++ b/packages/akamai/docs/README.md @@ -12,6 +12,17 @@ Use this API to get security event data generated on the ​Akamai​ platform a See [Akamai API get started](https://techdocs.akamai.com/siem-integration/reference/api-get-started) to set up your Akamai account and get your credentials. +### To collect data from GCS Bucket [Beta], follow the below steps: +- Configure the [Data Forwarder](https://techdocs.akamai.com/datastream2/docs/stream-google-cloud/) to ingest data into a GCS bucket. +- Configure the GCS bucket names and credentials along with the required configs under the "Collect Akamai SIEM logs via Google Cloud Storage" section. +- Make sure the service account and authentication being used, has proper levels of access to the GCS bucket [Manage Service Account Keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys/) + +**Note**: +- The GCS input currently does not support fetching of buckets using bucket prefixes, so the bucket names have to be configured manually for each data stream. +- The GCS input currently only accepts a service account json key or a service account json file for authentication. +- The GCS input currently only supports JSON data. +- This input is still in beta. + **Exported fields** | Field | Description | Type | diff --git a/packages/akamai/manifest.yml b/packages/akamai/manifest.yml index bd31534afbf..9ee52174826 100644 --- a/packages/akamai/manifest.yml +++ b/packages/akamai/manifest.yml @@ -1,6 +1,6 @@ name: akamai title: Akamai -version: "2.2.0" +version: "2.3.0" release: ga description: Collect logs from Akamai with Elastic Agent. type: integration @@ -22,5 +22,8 @@ policy_templates: - type: httpjson title: "Collect Akamai SIEM logs via API" description: "Collecting SIEM logs from Akamai via API" + - type: gcs + title: "Collect Akamai SIEM logs via Google Cloud Storage" + description: "Collecting SIEM logs from Akamai via Google Cloud Storage" owner: github: elastic/security-external-integrations From a8fb53c32e49dd32d31618346d40d17fcf30dedc Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Mon, 28 Nov 2022 17:09:32 +0530 Subject: [PATCH 2/7] updated manifest --- packages/akamai/data_stream/siem/manifest.yml | 208 +++++++++--------- 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/packages/akamai/data_stream/siem/manifest.yml b/packages/akamai/data_stream/siem/manifest.yml index 6e981a11c4c..f9b11d1be30 100644 --- a/packages/akamai/data_stream/siem/manifest.yml +++ b/packages/akamai/data_stream/siem/manifest.yml @@ -104,107 +104,107 @@ streams: show_user: false description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. \nThis executes in the agent before the logs are parsed. \nSee [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.\n" - input: gcs - title: Collect Akamai SIEM logs via Google Cloud Storage [Beta] - description: Collecting SIEM logs from Akamai via Google Cloud Storage. - template_path: gcs.yml.hbs - vars: - - name: collect_gcs_logs - required: true - show_user: true - title: Collect logs via GCS Bucket - description: To Collect logs via GCS bucket enable the toggle switch. By default, it will be toggled off. - type: bool - multi: false - default: false - - name: project_id - type: text - title: "[GCS] Project Id" - description: It is a required parameter to collect logs via GCS. - multi: false - required: true - show_user: true - default: my-project-id - - name: service_account_key - type: password - title: "[GCS] Credentials json key" - description: It is an optional parameter for authentication. - multi: false - required: false - show_user: true - - name: service_account_file - type: text - title: "[GCS] Credentials file path" - description: It is an optional parameter for authentication. - multi: false - required: false - show_user: false - - name: max_workers - type: integer - title: '[GCS] Maximum number of workers' - multi: false - required: false - show_user: true - default: 3 - description: Determines how many workers are spawned per bucket. - - name: poll - type: bool - title: '[GCS] Polling' - multi: false - required: false - show_user: true - default: true - description: Determines if the bucket will be continuously polled for new documents. - - name: poll_interval - type: text - title: '[GCS] Polling interval' - multi: false - required: false - show_user: true - default: 15s - description: Determines the time interval between polling operations. - - name: bucket_timeout - type: text - title: '[GCS] Bucket Timeout' - multi: false - required: false - show_user: true - default: 120s - description: Defines the maximum time that the sdk will wait for a bucket api response before timing out. - - name: buckets - type: yaml - title: Buckets - description: "This attribute contains the details about a specific bucket like, name, max_workers, poll, - poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, - while the fields max_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. - If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific - attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. - Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n" - required: true - show_user: true - default: | - # You can define as many buckets as you want here. - - name: siem_gcs_bucket_1 - - name: siem_gcs_bucket_2 - # The config below is an example of how to override the global config. - #- name: siem_gcs_bucket_3 - # max_workers: 3 - # poll: true - # poll_interval: 10s - # bucket_timeout: 30s - - name: processors - type: yaml - title: Processors - multi: false - required: false - show_user: false - description: | - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - name: tags - type: text - title: Tags - multi: true - required: true - show_user: false - default: - - forwarded - - akamai-siem \ No newline at end of file + title: Collect Akamai SIEM logs via Google Cloud Storage [Beta] + description: Collecting SIEM logs from Akamai via Google Cloud Storage. + template_path: gcs.yml.hbs + vars: + - name: collect_gcs_logs + required: true + show_user: true + title: Collect logs via GCS Bucket + description: To Collect logs via GCS bucket enable the toggle switch. By default, it will be toggled off. + type: bool + multi: false + default: false + - name: project_id + type: text + title: "[GCS] Project Id" + description: It is a required parameter to collect logs via GCS. + multi: false + required: true + show_user: true + default: my-project-id + - name: service_account_key + type: password + title: "[GCS] Credentials json key" + description: It is an optional parameter for authentication. + multi: false + required: false + show_user: true + - name: service_account_file + type: text + title: "[GCS] Credentials file path" + description: It is an optional parameter for authentication. + multi: false + required: false + show_user: false + - name: max_workers + type: integer + title: '[GCS] Maximum number of workers' + multi: false + required: false + show_user: true + default: 3 + description: Determines how many workers are spawned per bucket. + - name: poll + type: bool + title: '[GCS] Polling' + multi: false + required: false + show_user: true + default: true + description: Determines if the bucket will be continuously polled for new documents. + - name: poll_interval + type: text + title: '[GCS] Polling interval' + multi: false + required: false + show_user: true + default: 15s + description: Determines the time interval between polling operations. + - name: bucket_timeout + type: text + title: '[GCS] Bucket Timeout' + multi: false + required: false + show_user: true + default: 120s + description: Defines the maximum time that the sdk will wait for a bucket api response before timing out. + - name: buckets + type: yaml + title: Buckets + description: "This attribute contains the details about a specific bucket like, name, max_workers, poll, + poll_interval and bucket_timeout. The attribute 'name' is specific to a bucket as it describes the bucket name, + while the fields max_workers, poll, poll_interval and bucket_timeout can exist both at the bucket level and at the global level. + If you have already defined the attributes globally, then you can only specify the name in this yaml config. If you want to override any specific + attribute for a specific bucket, then, you can define it here. Any attribute defined in the yaml will override the global definitions. + Please see the relevant [Documentation](https://www.elastic.co/guide/en/beats/filebeat/8.5/filebeat-input-gcs.html#attrib-buckets) for further information.\n" + required: true + show_user: true + default: | + # You can define as many buckets as you want here. + - name: siem_gcs_bucket_1 + - name: siem_gcs_bucket_2 + # The config below is an example of how to override the global config. + #- name: siem_gcs_bucket_3 + # max_workers: 3 + # poll: true + # poll_interval: 10s + # bucket_timeout: 30s + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: | + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - akamai-siem \ No newline at end of file From a1925ba44e67f386bd17785628aff51c423fe617 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 29 Nov 2022 15:37:08 +0530 Subject: [PATCH 3/7] updated changelog --- packages/akamai/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml index dab2afbd594..1600a8dcc59 100644 --- a/packages/akamai/changelog.yml +++ b/packages/akamai/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Added support for GCS input. type: enhancement - link: link to updates soon + link: https://github.com/elastic/integrations/pull/4728 - version: "2.2.0" changes: - description: Update package to ECS 8.5.0. From dd09a5508909f61d0a3f7ef74d7e87f9ab928aa1 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 29 Nov 2022 15:39:38 +0530 Subject: [PATCH 4/7] updated config --- packages/akamai/data_stream/siem/manifest.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/akamai/data_stream/siem/manifest.yml b/packages/akamai/data_stream/siem/manifest.yml index f9b11d1be30..56a820ad5c7 100644 --- a/packages/akamai/data_stream/siem/manifest.yml +++ b/packages/akamai/data_stream/siem/manifest.yml @@ -118,7 +118,7 @@ streams: default: false - name: project_id type: text - title: "[GCS] Project Id" + title: "Project Id" description: It is a required parameter to collect logs via GCS. multi: false required: true @@ -126,21 +126,21 @@ streams: default: my-project-id - name: service_account_key type: password - title: "[GCS] Credentials json key" + title: "Credentials json key" description: It is an optional parameter for authentication. multi: false required: false show_user: true - name: service_account_file type: text - title: "[GCS] Credentials file path" + title: "Credentials file path" description: It is an optional parameter for authentication. multi: false required: false show_user: false - name: max_workers type: integer - title: '[GCS] Maximum number of workers' + title: 'Maximum number of workers' multi: false required: false show_user: true @@ -148,7 +148,7 @@ streams: description: Determines how many workers are spawned per bucket. - name: poll type: bool - title: '[GCS] Polling' + title: 'Polling' multi: false required: false show_user: true @@ -156,7 +156,7 @@ streams: description: Determines if the bucket will be continuously polled for new documents. - name: poll_interval type: text - title: '[GCS] Polling interval' + title: 'Polling interval' multi: false required: false show_user: true @@ -164,7 +164,7 @@ streams: description: Determines the time interval between polling operations. - name: bucket_timeout type: text - title: '[GCS] Bucket Timeout' + title: 'Bucket Timeout' multi: false required: false show_user: true From 32cb65476a4148b7f047ded7a56ec2590de58106 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Tue, 29 Nov 2022 15:41:18 +0530 Subject: [PATCH 5/7] updated config --- packages/akamai/_dev/build/docs/README.md | 2 +- packages/akamai/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/akamai/_dev/build/docs/README.md b/packages/akamai/_dev/build/docs/README.md index d7ea6a8e88e..d0901521329 100644 --- a/packages/akamai/_dev/build/docs/README.md +++ b/packages/akamai/_dev/build/docs/README.md @@ -19,7 +19,7 @@ See [Akamai API get started](https://techdocs.akamai.com/siem-integration/refere **Note**: - The GCS input currently does not support fetching of buckets using bucket prefixes, so the bucket names have to be configured manually for each data stream. -- The GCS input currently only accepts a service account json key or a service account json file for authentication. +- The GCS input currently only accepts a service account JSON key or a service account JSON file for authentication. - The GCS input currently only supports JSON data. - This input is still in beta. diff --git a/packages/akamai/docs/README.md b/packages/akamai/docs/README.md index fcac3234461..10f2c3f5a2f 100644 --- a/packages/akamai/docs/README.md +++ b/packages/akamai/docs/README.md @@ -19,7 +19,7 @@ See [Akamai API get started](https://techdocs.akamai.com/siem-integration/refere **Note**: - The GCS input currently does not support fetching of buckets using bucket prefixes, so the bucket names have to be configured manually for each data stream. -- The GCS input currently only accepts a service account json key or a service account json file for authentication. +- The GCS input currently only accepts a service account JSON key or a service account JSON file for authentication. - The GCS input currently only supports JSON data. - This input is still in beta. From e49a8b494d59ed315327a99fd64e360efb4f6a42 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 30 Nov 2022 11:50:40 +0530 Subject: [PATCH 6/7] updated yml.hbs --- packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs index d528536fbbb..0c271c8c647 100644 --- a/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs +++ b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs @@ -27,7 +27,7 @@ buckets: {{/if}} {{#if tags}} tags: -{{#each tags as |tag i|}} +{{#each tags as |tag|}} - {{tag}} {{/each}} {{/if}} From 58aef497036dfb5fb1ea1ca58e6f55554e189284 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 30 Nov 2022 14:25:23 +0530 Subject: [PATCH 7/7] updated pr removed unused if condition in gcs.yml.hbs --- packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs | 2 -- packages/akamai/data_stream/siem/manifest.yml | 8 -------- 2 files changed, 10 deletions(-) diff --git a/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs index 0c271c8c647..14fd42ca0cb 100644 --- a/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs +++ b/packages/akamai/data_stream/siem/agent/stream/gcs.yml.hbs @@ -1,5 +1,3 @@ -{{#if collect_gcs_logs}} - {{#if project_id}} project_id: {{project_id}} {{/if}} diff --git a/packages/akamai/data_stream/siem/manifest.yml b/packages/akamai/data_stream/siem/manifest.yml index 56a820ad5c7..825a4fdf4e1 100644 --- a/packages/akamai/data_stream/siem/manifest.yml +++ b/packages/akamai/data_stream/siem/manifest.yml @@ -108,14 +108,6 @@ streams: description: Collecting SIEM logs from Akamai via Google Cloud Storage. template_path: gcs.yml.hbs vars: - - name: collect_gcs_logs - required: true - show_user: true - title: Collect logs via GCS Bucket - description: To Collect logs via GCS bucket enable the toggle switch. By default, it will be toggled off. - type: bool - multi: false - default: false - name: project_id type: text title: "Project Id"