diff --git a/Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml b/Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml index af9d172b249..10e97931677 100644 --- a/Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml +++ b/Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml @@ -17,47 +17,49 @@ triggerThreshold: 0 tactics: - Impact query: | - let dt_lookBack = 1h; - let ioc_lookBack = 14d; - //Create a list of TLDs in our threat feed for later validation of extracted domains - let list_tlds = ThreatIntelligenceIndicator - | where TimeGenerated > ago(ioc_lookBack) - | where isnotempty(DomainName) - | extend DomainName = tolower(DomainName) - | extend parts = split(DomainName, '.') - | extend tld = parts[(array_length(parts)-1)] - | summarize count() by tostring(tld) - | summarize make_list(tld); - ThreatIntelligenceIndicator - | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | where Active == true - // Picking up only IOC's that contain the entities we want - | where isnotempty(DomainName) - | join ( - CommonSecurityLog - | extend IngestionTime = ingestion_time() - | where IngestionTime > ago(dt_lookBack) - | where DeviceEventClassID =~ 'url' - //Uncomment the line below to only alert on allowed connections - //| where DeviceAction !~ "block-url" - //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions - | extend PA_Url = columnifexists("RequestURL", "None") - | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) - | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) - | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host)) - | where isnotempty(Domain) - | extend Domain = tolower(Domain) - | extend parts = split(Domain, '.') - //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on - | extend tld = parts[(array_length(parts)-1)] - //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match - | where tld in~ (list_tlds) - | extend CommonSecurityLog_TimeGenerated = TimeGenerated - ) on $left.DomainName==$right.Domain - | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime - | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod - | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url + + let dt_lookBack = 1h; + let ioc_lookBack = 14d; + //Create a list of TLDs in our threat feed for later validation of extracted domains + let list_tlds = ThreatIntelligenceIndicator + | where TimeGenerated > ago(ioc_lookBack) + | where isnotempty(DomainName) + | extend DomainName = tolower(DomainName) + | extend parts = split(DomainName, '.') + | extend tld = parts[(array_length(parts)-1)] + | summarize count() by tostring(tld) + | summarize make_list(tld); + ThreatIntelligenceIndicator + | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() + | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | where Active == true + // Picking up only IOC's that contain the entities we want + | where isnotempty(DomainName) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + CommonSecurityLog + | extend IngestionTime = ingestion_time() + | where IngestionTime > ago(dt_lookBack) + | where DeviceEventClassID =~ 'url' + //Uncomment the line below to only alert on allowed connections + //| where DeviceAction !~ "block-url" + //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions + | extend PA_Url = columnifexists("RequestURL", "None") + | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) + | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) + | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host)) + | where isnotempty(Domain) + | extend Domain = tolower(Domain) + | extend parts = split(Domain, '.') + //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on + | extend tld = parts[(array_length(parts)-1)] + //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match + | where tld in~ (list_tlds) + | extend CommonSecurityLog_TimeGenerated = TimeGenerated + ) + on $left.DomainName==$right.Domain + | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod + | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url entityMappings: - entityType: Host fieldMappings: @@ -71,5 +73,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.0 -kind: Scheduled \ No newline at end of file +version: 1.1.1 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml b/Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml index 9e1579e45af..a365007c4f2 100644 --- a/Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml +++ b/Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml @@ -21,38 +21,39 @@ tactics: - Impact query: | - let dt_lookBack = 1h; - let ioc_lookBack = 14d; - //Create a list of TLDs in our threat feed for later validation - let list_tlds = ThreatIntelligenceIndicator - | where TimeGenerated > ago(ioc_lookBack) - | where isnotempty(DomainName) - | extend parts = split(DomainName, '.') - | extend tld = parts[(array_length(parts)-1)] - | summarize count() by tostring(tld) - | summarize make_list(tld); - ThreatIntelligenceIndicator - | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | where Active == true - // Picking up only IOC's that contain the entities we want - | where isnotempty(DomainName) - | join ( - DnsEvents - | where TimeGenerated > ago(dt_lookBack) - //Extract domain patterns from syslog message - | where isnotempty(Name) - | extend parts = split(Name, '.') - //Split out the TLD - | extend tld = parts[(array_length(parts)-1)] - //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed - | where tld in~ (list_tlds) - | extend DNS_TimeGenerated = TimeGenerated - ) on $left.DomainName==$right.Name - | where DNS_TimeGenerated >= TimeGenerated and DNS_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, DNS_TimeGenerated, Computer, ClientIP, Name, QueryType - | extend timestamp = DNS_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url + let dt_lookBack = 1h; + let ioc_lookBack = 14d; + //Create a list of TLDs in our threat feed for later validation + let list_tlds = ThreatIntelligenceIndicator + | where TimeGenerated > ago(ioc_lookBack) + | where isnotempty(DomainName) + | extend parts = split(DomainName, '.') + | extend tld = parts[(array_length(parts)-1)] + | summarize count() by tostring(tld) + | summarize make_list(tld); + ThreatIntelligenceIndicator + | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() + | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | where Active == true + // Picking up only IOC's that contain the entities we want + | where isnotempty(DomainName) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + DnsEvents + | where TimeGenerated > ago(dt_lookBack) + //Extract domain patterns from syslog message + | where isnotempty(Name) + | extend parts = split(Name, '.') + //Split out the TLD + | extend tld = parts[(array_length(parts)-1)] + //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed + | where tld in~ (list_tlds) + | extend DNS_TimeGenerated = TimeGenerated + ) + on $left.DomainName==$right.Name + | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated, *) by IndicatorId + | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, DNS_TimeGenerated, Computer, ClientIP, Name, QueryType + | extend timestamp = DNS_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url entityMappings: - entityType: Host fieldMappings: @@ -66,5 +67,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml b/Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml index 88d236d83f2..25452aefda8 100644 --- a/Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml +++ b/Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml @@ -21,48 +21,49 @@ tactics: - Impact query: | - let dt_lookBack = 1h; - let ioc_lookBack = 14d; - //Create a list of TLDs in our threat feed for later validation of extracted domains - let list_tlds = ThreatIntelligenceIndicator - | where TimeGenerated > ago(ioc_lookBack) - | where isnotempty(DomainName) - | extend DomainName = tolower(DomainName) - | extend parts = split(DomainName, '.') - | extend tld = parts[(array_length(parts)-1)] - | summarize count() by tostring(tld) - | summarize make_list(tld); - ThreatIntelligenceIndicator - | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | where Active == true - // Picking up only IOC's that contain the entities we want - | where isnotempty(DomainName) - | join ( - CommonSecurityLog - | extend IngestionTime = ingestion_time() - | where IngestionTime > ago(dt_lookBack) - | where DeviceVendor =~ 'Palo Alto Networks' - | where DeviceEventClassID =~ 'url' - //Uncomment the line below to only alert on allowed connections - //| where DeviceAction !~ "block-url" - //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions - | extend PA_Url = columnifexists("RequestURL", "None") - | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) - | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) - | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host)) - | where isnotempty(Domain) - | extend Domain = tolower(Domain) - | extend parts = split(Domain, '.') - //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on - | extend tld = parts[(array_length(parts)-1)] - //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match - | where tld in~ (list_tlds) - | extend CommonSecurityLog_TimeGenerated = TimeGenerated - ) on $left.DomainName==$right.Domain - | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime - | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod - | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url + let dt_lookBack = 1h; + let ioc_lookBack = 14d; + //Create a list of TLDs in our threat feed for later validation of extracted domains + let list_tlds = ThreatIntelligenceIndicator + | where TimeGenerated > ago(ioc_lookBack) + | where isnotempty(DomainName) + | extend DomainName = tolower(DomainName) + | extend parts = split(DomainName, '.') + | extend tld = parts[(array_length(parts)-1)] + | summarize count() by tostring(tld) + | summarize make_list(tld); + ThreatIntelligenceIndicator + | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() + | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | where Active == true + // Picking up only IOC's that contain the entities we want + | where isnotempty(DomainName) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + CommonSecurityLog + | extend IngestionTime = ingestion_time() + | where IngestionTime > ago(dt_lookBack) + | where DeviceVendor =~ 'Palo Alto Networks' + | where DeviceEventClassID =~ 'url' + //Uncomment the line below to only alert on allowed connections + //| where DeviceAction !~ "block-url" + //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions + | extend PA_Url = columnifexists("RequestURL", "None") + | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) + | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) + | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host)) + | where isnotempty(Domain) + | extend Domain = tolower(Domain) + | extend parts = split(Domain, '.') + //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on + | extend tld = parts[(array_length(parts)-1)] + //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match + | where tld in~ (list_tlds) + | extend CommonSecurityLog_TimeGenerated = TimeGenerated + ) + on $left.DomainName==$right.Domain + | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod + | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url entityMappings: - entityType: Host fieldMappings: @@ -76,5 +77,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.0 -kind: Scheduled \ No newline at end of file +version: 1.1.1 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/DomainEntity_SecurityAlert.yaml b/Detections/ThreatIntelligenceIndicator/DomainEntity_SecurityAlert.yaml index 3e39ef38e58..cdbd3a83ee8 100644 --- a/Detections/ThreatIntelligenceIndicator/DomainEntity_SecurityAlert.yaml +++ b/Detections/ThreatIntelligenceIndicator/DomainEntity_SecurityAlert.yaml @@ -24,48 +24,49 @@ tactics: - Impact query: | - let dt_lookBack = 1h; - let ioc_lookBack = 14d; - //Create a list of TLDs in our threat feed for later validation - let list_tlds = ThreatIntelligenceIndicator - | where TimeGenerated > ago(ioc_lookBack) - | where isnotempty(DomainName) - | extend parts = split(DomainName, '.') - | extend tld = parts[(array_length(parts)-1)] - | summarize count() by tostring(tld) - | summarize make_list(tld); - ThreatIntelligenceIndicator - | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | where Active == true - // Picking up only IOC's that contain the entities we want - | where isnotempty(DomainName) - | join ( - SecurityAlert - | where TimeGenerated > ago(dt_lookBack) - | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false) - | where MSTI == false - //Extract domain patterns from message - | extend domain = extract("(([a-z0-9]+(-[a-z0-9]+)*\\.)+[a-z]{2,})", 1, tolower(Entities)) - | where isnotempty(domain) - | extend parts = split(domain, '.') - //Split out the TLD - | extend tld = parts[(array_length(parts)-1)] - //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed - | where tld in~ (list_tlds) - // Converting Entities into dynamic data type and use mv-expand to unpack the array - | extend EntitiesDynamicArray = parse_json(Entities) | mv-expand EntitiesDynamicArray - // Parsing relevant entity column extract hostname and IP address - | extend EntityType = tostring(parse_json(EntitiesDynamicArray).Type), EntityAddress = tostring(EntitiesDynamicArray.Address), EntityHostName = tostring(EntitiesDynamicArray.HostName) - | extend HostName = iif(EntityType == 'host', EntityHostName, '') - | extend IP_addr = iif(EntityType == 'ip', EntityAddress, '') - | extend Alert_TimeGenerated = TimeGenerated - | extend Alert_Description = Description - ) on $left.DomainName==$right.domain - | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated, AlertName, Alert_Description, ProviderName, AlertSeverity, ConfidenceLevel, HostName, IP_addr, Url - | extend timestamp = Alert_TimeGenerated, HostCustomEntity = HostName, IPCustomEntity = IP_addr, URLCustomEntity = Url + let dt_lookBack = 1h; + let ioc_lookBack = 14d; + //Create a list of TLDs in our threat feed for later validation + let list_tlds = ThreatIntelligenceIndicator + | where TimeGenerated > ago(ioc_lookBack) + | where isnotempty(DomainName) + | extend parts = split(DomainName, '.') + | extend tld = parts[(array_length(parts)-1)] + | summarize count() by tostring(tld) + | summarize make_list(tld); + ThreatIntelligenceIndicator + | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() + | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | where Active == true + // Picking up only IOC's that contain the entities we want + | where isnotempty(DomainName) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + SecurityAlert + | where TimeGenerated > ago(dt_lookBack) + | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false) + | where MSTI == false + //Extract domain patterns from message + | extend domain = extract("(([a-z0-9]+(-[a-z0-9]+)*\\.)+[a-z]{2,})", 1, tolower(Entities)) + | where isnotempty(domain) + | extend parts = split(domain, '.') + //Split out the TLD + | extend tld = parts[(array_length(parts)-1)] + //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed + | where tld in~ (list_tlds) + // Converting Entities into dynamic data type and use mv-expand to unpack the array + | extend EntitiesDynamicArray = parse_json(Entities) | mv-expand EntitiesDynamicArray + // Parsing relevant entity column extract hostname and IP address + | extend EntityType = tostring(parse_json(EntitiesDynamicArray).Type), EntityAddress = tostring(EntitiesDynamicArray.Address), EntityHostName = tostring(EntitiesDynamicArray.HostName) + | extend HostName = iif(EntityType == 'host', EntityHostName, '') + | extend IP_addr = iif(EntityType == 'ip', EntityAddress, '') + | extend Alert_TimeGenerated = TimeGenerated + | extend Alert_Description = Description + ) + on $left.DomainName==$right.domain + | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId + | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated, AlertName, Alert_Description, ProviderName, AlertSeverity, ConfidenceLevel, HostName, IP_addr, Url + | extend timestamp = Alert_TimeGenerated, HostCustomEntity = HostName, IPCustomEntity = IP_addr, URLCustomEntity = Url entityMappings: - entityType: Host fieldMappings: @@ -79,5 +80,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.2 -kind: Scheduled \ No newline at end of file +version: 1.1.3 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/DomainEntity_Syslog.yaml b/Detections/ThreatIntelligenceIndicator/DomainEntity_Syslog.yaml index 256317b2186..4a93f8749f7 100644 --- a/Detections/ThreatIntelligenceIndicator/DomainEntity_Syslog.yaml +++ b/Detections/ThreatIntelligenceIndicator/DomainEntity_Syslog.yaml @@ -21,39 +21,40 @@ tactics: - Impact query: | - let dt_lookBack = 1h; - let ioc_lookBack = 14d; - //Create a list of TLDs in our threat feed for later validation - let list_tlds = ThreatIntelligenceIndicator - | where TimeGenerated > ago(ioc_lookBack) - | where isnotempty(DomainName) - | extend parts = split(DomainName, '.') - | extend tld = parts[(array_length(parts)-1)] - | summarize count() by tostring(tld) - | summarize make_list(tld); - ThreatIntelligenceIndicator - | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | where Active == true - // Picking up only IOC's that contain the entities we want - | where isnotempty(DomainName) - | join ( - Syslog - | where TimeGenerated > ago(dt_lookBack) - //Extract domain patterns from syslog message - | extend domain = extract("(([a-z0-9]+(-[a-z0-9]+)*\\.)+[a-z]{2,})",1, tolower(SyslogMessage)) - | where isnotempty(domain) - | extend parts = split(domain, '.') - //Split out the TLD - | extend tld = parts[(array_length(parts)-1)] - //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed - | where tld in~ (list_tlds) - | extend Syslog_TimeGenerated = TimeGenerated - ) on $left.DomainName==$right.domain - | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId - | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, domain, HostIP, Url - | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url + let dt_lookBack = 1h; + let ioc_lookBack = 14d; + //Create a list of TLDs in our threat feed for later validation + let list_tlds = ThreatIntelligenceIndicator + | where TimeGenerated > ago(ioc_lookBack) + | where isnotempty(DomainName) + | extend parts = split(DomainName, '.') + | extend tld = parts[(array_length(parts)-1)] + | summarize count() by tostring(tld) + | summarize make_list(tld); + ThreatIntelligenceIndicator + | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now() + | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | where Active == true + // Picking up only IOC's that contain the entities we want + | where isnotempty(DomainName) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + Syslog + | where TimeGenerated > ago(dt_lookBack) + //Extract domain patterns from syslog message + | extend domain = extract("(([a-z0-9]+(-[a-z0-9]+)*\\.)+[a-z]{2,})",1, tolower(SyslogMessage)) + | where isnotempty(domain) + | extend parts = split(domain, '.') + //Split out the TLD + | extend tld = parts[(array_length(parts)-1)] + //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed + | where tld in~ (list_tlds) + | extend Syslog_TimeGenerated = TimeGenerated + ) + on $left.DomainName==$right.domain + | summarize Syslog_TimeGenerated = arg_max(Syslog_TimeGenerated, *) by IndicatorId + | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, domain, HostIP, Url + | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url entityMappings: - entityType: Host fieldMappings: @@ -67,5 +68,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_AzureActivity.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_AzureActivity.yaml index 99a6177c966..3555c00d2c4 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_AzureActivity.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_AzureActivity.yaml @@ -30,15 +30,15 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AzureActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(Caller) | extend Caller = tolower(Caller) | where Caller matches regex emailregex | extend AzureActivity_TimeGenerated = TimeGenerated ) on $left.EmailSenderAddress == $right.Caller - | where AzureActivity_TimeGenerated >= TimeGenerated and AzureActivity_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize AzureActivity_TimeGenerated = arg_max(AzureActivity_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, AzureActivity_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, Caller, Level, CallerIpAddress, CategoryValue, OperationNameValue, ActivityStatusValue, ResourceGroup, SubscriptionId @@ -56,5 +56,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_OfficeActivity.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_OfficeActivity.yaml index ff17cd3dd24..8373931fff6 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_OfficeActivity.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_OfficeActivity.yaml @@ -30,14 +30,14 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( OfficeActivity | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserId) | where UserId matches regex emailregex | extend OfficeActivity_TimeGenerated = TimeGenerated ) on $left.EmailSenderAddress == $right.UserId - | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, OfficeActivity_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, UserId, ClientIP, Operation, UserType, RecordType, OfficeWorkload, Parameters | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = UserId, IPCustomEntity = ClientIP, URLCustomEntity = Url @@ -54,5 +54,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_PaloAlto.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_PaloAlto.yaml index 0bbfc026907..c97586879a4 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_PaloAlto.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_PaloAlto.yaml @@ -30,7 +30,8 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) and isnotempty(DestinationUserID) // Filtering PAN Logs for specific event type to match relevant email entities | where DeviceVendor == "Palo Alto Networks" and DeviceEventClassID == "wildfire" and ApplicationProtocol in ("smtp","pop3") @@ -39,8 +40,7 @@ query: | | extend CommonSecurityLog_TimeGenerated = TimeGenerated ) on $left.EmailSenderAddress == $right.DestinationUserID - | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, DestinationUserID, DeviceEventClassID, LogSeverity, DeviceAction, SourceIP, SourcePort, DestinationIP, DestinationPort, Protocol, ApplicationProtocol @@ -58,5 +58,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityAlert.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityAlert.yaml index 2242c2e22a1..f9f9be31619 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityAlert.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityAlert.yaml @@ -30,7 +30,8 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( SecurityAlert | where TimeGenerated >= ago(dt_lookBack) | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false) @@ -46,8 +47,7 @@ query: | | extend Alert_TimeGenerated = TimeGenerated ) on $left.EmailSenderAddress == $right.EntityEmail - | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, EntityEmail, AlertName, AlertType, AlertSeverity, Entities, ProviderName, VendorName @@ -61,5 +61,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.2 -kind: Scheduled \ No newline at end of file +version: 1.1.3 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityEvent.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityEvent.yaml index d13bd818a84..9ef9323f1b2 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityEvent.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_SecurityEvent.yaml @@ -30,7 +30,8 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( SecurityEvent | where TimeGenerated >= ago(dt_lookBack) and isnotempty(TargetUserName) //Normalizing the column to lower case for exact match with EmailSenderAddress column | extend TargetUserName = tolower(TargetUserName) @@ -38,8 +39,7 @@ query: | | extend SecurityEvent_TimeGenerated = TimeGenerated ) on $left.EmailSenderAddress == $right.TargetUserName - | where SecurityEvent_TimeGenerated >= TimeGenerated and SecurityEvent_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize SecurityEvent_TimeGenerated = arg_max(SecurityEvent_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, SecurityEvent_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, Computer, EventID, TargetUserName, Activity, IpAddress, AccountType, LogonTypeName, LogonProcessName, Status, SubStatus @@ -61,5 +61,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/EmailEntity_SigninLogs.yaml b/Detections/ThreatIntelligenceIndicator/EmailEntity_SigninLogs.yaml index 7cfe0d2676d..4af79c91c2f 100644 --- a/Detections/ThreatIntelligenceIndicator/EmailEntity_SigninLogs.yaml +++ b/Detections/ThreatIntelligenceIndicator/EmailEntity_SigninLogs.yaml @@ -34,7 +34,8 @@ query: | | where Active == true //Filtering the table for Email related IOCs | where isnotempty(EmailSenderAddress) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( table(tableName) | where TimeGenerated >= ago(dt_lookBack) and isnotempty(UserPrincipalName) //Normalizing the column to lower case for exact match with EmailSenderAddress column | extend UserPrincipalName = tolower(UserPrincipalName) @@ -46,8 +47,7 @@ query: | | extend SigninLogs_TimeGenerated = TimeGenerated, Type = Type ) on $left.EmailSenderAddress == $right.UserPrincipalName - | where SigninLogs_TimeGenerated >= TimeGenerated and SigninLogs_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize SigninLogs_TimeGenerated = arg_max(SigninLogs_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, SigninLogs_TimeGenerated, EmailSenderName, EmailRecipient, EmailSourceDomain, EmailSourceIpAddress, EmailSubject, FileHashValue, FileHashType, IPAddress, UserPrincipalName, AppDisplayName, StatusCode, StatusDetails, NetworkIP, NetworkDestinationIP, NetworkSourceIP, Type @@ -69,5 +69,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/FileHashEntity_CommonSecurityLog.yaml b/Detections/ThreatIntelligenceIndicator/FileHashEntity_CommonSecurityLog.yaml index eb519036420..17b0f96b08a 100644 --- a/Detections/ThreatIntelligenceIndicator/FileHashEntity_CommonSecurityLog.yaml +++ b/Detections/ThreatIntelligenceIndicator/FileHashEntity_CommonSecurityLog.yaml @@ -30,15 +30,15 @@ query: | | where isnotempty(FileHashValue); // Handle matches against both lower case and uppercase versions of the hash: ( fileHashIndicators | extend FileHashValue = tolower(FileHashValue) - |union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue))) - | join ( - CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) - | where isnotempty(FileHash) - | extend CommonSecurityLog_TimeGenerated = TimeGenerated + | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue))) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) + | where isnotempty(FileHash) + | extend CommonSecurityLog_TimeGenerated = TimeGenerated ) on $left.FileHashValue == $right.FileHash - | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize CommonSecurityLog_TimeGenerated = arg_max(CommonSecurityLog_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, SourceIP, SourcePort, DestinationIP, DestinationPort, SourceUserID, SourceUserName, DeviceName, DeviceAction, RequestURL, DestinationUserName, DestinationUserID, ApplicationProtocol, Activity @@ -60,5 +60,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/FileHashEntity_Covid19_CommonSecurityLog.yaml b/Detections/ThreatIntelligenceIndicator/FileHashEntity_Covid19_CommonSecurityLog.yaml index f5d504a1345..ccd35b27110 100644 --- a/Detections/ThreatIntelligenceIndicator/FileHashEntity_Covid19_CommonSecurityLog.yaml +++ b/Detections/ThreatIntelligenceIndicator/FileHashEntity_Covid19_CommonSecurityLog.yaml @@ -23,10 +23,11 @@ query: | // Handle matches against both lower case and uppercase versions of the hash: ( fileHashIndicators | extend FileHashValue = tolower(FileHashValue) | union (fileHashIndicators | extend FileHashValue = toupper(FileHashValue))) - | join ( - CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) - | where isnotempty(FileHash) - | extend CommonSecurityLog_TimeGenerated = TimeGenerated + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + CommonSecurityLog | where TimeGenerated >= ago(dt_lookBack) + | where isnotempty(FileHash) + | extend CommonSecurityLog_TimeGenerated = TimeGenerated ) on $left.FileHashValue == $right.FileHash | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by FileHashValue @@ -47,5 +48,5 @@ entityMappings: fieldMappings: - identifier: Address columnName: IPCustomEntity -version: 1.0.0 +version: 1.0.1 kind: Scheduled \ No newline at end of file diff --git a/Detections/ThreatIntelligenceIndicator/FileHashEntity_SecurityEvent.yaml b/Detections/ThreatIntelligenceIndicator/FileHashEntity_SecurityEvent.yaml index b20affd6b2f..66286ae20c0 100644 --- a/Detections/ThreatIntelligenceIndicator/FileHashEntity_SecurityEvent.yaml +++ b/Detections/ThreatIntelligenceIndicator/FileHashEntity_SecurityEvent.yaml @@ -28,15 +28,15 @@ query: | | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId | where Active == true | where isnotempty(FileHashValue) - | join ( - SecurityEvent | where TimeGenerated >= ago(dt_lookBack) - | where EventID in ("8003","8002","8005") - | where isnotempty(FileHash) - | extend SecurityEvent_TimeGenerated = TimeGenerated, Event = EventID + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + SecurityEvent | where TimeGenerated >= ago(dt_lookBack) + | where EventID in ("8003","8002","8005") + | where isnotempty(FileHash) + | extend SecurityEvent_TimeGenerated = TimeGenerated, Event = EventID ) on $left.FileHashValue == $right.FileHash - | where SecurityEvent_TimeGenerated >= TimeGenerated and SecurityEvent_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize SecurityEvent_TimeGenerated = arg_max(SecurityEvent_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, SecurityEvent_TimeGenerated, Process, FileHash, Computer, Account, Event | extend timestamp = SecurityEvent_TimeGenerated, AccountCustomEntity = Account, HostCustomEntity = Computer, URLCustomEntity = Url @@ -53,5 +53,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AWSCloudTrail.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AWSCloudTrail.yaml index a4285c26198..7cc245578e5 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AWSCloudTrail.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AWSCloudTrail.yaml @@ -34,14 +34,14 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AWSCloudTrail | where TimeGenerated >= ago(dt_lookBack) // renaming time column so it is clear the log this came from | extend AWSCloudTrail_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.SourceIpAddress - | where AWSCloudTrail_TimeGenerated >= TimeGenerated and AWSCloudTrail_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize AWSCloudTrail_TimeGenerated = arg_max(AWSCloudTrail_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AWSCloudTrail_TimeGenerated, TI_ipEntity, EventName, EventTypeName, UserIdentityAccountId, UserIdentityPrincipalid, UserIdentityUserName, SourceIpAddress, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress @@ -59,5 +59,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml index 9a38686f011..46f956f4523 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AppServiceHTTPLogs.yaml @@ -31,15 +31,16 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AppServiceHTTPLogs | where TimeGenerated >= ago(dt_lookBack) - | where isnotempty(CIp) - | extend WebApp = split(_ResourceId, '/')[8] - // renaming time column so it is clear the log this came from - | extend AppService_TimeGenerated = TimeGenerated + | where isnotempty(CIp) + | extend WebApp = split(_ResourceId, '/')[8] + // renaming time column so it is clear the log this came from + | extend AppService_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.CIp - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize AppService_TimeGenerated = arg_max(AppService_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AppService_TimeGenerated, TI_ipEntity, CsUsername, WebApp = split(_ResourceId, '/')[8], CIp, CsHost, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = AppService_TimeGenerated, AccountCustomEntity = CsUsername, IPCustomEntity = CIp, URLCustomEntity = CsHost entityMappings: @@ -59,5 +60,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureActivity.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureActivity.yaml index 9a4e2d62bab..2172a0f4818 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureActivity.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureActivity.yaml @@ -34,14 +34,14 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AzureActivity | where TimeGenerated >= ago(dt_lookBack) // renaming time column so it is clear the log this came from | extend AzureActivity_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.CallerIpAddress - | where AzureActivity_TimeGenerated >= TimeGenerated and AzureActivity_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize AzureActivity_TimeGenerated = arg_max(AzureActivity_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AzureActivity_TimeGenerated, TI_ipEntity, CallerIpAddress, Caller, OperationNameValue, ActivityStatusValue, CategoryValue, ResourceId, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = AzureActivity_TimeGenerated, IPCustomEntity = CallerIpAddress, AccountCustomEntity = Caller, URLCustomEntity = Url @@ -58,5 +58,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureFirewall.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureFirewall.yaml index f4a850756cc..39ad268fcc1 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureFirewall.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureFirewall.yaml @@ -34,7 +34,8 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AzureDiagnostics | where TimeGenerated >= ago(dt_lookBack) | where OperationName in ("AzureFirewallApplicationRuleLog","AzureFirewallNetworkRuleLog") @@ -45,7 +46,6 @@ query: | | project-rename AzureFirewall_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.DestinationAddress - | where AzureFirewall_TimeGenerated < ExpirationDateTime | summarize AzureFirewall_TimeGenerated = arg_max(AzureFirewall_TimeGenerated, *) by IndicatorId, SourceAddress | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, DomainName, ExpirationDateTime, ConfidenceScore, AzureFirewall_TimeGenerated, TI_ipEntity, Resource, Category, msg_s, SourceAddress, DestinationAddress, Action, Protocol, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress @@ -59,5 +59,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.0.0 +version: 1.0.1 kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureKeyVault.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureKeyVault.yaml index cac67b08732..ba3911985db 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureKeyVault.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureKeyVault.yaml @@ -31,13 +31,12 @@ query: | | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated | join kind=innerunique ( - AzureDiagnostics - | where ResourceType =~ "VAULTS" - | where TimeGenerated >= ago(dt_lookBack) - | extend KeyVaultEvents_TimeGenerated = TimeGenerated, ClientIP = CallerIPAddress + AzureDiagnostics + | where ResourceType =~ "VAULTS" + | where TimeGenerated >= ago(dt_lookBack) + | extend KeyVaultEvents_TimeGenerated = TimeGenerated, ClientIP = CallerIPAddress ) on $left.TI_ipEntity == $right.ClientIP - | where KeyVaultEvents_TimeGenerated < ExpirationDateTime | summarize KeyVaultEvents_TimeGenerated = arg_max(KeyVaultEvents_TimeGenerated, *) by IndicatorId, ClientIP | project KeyVaultEvents_TimeGenerated , Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TI_ipEntity, ClientIP, ResourceId, SubscriptionId, OperationName, ResultType, CorrelationId, id_s, clientInfo_s, httpStatusCode_d, identity_claim_appid_g, identity_claim_http_schemas_microsoft_com_identity_claims_objectidentifier_g @@ -51,5 +50,5 @@ entityMappings: fieldMappings: - identifier: ResourceId columnName: ResourceId -version: 1.0.0 +version: 1.0.1 kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureNetworkAnalytics.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureNetworkAnalytics.yaml index 877f61f65f2..2c3d07450c9 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureNetworkAnalytics.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureNetworkAnalytics.yaml @@ -31,7 +31,8 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( AzureNetworkAnalytics_CL | where TimeGenerated >= ago(dt_lookBack) // renaming time column so it is clear the log this came from @@ -41,8 +42,7 @@ query: | | extend PIP = tostring(PIPs[0]) ) on $left.TI_ipEntity == $right.PIP - | where AzureNetworkAnalytics_CL_TimeGenerated >= TimeGenerated and AzureNetworkAnalytics_CL_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize AzureNetworkAnalytics_CL_TimeGenerated = arg_max(AzureNetworkAnalytics_CL_TimeGenerated, *) by IndicatorId // Set to alert on Allowed NSG Flows from TI Public IP IOC | where FlowStatus_s == "A" | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, AzureNetworkAnalytics_CL_TimeGenerated, @@ -61,5 +61,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureSQL.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureSQL.yaml index 999209c70eb..240026477e5 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_AzureSQL.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_AzureSQL.yaml @@ -28,17 +28,16 @@ query: | | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated | join kind=innerunique ( - AzureDiagnostics - | where TimeGenerated >= ago(dt_lookBack) - | where ResourceProvider == 'MICROSOFT.SQL' - | where Category == 'SQLSecurityAuditEvents' - | extend SQLSecurityAuditEvents_TimeGenerated = TimeGenerated - // projecting fields with column if exists as this is in AzureDiag and if the event is not in the table, then queries will fail due to event specific schemas - | extend ClientIP = column_ifexists("client_ip_s", "Not Available"), Action = column_ifexists("action_name_s", "Not Available"), - Application = column_ifexists("application_name_s", "Not Available"), HostName = column_ifexists("host_name_s", "Not Available") + AzureDiagnostics + | where TimeGenerated >= ago(dt_lookBack) + | where ResourceProvider == 'MICROSOFT.SQL' + | where Category == 'SQLSecurityAuditEvents' + | extend SQLSecurityAuditEvents_TimeGenerated = TimeGenerated + // projecting fields with column if exists as this is in AzureDiag and if the event is not in the table, then queries will fail due to event specific schemas + | extend ClientIP = column_ifexists("client_ip_s", "Not Available"), Action = column_ifexists("action_name_s", "Not Available"), + Application = column_ifexists("application_name_s", "Not Available"), HostName = column_ifexists("host_name_s", "Not Available") ) on $left.TI_ipEntity == $right.ClientIP - | where SQLSecurityAuditEvents_TimeGenerated < ExpirationDateTime | summarize SQLSecurityAuditEvents_TimeGenerated = arg_max(SQLSecurityAuditEvents_TimeGenerated, *) by IndicatorId, ClientIP | project SQLSecurityAuditEvents_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TI_ipEntity, ResourceId, ClientIP, Action, Application, HostName, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress @@ -48,5 +47,5 @@ entityMappings: fieldMappings: - identifier: Address columnName: ClientIP -version: 1.0.0 +version: 1.0.1 kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_DnsEvents.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_DnsEvents.yaml index 034e2c57f05..4d34b45098e 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_DnsEvents.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_DnsEvents.yaml @@ -34,7 +34,8 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( DnsEvents | where TimeGenerated >= ago(dt_lookBack) | where SubType =~ "LookupQuery" and isnotempty(IPAddresses) | extend SingleIP = split(IPAddresses, ",") @@ -44,8 +45,7 @@ query: | | extend DNS_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.SingleIP - | where DNS_TimeGenerated >= TimeGenerated and DNS_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, DomainName, ExpirationDateTime, ConfidenceScore, DNS_TimeGenerated, TI_ipEntity, Computer, EventId, SubType, ClientIP, Name, IPAddresses, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = DNS_TimeGenerated, IPCustomEntity = ClientIP, HostCustomEntity = Computer, URLCustomEntity = Url @@ -62,5 +62,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.2 +version: 1.1.3 kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_OfficeActivity.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_OfficeActivity.yaml index 959dd8463b9..bc4fa1c8435 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_OfficeActivity.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_OfficeActivity.yaml @@ -34,14 +34,14 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( OfficeActivity | where TimeGenerated >= ago(dt_lookBack) // renaming time column so it is clear the log this came from | extend OfficeActivity_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.ClientIP - | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, OfficeActivity_TimeGenerated, TI_ipEntity, ClientIP, UserId, Operation, ResultStatus, RecordType, OfficeObjectId, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = OfficeActivity_TimeGenerated, IPCustomEntity = ClientIP, AccountCustomEntity = UserId, URLCustomEntity = Url @@ -58,5 +58,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_VMConnection.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_VMConnection.yaml index 4d7f9979ec2..c8050fbffb9 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_VMConnection.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_VMConnection.yaml @@ -34,15 +34,15 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( VMConnection | where TimeGenerated >= ago(dt_lookBack) // renaming time column so it is clear the log this came from | extend VMConnection_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.RemoteIp - | where VMConnection_TimeGenerated >= TimeGenerated and VMConnection_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize VMConnection_TimeGenerated = arg_max(VMConnection_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, VMConnection_TimeGenerated, TI_ipEntity, Computer, Direction, ProcessName, SourceIp, DestinationIp, RemoteIp, Protocol, DestinationPort, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = VMConnection_TimeGenerated, IPCustomEntity = RemoteIp, HostCustomEntity = Computer, URLCustomEntity = Url @@ -59,5 +59,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml index 90913fbd877..fc5133f4ca2 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_W3CIISLog.yaml @@ -34,7 +34,8 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( W3CIISLog | where TimeGenerated >= ago(dt_lookBack) | where isnotempty(cIP) @@ -42,8 +43,7 @@ query: | | extend W3CIISLog_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.cIP - | where W3CIISLog_TimeGenerated >= TimeGenerated and W3CIISLog_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize W3CIISLog_TimeGenerated = arg_max(W3CIISLog_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, W3CIISLog_TimeGenerated, TI_ipEntity, Computer, sSiteName, cIP, sIP, sPort, csMethod, csUserName, scStatus, scSubStatus, scWin32Status, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress @@ -65,5 +65,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPEntity_WireData.yaml b/Detections/ThreatIntelligenceIndicator/IPEntity_WireData.yaml index 390b5ab7634..551dd956c35 100644 --- a/Detections/ThreatIntelligenceIndicator/IPEntity_WireData.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPEntity_WireData.yaml @@ -34,15 +34,15 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) - | join ( + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( WireData | where TimeGenerated >= ago(dt_lookBack) | where isnotempty(RemoteIP) // renaming time column so it is clear the log this came from | extend WireData_TimeGenerated = TimeGenerated ) on $left.TI_ipEntity == $right.RemoteIP - | where WireData_TimeGenerated >= TimeGenerated and WireData_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + | summarize WireData_TimeGenerated = arg_max(WireData_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, WireData_TimeGenerated, TI_ipEntity, Computer, LocalIP, RemoteIP, ProcessName, ApplicationProtocol, LocalPortNumber, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress | extend timestamp = WireData_TimeGenerated, IPCustomEntity = RemoteIP, HostCustomEntity = Computer, URLCustomEntity = Url @@ -59,5 +59,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/IPentity_SigninLogs.yaml b/Detections/ThreatIntelligenceIndicator/IPentity_SigninLogs.yaml index 1fa968a452b..6f7ebd9f17a 100644 --- a/Detections/ThreatIntelligenceIndicator/IPentity_SigninLogs.yaml +++ b/Detections/ThreatIntelligenceIndicator/IPentity_SigninLogs.yaml @@ -38,6 +38,7 @@ query: | | extend TI_ipEntity = iff(isnotempty(NetworkIP), NetworkIP, NetworkDestinationIP) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(NetworkSourceIP), NetworkSourceIP, TI_ipEntity) | extend TI_ipEntity = iff(isempty(TI_ipEntity) and isnotempty(EmailSourceIpAddress), EmailSourceIpAddress, TI_ipEntity) + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated | join kind=innerunique ( table(tableName) | where TimeGenerated >= ago(dt_lookBack) | extend Status = todynamic(Status), LocationDetails = todynamic(LocationDetails) @@ -47,7 +48,6 @@ query: | | extend SigninLogs_TimeGenerated = TimeGenerated, Type = Type ) on $left.TI_ipEntity == $right.IPAddress - | where SigninLogs_TimeGenerated < ExpirationDateTime | summarize SigninLogs_TimeGenerated = arg_max(SigninLogs_TimeGenerated, *) by IndicatorId, IPAddress | project SigninLogs_TimeGenerated, Description, ActivityGroupNames, IndicatorId, ThreatType, Url, ExpirationDateTime, ConfidenceScore, TI_ipEntity, IPAddress, UserPrincipalName, AppDisplayName, StatusCode, StatusDetails, StatusReason, NetworkIP, NetworkDestinationIP, NetworkSourceIP, EmailSourceIpAddress, Type @@ -69,5 +69,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.2 +version: 1.1.3 kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/URLEntity_AuditLogs.yaml b/Detections/ThreatIntelligenceIndicator/URLEntity_AuditLogs.yaml index d1bcfbe78b1..030e8fcbd19 100644 --- a/Detections/ThreatIntelligenceIndicator/URLEntity_AuditLogs.yaml +++ b/Detections/ThreatIntelligenceIndicator/URLEntity_AuditLogs.yaml @@ -29,18 +29,19 @@ query: | | where Active == true // Picking up only IOC's that contain the entities we want | where isnotempty(Url) - | join ( - AuditLogs - | where TimeGenerated >= ago(dt_lookBack) - // Extract the URL that is contained within the JSON data - | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);", 1,tostring(TargetResources)) - | where isnotempty(Url) - | extend userPrincipalName = tostring(parse_json(tostring(InitiatedBy.user)).userPrincipalName) - | extend TargetResourceDisplayName = tostring(TargetResources[0].displayName) - | extend Audit_TimeGenerated = TimeGenerated - ) on Url - | where Audit_TimeGenerated >= TimeGenerated and Audit_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + AuditLogs + | where TimeGenerated >= ago(dt_lookBack) + // Extract the URL that is contained within the JSON data + | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);", 1,tostring(TargetResources)) + | where isnotempty(Url) + | extend userPrincipalName = tostring(parse_json(tostring(InitiatedBy.user)).userPrincipalName) + | extend TargetResourceDisplayName = tostring(TargetResources[0].displayName) + | extend Audit_TimeGenerated = TimeGenerated + ) + on Url + | summarize Audit_TimeGenerated = arg_max(Audit_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Audit_TimeGenerated, OperationName, Identity, userPrincipalName, TargetResourceDisplayName, Url | extend timestamp = Audit_TimeGenerated, AccountCustomEntity = userPrincipalName, HostCustomEntity = TargetResourceDisplayName, URLCustomEntity = Url @@ -57,5 +58,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/URLEntity_OfficeActivity.yaml b/Detections/ThreatIntelligenceIndicator/URLEntity_OfficeActivity.yaml index 1f152fcfaa7..b84510fe764 100644 --- a/Detections/ThreatIntelligenceIndicator/URLEntity_OfficeActivity.yaml +++ b/Detections/ThreatIntelligenceIndicator/URLEntity_OfficeActivity.yaml @@ -26,20 +26,21 @@ query: | | where Active == true // Picking up only IOC's that contain the entities we want | where isnotempty(Url) - | join ( - OfficeActivity - | where TimeGenerated >= ago(dt_lookBack) - //Extract the Url from a number of potential fields - | extend Url = iif(OfficeWorkload == "AzureActiveDirectory",extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);", 1,ModifiedProperties),tostring(parse_json(ModifiedProperties)[12].NewValue)) - | where isnotempty(Url) - // Ensure we get a clean URL - | extend Url = tostring(split(Url, ';')[0]) - | extend OfficeActivity_TimeGenerated = TimeGenerated - // Project a single user identity that we can use for entity mapping - | extend User = iif(isnotempty(UserId), UserId, iif(isnotempty(Actor), tostring(parse_json(Actor)[0].ID), tostring(parse_json(Parameters)[0].Vlaue))) - ) on Url - | where OfficeActivity_TimeGenerated >= TimeGenerated and OfficeActivity_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + OfficeActivity + | where TimeGenerated >= ago(dt_lookBack) + //Extract the Url from a number of potential fields + | extend Url = iif(OfficeWorkload == "AzureActiveDirectory",extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+);", 1,ModifiedProperties),tostring(parse_json(ModifiedProperties)[12].NewValue)) + | where isnotempty(Url) + // Ensure we get a clean URL + | extend Url = tostring(split(Url, ';')[0]) + | extend OfficeActivity_TimeGenerated = TimeGenerated + // Project a single user identity that we can use for entity mapping + | extend User = iif(isnotempty(UserId), UserId, iif(isnotempty(Actor), tostring(parse_json(Actor)[0].ID), tostring(parse_json(Parameters)[0].Vlaue))) + ) + on Url + | summarize OfficeActivity_TimeGenerated = arg_max(OfficeActivity_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Operation, UserType, OfficeWorkload, Parameters, OfficeActivity_TimeGenerated, Url, User | extend timestamp = OfficeActivity_TimeGenerated, AccountCustomEntity = User, URLCustomEntity = Url @@ -52,5 +53,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/URLEntity_PaloAlto.yaml b/Detections/ThreatIntelligenceIndicator/URLEntity_PaloAlto.yaml index ff4fa4d0537..bb588e25a7f 100644 --- a/Detections/ThreatIntelligenceIndicator/URLEntity_PaloAlto.yaml +++ b/Detections/ThreatIntelligenceIndicator/URLEntity_PaloAlto.yaml @@ -29,23 +29,24 @@ query: | | where Active == true // Picking up only IOC's that contain the entities we want | where isnotempty(Url) - | join ( - CommonSecurityLog - | extend IngestionTime = ingestion_time() - | where IngestionTime > ago(dt_lookBack) - // Select on Palo Alto logs - | where DeviceVendor =~ "Palo Alto Networks" - | where DeviceEventClassID =~ 'url' - //Uncomment the line below to only alert on allowed connections - //| where DeviceAction !~ "block-url" - //Select logs where URL data is populated - | extend PA_Url = columnifexists("RequestURL", "None") - | extend PA_Url = iif(isempty(PA_Url), extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) - | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) - | where isnotempty(PA_Url) - | extend CommonSecurityLog_TimeGenerated = TimeGenerated - ) on $left.Url == $right.PA_Url - | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + CommonSecurityLog + | extend IngestionTime = ingestion_time() + | where IngestionTime > ago(dt_lookBack) + // Select on Palo Alto logs + | where DeviceVendor =~ "Palo Alto Networks" + | where DeviceEventClassID =~ 'url' + //Uncomment the line below to only alert on allowed connections + //| where DeviceAction !~ "block-url" + //Select logs where URL data is populated + | extend PA_Url = columnifexists("RequestURL", "None") + | extend PA_Url = iif(isempty(PA_Url), extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url)) + | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url)) + | where isnotempty(PA_Url) + | extend CommonSecurityLog_TimeGenerated = TimeGenerated + ) + on $left.Url == $right.PA_Url | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, DeviceAction, SourceIP, CommonSecurityLog_TimeGenerated, PA_Url, DeviceName | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url entityMappings: @@ -61,5 +62,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.0 -kind: Scheduled \ No newline at end of file +version: 1.1.1 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/URLEntity_SecurityAlerts.yaml b/Detections/ThreatIntelligenceIndicator/URLEntity_SecurityAlerts.yaml index 239511f3cb2..1b5379f3fd1 100644 --- a/Detections/ThreatIntelligenceIndicator/URLEntity_SecurityAlerts.yaml +++ b/Detections/ThreatIntelligenceIndicator/URLEntity_SecurityAlerts.yaml @@ -32,21 +32,22 @@ query: | | where Active == true // Picking up only IOC's that contain the entities we want | where isnotempty(Url) - | join ( - SecurityAlert - | where TimeGenerated >= ago(dt_lookBack) - | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false) - | where MSTI == false - // Extract URL from JSON data - | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,Entities) - // We only want alerts that actually contain URL data - | where isnotempty(Url) - // Extract hostname from JSON data for entity mapping - | extend Compromised_Host = tostring(parse_json(ExtendedProperties).["Compromised Host"]) - | extend Alert_TimeGenerated = TimeGenerated - ) on Url - | where Alert_TimeGenerated >= TimeGenerated and Alert_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + SecurityAlert + | where TimeGenerated >= ago(dt_lookBack) + | extend MSTI = case(AlertName has "TI map" and VendorName == "Microsoft" and ProductName == 'Azure Sentinel', true, false) + | where MSTI == false + // Extract URL from JSON data + | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,Entities) + // We only want alerts that actually contain URL data + | where isnotempty(Url) + // Extract hostname from JSON data for entity mapping + | extend Compromised_Host = tostring(parse_json(ExtendedProperties).["Compromised Host"]) + | extend Alert_TimeGenerated = TimeGenerated + ) + on Url + | summarize Alert_TimeGenerated = arg_max(Alert_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Alert_TimeGenerated, AlertName, AlertSeverity, Description, Url, Compromised_Host | extend timestamp = Alert_TimeGenerated, HostCustomEntity = Compromised_Host, URLCustomEntity = Url @@ -59,5 +60,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.2 -kind: Scheduled \ No newline at end of file +version: 1.1.3 +kind: Scheduled diff --git a/Detections/ThreatIntelligenceIndicator/URLEntity_Syslog.yaml b/Detections/ThreatIntelligenceIndicator/URLEntity_Syslog.yaml index ae429654144..833e4a6d271 100644 --- a/Detections/ThreatIntelligenceIndicator/URLEntity_Syslog.yaml +++ b/Detections/ThreatIntelligenceIndicator/URLEntity_Syslog.yaml @@ -29,16 +29,17 @@ query: | | where Active == true // Picking up only IOC's that contain the entities we want | where isnotempty(Url) - | join ( - Syslog - | where TimeGenerated >= ago(dt_lookBack) - // Extract URL from the Syslog message but only take messages that include URLs - | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,SyslogMessage) - | where isnotempty(Url) - | extend Syslog_TimeGenerated = TimeGenerated - ) on Url - | where Syslog_TimeGenerated >= TimeGenerated and Syslog_TimeGenerated < ExpirationDateTime - | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId + // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated + | join kind=innerunique ( + Syslog + | where TimeGenerated >= ago(dt_lookBack) + // Extract URL from the Syslog message but only take messages that include URLs + | extend Url = extract("(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)", 1,SyslogMessage) + | where isnotempty(Url) + | extend Syslog_TimeGenerated = TimeGenerated + ) + on Url + | summarize Syslog_TimeGenerated = arg_max(Syslog_TimeGenerated, *) by IndicatorId | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Syslog_TimeGenerated, SyslogMessage, Computer, ProcessName, Url, HostIP | extend timestamp = Syslog_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = HostIP, URLCustomEntity = Url entityMappings: @@ -54,5 +55,5 @@ entityMappings: fieldMappings: - identifier: Url columnName: URLCustomEntity -version: 1.1.1 -kind: Scheduled \ No newline at end of file +version: 1.1.2 +kind: Scheduled