Save a summarize and use correct Time column in most TI Map Detections

Detections/ThreatIntelligenceIndicator/DomainEntity_CommonSecurityLog.yaml

@@ -17,47 +17,49 @@ triggerThreshold: 0
 tactics:
   - Impact
 query: |
-    let dt_lookBack = 1h;
-    let ioc_lookBack = 14d;
-    //Create a list of TLDs in our threat feed for later validation of extracted domains
-    let list_tlds = ThreatIntelligenceIndicator
-        | where TimeGenerated > ago(ioc_lookBack)
-        | where isnotempty(DomainName)
-        | extend DomainName = tolower(DomainName)
-        | extend parts = split(DomainName, '.')
-        | extend tld = parts[(array_length(parts)-1)]
-        | summarize count() by tostring(tld)
-        | summarize make_list(tld);
-        ThreatIntelligenceIndicator
-        | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-        | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-        | where Active == true
-        // Picking up only IOC's that contain the entities we want
-        | where isnotempty(DomainName)
-        | join (
-            CommonSecurityLog
-            | extend IngestionTime = ingestion_time()
-            | where IngestionTime > ago(dt_lookBack)
-            | where DeviceEventClassID =~ 'url'
-            //Uncomment the line below to only alert on allowed connections
-            //| where DeviceAction !~ "block-url"
-            //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
-            | extend PA_Url = columnifexists("RequestURL", "None")
-            | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
-            | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
-            | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
-            | where isnotempty(Domain)
-            | extend Domain = tolower(Domain)
-            | extend parts = split(Domain, '.')
-            //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
-            | extend tld = parts[(array_length(parts)-1)]
-            //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
-            | where tld in~ (list_tlds)
-            | extend CommonSecurityLog_TimeGenerated = TimeGenerated
-        ) on $left.DomainName==$right.Domain
-        | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-        | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
-        | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
+
+  let dt_lookBack = 1h;
+  let ioc_lookBack = 14d;
+  //Create a list of TLDs in our threat feed for later validation of extracted domains
+  let list_tlds = ThreatIntelligenceIndicator
+  | where TimeGenerated > ago(ioc_lookBack)
+  | where isnotempty(DomainName)
+  | extend DomainName = tolower(DomainName)
+  | extend parts = split(DomainName, '.')
+  | extend tld = parts[(array_length(parts)-1)]
+  | summarize count() by tostring(tld)
+  | summarize make_list(tld);
+  ThreatIntelligenceIndicator
+  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true
+  // Picking up only IOC's that contain the entities we want
+  | where isnotempty(DomainName)
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+      CommonSecurityLog
+      | extend IngestionTime = ingestion_time()
+      | where IngestionTime > ago(dt_lookBack)
+      | where DeviceEventClassID =~ 'url'
+      //Uncomment the line below to only alert on allowed connections
+      //| where DeviceAction !~ "block-url"
+      //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
+      | extend PA_Url = columnifexists("RequestURL", "None")
+      | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
+      | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
+      | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
+      | where isnotempty(Domain)
+      | extend Domain = tolower(Domain)
+      | extend parts = split(Domain, '.')
+      //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
+      | extend tld = parts[(array_length(parts)-1)]
+      //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
+      | where tld in~ (list_tlds)
+      | extend CommonSecurityLog_TimeGenerated = TimeGenerated
+  )
+  on $left.DomainName==$right.Domain
+  | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
+  | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
 entityMappings:
   - entityType: Host
     fieldMappings:
@@ -71,5 +73,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
-version: 1.1.0
-kind: Scheduled
+version: 1.1.1
+kind: Scheduled

Detections/ThreatIntelligenceIndicator/DomainEntity_DnsEvents.yaml

@@ -21,38 +21,39 @@ tactics:
   - Impact
 query: |
 
-    let dt_lookBack = 1h;
-    let ioc_lookBack = 14d;
-    //Create a list of TLDs in our threat feed for later validation
-    let list_tlds = ThreatIntelligenceIndicator
-    | where TimeGenerated > ago(ioc_lookBack)
-    | where isnotempty(DomainName)
-    | extend parts = split(DomainName, '.')
-    | extend tld = parts[(array_length(parts)-1)]
-    | summarize count() by tostring(tld)
-    | summarize make_list(tld);
-    ThreatIntelligenceIndicator
-    | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | where Active == true
-    // Picking up only IOC's that contain the entities we want
-    | where isnotempty(DomainName)
-    | join (
-         DnsEvents
-        | where TimeGenerated > ago(dt_lookBack)
-        //Extract domain patterns from syslog message
-        | where isnotempty(Name)
-        | extend parts = split(Name, '.')
-        //Split out the TLD
-        | extend tld = parts[(array_length(parts)-1)]
-        //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed
-        | where tld in~ (list_tlds)
-        | extend DNS_TimeGenerated = TimeGenerated
-    ) on $left.DomainName==$right.Name
-    | where DNS_TimeGenerated >= TimeGenerated and DNS_TimeGenerated < ExpirationDateTime
-    | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-    | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, DNS_TimeGenerated, Computer, ClientIP, Name, QueryType
-    | extend timestamp = DNS_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url
+  let dt_lookBack = 1h;
+  let ioc_lookBack = 14d;
+  //Create a list of TLDs in our threat feed for later validation
+  let list_tlds = ThreatIntelligenceIndicator
+  | where TimeGenerated > ago(ioc_lookBack)
+  | where isnotempty(DomainName)
+  | extend parts = split(DomainName, '.')
+  | extend tld = parts[(array_length(parts)-1)]
+  | summarize count() by tostring(tld)
+  | summarize make_list(tld);
+  ThreatIntelligenceIndicator
+  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true
+  // Picking up only IOC's that contain the entities we want
+  | where isnotempty(DomainName)
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+      DnsEvents
+      | where TimeGenerated > ago(dt_lookBack)
+      //Extract domain patterns from syslog message
+      | where isnotempty(Name)
+      | extend parts = split(Name, '.')
+      //Split out the TLD
+      | extend tld = parts[(array_length(parts)-1)]
+      //Validate parsed domain by checking if the TLD is in the list of TLDs in our threat feed
+      | where tld in~ (list_tlds)
+      | extend DNS_TimeGenerated = TimeGenerated
+  )
+  on $left.DomainName==$right.Name
+  | summarize DNS_TimeGenerated = arg_max(DNS_TimeGenerated, *) by IndicatorId
+  | project LatestIndicatorTime, Description, ActivityGroupNames, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, Url, DNS_TimeGenerated, Computer, ClientIP, Name, QueryType
+  | extend timestamp = DNS_TimeGenerated, HostCustomEntity = Computer, IPCustomEntity = ClientIP, URLCustomEntity = Url
 entityMappings:
   - entityType: Host
     fieldMappings:
@@ -66,5 +67,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
-version: 1.1.1
-kind: Scheduled
+version: 1.1.2
+kind: Scheduled

Detections/ThreatIntelligenceIndicator/DomainEntity_PaloAlto.yaml

@@ -21,48 +21,49 @@ tactics:
   - Impact
 query: |
 
-    let dt_lookBack = 1h;
-    let ioc_lookBack = 14d;
-    //Create a list of TLDs in our threat feed for later validation of extracted domains
-    let list_tlds = ThreatIntelligenceIndicator
-        | where TimeGenerated > ago(ioc_lookBack)
-        | where isnotempty(DomainName)
-        | extend DomainName = tolower(DomainName)
-        | extend parts = split(DomainName, '.')
-        | extend tld = parts[(array_length(parts)-1)]
-        | summarize count() by tostring(tld)
-        | summarize make_list(tld);
-        ThreatIntelligenceIndicator
-        | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
-        | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
-        | where Active == true
-        // Picking up only IOC's that contain the entities we want
-        | where isnotempty(DomainName)
-        | join (
-            CommonSecurityLog
-            | extend IngestionTime = ingestion_time()
-            | where IngestionTime > ago(dt_lookBack)
-            | where DeviceVendor =~ 'Palo Alto Networks'
-            | where DeviceEventClassID =~ 'url'
-            //Uncomment the line below to only alert on allowed connections
-            //| where DeviceAction !~ "block-url"
-            //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
-            | extend PA_Url = columnifexists("RequestURL", "None")
-            | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
-            | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
-            | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
-            | where isnotempty(Domain)
-            | extend Domain = tolower(Domain)
-            | extend parts = split(Domain, '.')
-            //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
-            | extend tld = parts[(array_length(parts)-1)]
-            //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
-            | where tld in~ (list_tlds)
-            | extend CommonSecurityLog_TimeGenerated = TimeGenerated
-        ) on $left.DomainName==$right.Domain
-        | where CommonSecurityLog_TimeGenerated >= TimeGenerated and CommonSecurityLog_TimeGenerated < ExpirationDateTime
-        | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
-        | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
+  let dt_lookBack = 1h;
+  let ioc_lookBack = 14d;
+  //Create a list of TLDs in our threat feed for later validation of extracted domains
+  let list_tlds = ThreatIntelligenceIndicator
+  | where TimeGenerated > ago(ioc_lookBack)
+  | where isnotempty(DomainName)
+  | extend DomainName = tolower(DomainName)
+  | extend parts = split(DomainName, '.')
+  | extend tld = parts[(array_length(parts)-1)]
+  | summarize count() by tostring(tld)
+  | summarize make_list(tld);
+  ThreatIntelligenceIndicator
+  | where TimeGenerated >= ago(ioc_lookBack) and ExpirationDateTime > now()
+  | summarize LatestIndicatorTime = arg_max(TimeGenerated, *) by IndicatorId
+  | where Active == true
+  // Picking up only IOC's that contain the entities we want
+  | where isnotempty(DomainName)
+  // using innerunique to keep perf fast and result set low, we only need one match to indicate potential malicious activity that needs to be investigated
+  | join kind=innerunique (
+      CommonSecurityLog
+      | extend IngestionTime = ingestion_time()
+      | where IngestionTime > ago(dt_lookBack)
+      | where DeviceVendor =~ 'Palo Alto Networks'
+      | where DeviceEventClassID =~ 'url'
+      //Uncomment the line below to only alert on allowed connections
+      //| where DeviceAction !~ "block-url"
+      //Extract domain from RequestURL, if not present extarct it from AdditionalExtentions
+      | extend PA_Url = columnifexists("RequestURL", "None")
+      | extend PA_Url = iif(isempty(PA_Url) and AdditionalExtensions !startswith "PanOS", extract("([^\"]+)", 1, tolower(AdditionalExtensions)), trim('"', PA_Url))
+      | extend PA_Url = iif(PA_Url !startswith "http://" and ApplicationProtocol !~ "ssl", strcat('http://', PA_Url), iif(PA_Url !startswith "https://" and ApplicationProtocol =~ "ssl", strcat('https://', PA_Url), PA_Url))
+      | extend Domain = trim(@"""",tostring(parse_url(PA_Url).Host))
+      | where isnotempty(Domain)
+      | extend Domain = tolower(Domain)
+      | extend parts = split(Domain, '.')
+      //Split out the TLD for the purpose of checking if we have any TI indicators with this TLD to match on
+      | extend tld = parts[(array_length(parts)-1)]
+      //Validate parsed domain by checking TLD against TLDs from threat feed and drop domains where there is no chance of a match
+      | where tld in~ (list_tlds)
+      | extend CommonSecurityLog_TimeGenerated = TimeGenerated
+  )
+  on $left.DomainName==$right.Domain
+  | project LatestIndicatorTime, Description, ActivityGroupNames, PA_Url, Domain, IndicatorId, ThreatType, ExpirationDateTime, ConfidenceScore, CommonSecurityLog_TimeGenerated, DeviceAction, DestinationIP, DestinationPort, DeviceName, SourceIP, SourcePort, ApplicationProtocol, RequestMethod
+  | extend timestamp = CommonSecurityLog_TimeGenerated, IPCustomEntity = SourceIP, HostCustomEntity = DeviceName, URLCustomEntity = PA_Url
 entityMappings:
   - entityType: Host
     fieldMappings:
@@ -76,5 +77,5 @@ entityMappings:
     fieldMappings:
       - identifier: Url
         columnName: URLCustomEntity
-version: 1.1.0
-kind: Scheduled
+version: 1.1.1
+kind: Scheduled