make threat intel run with standard detectors (opensearch-project#1234)

Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>

Author: sbcd90

src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java

@@ -108,8 +108,8 @@ protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionL
         SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest();
         threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config");
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-        boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
-        boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
+        boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
+        boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
         threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder));
         transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap(
                 searchResponse -> {

src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java

@@ -118,8 +118,8 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti
             SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest();
             threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config");
             BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-            boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
-            boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
+            boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
+            boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
             threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder));
             transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap(
                     searchResponse -> {

src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java

@@ -2,19 +2,24 @@
 
 import org.apache.hc.core5.http.ContentType;
 import org.apache.hc.core5.http.io.entity.StringEntity;
+import org.apache.http.HttpStatus;
 import org.junit.Assert;
+import org.opensearch.client.Request;
 import org.opensearch.client.Response;
 import org.opensearch.client.ResponseException;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.commons.alerting.model.IntervalSchedule;
 import org.opensearch.commons.alerting.model.Monitor;
+import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.search.SearchHit;
 import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
 import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
 import org.opensearch.securityanalytics.action.ListIOCsActionRequest;
 import org.opensearch.securityanalytics.commons.model.IOCType;
+import org.opensearch.securityanalytics.model.Detector;
+import org.opensearch.securityanalytics.model.DetectorTrigger;
 import org.opensearch.securityanalytics.model.STIX2IOC;
 import org.opensearch.securityanalytics.threatIntel.common.RefreshType;
 import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType;
@@ -37,6 +42,8 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyList;
+import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
+import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers;
 import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
 import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
 import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH;
@@ -422,6 +429,154 @@ public void testCreateThreatIntelMonitor() throws IOException {
         assertEquals(totalHitsVal.intValue(), 0);
     }
 
+    public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException {
+        String index = createTestIndex(randomIndex(), windowsIndexMapping());
+
+        // Execute CreateMappingsAction to add alias mapping for index
+        Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
+        // both req params and req body are supported
+        createMappingRequest.setJsonEntity(
+                "{ \"index_name\":\"" + index + "\"," +
+                        "  \"rule_topic\":\"" + randomDetectorType() + "\", " +
+                        "  \"partial\":true" +
+                        "}"
+        );
+
+        Response response = client().performRequest(createMappingRequest);
+        assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+
+        Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())));
+
+        Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
+        Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
+
+        Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
+                Map.of(), null);
+        Map<String, Object> responseAsMap = responseAsMap(iocFindingsResponse);
+        Assert.assertEquals(0, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
+        List<String> vals = List.of("ip1", "ip2");
+        indexSourceConfigsAndIocs(1, vals);
+        String monitorName = "test_monitor_name";
+
+
+        /**create monitor */
+        ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index);
+        response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
+        Assert.assertEquals(201, response.getStatusLine().getStatusCode());
+        Map<String, Object> responseBody = asMap(response);
+
+        try {
+            makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
+            fail();
+        } catch (Exception e) {
+            /** creating a second threat intel monitor should fail*/
+            assertTrue(e.getMessage().contains("already exists"));
+        }
+
+        final String monitorId = responseBody.get("id").toString();
+        Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId);
+
+        Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId);
+        Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
+        int i = 1;
+        for (String val : vals) {
+            String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
+            try {
+                indexDoc(index, "" + i++, doc);
+            } catch (IOException e) {
+                fail();
+            }
+        }
+
+        Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
+        Map<String, Object> executeResults = entityAsMap(executeResponse);
+        assertEquals(1, 1);
+
+        String matchAllRequest = getMatchAllRequest();
+        Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
+        Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
+        HashMap<String, Object> hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
+        HashMap<String, Object> totalHits = (HashMap<String, Object>) hits.get("total");
+        Integer totalHitsVal = (Integer) totalHits.get("value");
+        assertEquals(totalHitsVal.intValue(), 1);
+        makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
+
+
+        iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
+                Map.of(), null);
+        responseAsMap = responseAsMap(iocFindingsResponse);
+        Assert.assertEquals(2, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
+
+        //alerts
+        List<SearchHit> searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
+        Assert.assertEquals(4, searchHits.size());
+
+        for (String val : vals) {
+            String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
+            try {
+                indexDoc(index, "" + i++, doc);
+            } catch (IOException e) {
+                fail();
+            }
+        }
+        executeAlertingMonitor(monitorId, Collections.emptyMap());
+        iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
+                Map.of(), null);
+        responseAsMap = responseAsMap(iocFindingsResponse);
+        Assert.assertEquals(4, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
+
+        // Use ListIOCs API to confirm expected number of findings are returned
+        String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0");
+        Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null);
+        Map<String, Object> listIocsResponseMap = responseAsMap(listIocsResponse);
+        List<Map<String, Object>> iocsMap = (List<Map<String, Object>>) listIocsResponseMap.get("iocs");
+        assertEquals(2, iocsMap.size());
+        iocsMap.forEach((iocDetails) -> {
+            String iocId = (String) iocDetails.get("id");
+            int numFindings = (Integer) iocDetails.get("num_findings");
+            assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId())));
+            assertEquals(2, numFindings);
+        });
+
+        //alerts via system index search
+        searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
+        Assert.assertEquals(4, searchHits.size());
+
+        // alerts via API
+        Map<String, String> params = new HashMap<>();
+        Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null);
+        Map<String, Object> getAlertsBody = asMap(getAlertsResponse);
+        Assert.assertEquals(4, getAlertsBody.get("total_alerts"));
+
+
+        ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto(
+                monitorId,
+                iocScanMonitor.getName() + "update",
+                iocScanMonitor.getPerIocTypeScanInputList(),
+                new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()),
+                false,
+                null,
+                List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1))
+        );
+        //update monitor
+        response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto));
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        responseBody = asMap(response);
+        assertEquals(responseBody.get("id").toString(), monitorId);
+        assertEquals(((HashMap<String, Object>) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update");
+
+        //delete
+        Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null);
+        Assert.assertEquals(200, delete.getStatusLine().getStatusCode());
+
+        searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
+        Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
+        hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
+        totalHits = (HashMap<String, Object>) hits.get("total");
+        totalHitsVal = (Integer) totalHits.get("value");
+        assertEquals(totalHitsVal.intValue(), 0);
+    }
+
     public void testCreateThreatIntelMonitor_invalidMonitorJson() throws IOException {
         ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto("test-index");