Skip to content

Commit 16bcef3

Browse files
eirsepeirsep
authored
adds new tif source config type - url download (opensearch-project#1142)
* adds new tif source config type - url download Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * set up create default tif configs Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * address review comments Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> * add check to block create and delete operation url download type tif source configs Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
1 parent 3be4828 commit 16bcef3

16 files changed

+534
-31
lines changed

‎src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestIndexThreatIntelMonitorAction;
158158
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction;
159159
import org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestUpdateThreatIntelAlertsStatusAction;
160+
import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService;
160161
import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService;
161162
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService;
162163
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
@@ -326,12 +327,13 @@ public Collection<Object> createComponents(Client client,
326327
IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
327328
ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
328329
SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
330+
DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
329331
return List.of(
330332
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
331333
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
332334
correlationAlertService, notificationService,
333335
tifJobUpdateService, tifJobParameterService, threatIntelLockService, saTifSourceConfigService, saTifSourceConfigManagementService, stix2IOCFetchService,
334-
ioCScanService);
336+
ioCScanService, defaultTifSourceConfigLoaderService);
335337
}
336338

337339
@Override

‎src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java‎

Lines changed: 87 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,11 @@
77

88
import com.amazonaws.AmazonServiceException;
99
import com.amazonaws.SdkClientException;
10+
import org.apache.commons.csv.CSVParser;
11+
import org.apache.commons.csv.CSVRecord;
1012
import org.apache.logging.log4j.LogManager;
1113
import org.apache.logging.log4j.Logger;
14+
import org.opensearch.action.bulk.BulkRequest;
1215
import org.opensearch.client.Client;
1316
import org.opensearch.cluster.service.ClusterService;
1417
import org.opensearch.core.action.ActionListener;
@@ -29,14 +32,17 @@
2932
import org.opensearch.securityanalytics.commons.connector.model.S3ConnectorConfig;
3033
import org.opensearch.securityanalytics.commons.model.FeedConfiguration;
3134
import org.opensearch.securityanalytics.commons.model.IOCSchema;
35+
import org.opensearch.securityanalytics.commons.model.IOCType;
3236
import org.opensearch.securityanalytics.commons.model.STIX2;
3337
import org.opensearch.securityanalytics.commons.model.UpdateType;
3438
import org.opensearch.securityanalytics.model.STIX2IOC;
3539
import org.opensearch.securityanalytics.model.STIX2IOCDto;
3640
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
3741
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
3842
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
43+
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
3944
import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService;
45+
import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser;
4046
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
4147
import software.amazon.awssdk.core.exception.SdkException;
4248
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
@@ -49,10 +55,16 @@
4955
import java.io.InputStream;
5056
import java.io.InputStreamReader;
5157
import java.nio.charset.StandardCharsets;
58+
import java.time.Instant;
5259
import java.util.ArrayList;
60+
import java.util.Collections;
61+
import java.util.Iterator;
5362
import java.util.List;
63+
import java.util.UUID;
5464
import java.util.stream.Collectors;
5565

66+
import static org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService.isValidIp;
67+
5668
/**
5769
* IOC Service implements operations that interact with retrieving IOCs from data sources,
5870
* parsing them into threat intel data models (i.e., [IOC]), and ingesting them to system indexes.
@@ -84,14 +96,14 @@ public STIX2IOCFetchService(Client client, ClusterService clusterService) {
8496

8597
/**
8698
* Method takes in and calls method to rollover and bulk index a list of STIX2IOCs
99+
*
87100
* @param saTifSourceConfig
88101
* @param stix2IOCList
89102
* @param listener
90103
*/
91104
public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
92105
List<STIX2IOC> stix2IOCList,
93-
ActionListener<STIX2IOCFetchResponse> listener)
94-
{
106+
ActionListener<STIX2IOCFetchResponse> listener) {
95107
STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
96108
try {
97109
feedStore.indexIocs(stix2IOCList);
@@ -100,6 +112,7 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig,
100112
listener.onFailure(e);
101113
}
102114
}
115+
103116
public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
104117
S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig);
105118
Connector<STIX2> s3Connector = constructS3Connector(s3ConnectorConfig);
@@ -144,7 +157,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL
144157
} catch (StsException stsException) {
145158
log.warn("S3Client connection test failed with StsException: ", stsException);
146159
listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage()));
147-
} catch (SdkException sdkException ) {
160+
} catch (SdkException sdkException) {
148161
// SdkException is a RunTimeException that doesn't have a status code.
149162
// Logging the full exception, and providing generic response as output.
150163
log.warn("S3Client connection test failed with SdkException: ", sdkException);
@@ -227,6 +240,77 @@ private String getEndpoint() {
227240
return "";
228241
}
229242

243+
public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) {
244+
UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
245+
switch (source.getFeedFormat()) { // todo add check to stop user from creating url type config from rest api. only internal allowed
246+
case "csv":
247+
try (CSVParser reader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl())) {
248+
CSVParser noHeaderReader = ThreatIntelFeedParser.getThreatIntelFeedReaderCSV(source.getUrl());
249+
boolean notFound = true;
250+
251+
while (notFound) {
252+
CSVRecord hasHeaderRecord = reader.iterator().next();
253+
254+
//if we want to skip this line and keep iterating
255+
if ((hasHeaderRecord.values().length == 1 && "".equals(hasHeaderRecord.values()[0])) || hasHeaderRecord.get(0).charAt(0) == '#' || hasHeaderRecord.get(0).charAt(0) == ' ') {
256+
noHeaderReader.iterator().next();
257+
} else { // we found the first line that contains information
258+
notFound = false;
259+
}
260+
}
261+
if (source.hasCsvHeader()) {
262+
parseAndSaveThreatIntelFeedDataCSV(reader.iterator(), saTifSourceConfig, listener);
263+
} else {
264+
parseAndSaveThreatIntelFeedDataCSV(noHeaderReader.iterator(), saTifSourceConfig, listener);
265+
}
266+
} catch (Exception e) {
267+
log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId());
268+
listener.onFailure(e);
269+
return;
270+
}
271+
break;
272+
default:
273+
log.error("unsupported feed format for url download:" + source.getFeedFormat());
274+
listener.onFailure(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat()));
275+
}
276+
}
277+
278+
private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SATIFSourceConfig saTifSourceConfig, ActionListener<STIX2IOCFetchResponse> listener) throws IOException {
279+
List<BulkRequest> bulkRequestList = new ArrayList<>();
280+
281+
UrlDownloadSource source = (UrlDownloadSource) saTifSourceConfig.getSource();
282+
List<STIX2IOC> iocs = new ArrayList<>();
283+
while (iterator.hasNext()) {
284+
CSVRecord record = iterator.next();
285+
String iocType = saTifSourceConfig.getIocTypes().stream().findFirst().orElse(null);
286+
Integer colNum = source.getCsvIocValueColumnNo();
287+
String iocValue = record.values()[colNum].split(" ")[0];
288+
if (iocType.equalsIgnoreCase(IOCType.ipv6_addr.toString()) && !isValidIp(iocValue)) {
289+
log.info("Invalid IP address, skipping this ioc record: {}", iocValue);
290+
continue;
291+
}
292+
Instant now = Instant.now();
293+
STIX2IOC stix2IOC = new STIX2IOC(
294+
UUID.randomUUID().toString(),
295+
UUID.randomUUID().toString(),
296+
iocType == null ? IOCType.ipv4_addr : IOCType.valueOf(iocType),
297+
iocValue,
298+
"high",
299+
now,
300+
now,
301+
"",
302+
Collections.emptyList(),
303+
"",
304+
saTifSourceConfig.getId(),
305+
saTifSourceConfig.getName(),
306+
STIX2IOC.NO_VERSION
307+
);
308+
iocs.add(stix2IOC);
309+
}
310+
STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
311+
feedStore.indexIocs(iocs);
312+
}
313+
230314
public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject {
231315
public static String IOCS_FIELD = "iocs";
232316
public static String TOTAL_FIELD = "total";

‎src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
1010
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
1111
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
12+
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
1213

1314
import java.util.ArrayList;
1415
import java.util.Arrays;
@@ -55,6 +56,14 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
5556
errorMsgs.add("Source must be S3_CUSTOM type");
5657
}
5758
break;
59+
case URL_DOWNLOAD:
60+
if (sourceConfigDto.getSchedule() == null) {
61+
errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type");
62+
}
63+
if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) {
64+
errorMsgs.add("Source must be URL_DOWNLOAD source type");
65+
}
66+
break;
5867
}
5968
return errorMsgs;
6069
}

‎src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigType.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77

88
/**
99
* Types of feeds threat intel can support
10-
* Feed types include: S3_CUSTOM
1110
*/
1211
public enum SourceConfigType {
1312
S3_CUSTOM,
14-
IOC_UPLOAD
13+
IOC_UPLOAD,
14+
URL_DOWNLOAD
1515

1616
// LICENSED,
1717
//

‎src/main/java/org/opensearch/securityanalytics/threatIntel/model/IocUploadSource.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,4 @@ public void setIocs(List<STIX2IOCDto> iocs) {
9696
public String getFileName() {
9797
return fileName;
9898
}
99-
100-
public void setFileName(String fileName) {
101-
this.fileName = fileName;
102-
}
10399
}

‎src/main/java/org/opensearch/securityanalytics/threatIntel/model/Source.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public abstract class Source {
2020
abstract String name();
2121
public static final String S3_FIELD = "s3";
2222
public static final String IOC_UPLOAD_FIELD = "ioc_upload";
23+
public static final String URL_DOWNLOAD_FIELD = "url_download";
2324

2425
static Source readFrom(StreamInput sin) throws IOException {
2526
Type type = sin.readEnum(Type.class);
@@ -28,6 +29,8 @@ static Source readFrom(StreamInput sin) throws IOException {
2829
return new S3Source(sin);
2930
case IOC_UPLOAD:
3031
return new IocUploadSource(sin);
32+
case URL_DOWNLOAD:
33+
return new UrlDownloadSource(sin);
3134
default:
3235
throw new IllegalStateException("Unexpected input ["+ type + "] when reading ioc store config");
3336
}
@@ -47,6 +50,9 @@ static Source parse(XContentParser xcp) throws IOException {
4750
case IOC_UPLOAD_FIELD:
4851
source = IocUploadSource.parse(xcp);
4952
break;
53+
case URL_DOWNLOAD_FIELD:
54+
source = UrlDownloadSource.parse(xcp);
55+
break;
5056
}
5157
}
5258
return source;
@@ -57,7 +63,9 @@ public void writeTo(StreamOutput out) throws IOException {}
5763
enum Type {
5864
S3(),
5965

60-
IOC_UPLOAD();
66+
IOC_UPLOAD(),
67+
68+
URL_DOWNLOAD();
6169

6270
@Override
6371
public String toString() {

‎src/main/java/org/opensearch/securityanalytics/threatIntel/model/UrlDownloadSource.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package org.opensearch.securityanalytics.threatIntel.model;
2+
3+
import org.opensearch.core.common.io.stream.StreamInput;
4+
import org.opensearch.core.common.io.stream.StreamOutput;
5+
import org.opensearch.core.common.io.stream.Writeable;
6+
import org.opensearch.core.xcontent.ToXContent;
7+
import org.opensearch.core.xcontent.XContentBuilder;
8+
import org.opensearch.core.xcontent.XContentParser;
9+
10+
import java.io.IOException;
11+
import java.net.URL;
12+
13+
/**
14+
* This is a Threat Intel Source config where the iocs are downloaded from the URL
15+
*/
16+
public class UrlDownloadSource extends Source implements Writeable, ToXContent {
17+
public static final String URL_FIELD = "url";
18+
public static final String FEED_FORMAT_FIELD = "feed_format";
19+
public static final String HAS_CSV_HEADER_FIELD = "has_csv_header_field";
20+
public static final String CSV_IOC_VALUE_COLUMN_NUM_FIELD = "csv_ioc_value_colum_num";
21+
public static final String SOURCE_NAME = "URL_DOWNLOAD";
22+
23+
private final URL url;
24+
private final String feedFormat;
25+
private final Boolean hasCsvHeader;
26+
private final Integer csvIocValueColumnNo;
27+
28+
public UrlDownloadSource(URL url, String feedFormat, Boolean hasCsvHeader, Integer csvIocValueColumnNo) {
29+
this.url = url;
30+
this.feedFormat = feedFormat;
31+
this.hasCsvHeader = hasCsvHeader;
32+
this.csvIocValueColumnNo = csvIocValueColumnNo;
33+
34+
}
35+
36+
public UrlDownloadSource(StreamInput sin) throws IOException {
37+
this(
38+
new URL(sin.readString()),
39+
sin.readString(),
40+
sin.readOptionalBoolean(),
41+
sin.readOptionalInt()
42+
);
43+
}
44+
45+
@Override
46+
public void writeTo(StreamOutput out) throws IOException {
47+
out.writeString(url.toString());
48+
out.writeString(feedFormat);
49+
out.writeOptionalBoolean(hasCsvHeader);
50+
out.writeOptionalInt(csvIocValueColumnNo);
51+
}
52+
53+
@Override
54+
String name() {
55+
return SOURCE_NAME;
56+
}
57+
58+
public URL getUrl() {
59+
return url;
60+
}
61+
62+
public static UrlDownloadSource parse(XContentParser xcp) throws IOException {
63+
URL url = null;
64+
String feedFormat = null;
65+
Boolean hasCsvHeader = false;
66+
Integer csvIocValueColumnNo = null;
67+
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
68+
String fieldName = xcp.currentName();
69+
xcp.nextToken();
70+
switch (fieldName) {
71+
case URL_FIELD:
72+
String urlString = xcp.text();
73+
url = new URL(urlString);
74+
break;
75+
case FEED_FORMAT_FIELD:
76+
feedFormat = xcp.text();
77+
break;
78+
case HAS_CSV_HEADER_FIELD:
79+
hasCsvHeader = xcp.booleanValue();
80+
break;
81+
case CSV_IOC_VALUE_COLUMN_NUM_FIELD:
82+
if (xcp.currentToken() == null)
83+
xcp.skipChildren();
84+
else
85+
csvIocValueColumnNo = xcp.intValue();
86+
break;
87+
default:
88+
xcp.skipChildren();
89+
}
90+
}
91+
return new UrlDownloadSource(url, feedFormat, hasCsvHeader, csvIocValueColumnNo);
92+
}
93+
94+
@Override
95+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
96+
return builder.startObject()
97+
.startObject(URL_DOWNLOAD_FIELD)
98+
.field(URL_FIELD, url.toString())
99+
.field(FEED_FORMAT_FIELD, feedFormat)
100+
.field(HAS_CSV_HEADER_FIELD, hasCsvHeader)
101+
.field(CSV_IOC_VALUE_COLUMN_NUM_FIELD, csvIocValueColumnNo)
102+
.endObject()
103+
.endObject();
104+
}
105+
106+
public String getFeedFormat() {
107+
return feedFormat;
108+
}
109+
110+
public boolean hasCsvHeader() {
111+
return hasCsvHeader;
112+
}
113+
114+
public Integer getCsvIocValueColumnNo() {
115+
return csvIocValueColumnNo;
116+
}
117+
}

0 commit comments

Comments
 (0)