Skip to content

Commit 3be4828

Browse files
jowg-amazoneirsep
jowg-amazon
and
eirsep
authored
Changes threat intel default store config model (opensearch-project#1133)
* change store config model Signed-off-by: Joanne Wang <jowg@amazon.com> * add validation to ioc type enum name Signed-off-by: Joanne Wang <jowg@amazon.com> * change alias to index pattern Signed-off-by: Joanne Wang <jowg@amazon.com> * make variables final Signed-off-by: Joanne Wang <jowg@amazon.com> * ensure ioc types are part of ioc_types Signed-off-by: Joanne Wang <jowg@amazon.com> * fix integ test Signed-off-by: Joanne Wang <jowg@amazon.com> * remove alias and rename active index Signed-off-by: Joanne Wang <jowg@amazon.com> * fix test Signed-off-by: Joanne Wang <jowg@amazon.com> * add enabled for scan flag in source config. Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> --------- Signed-off-by: Joanne Wang <jowg@amazon.com> Signed-off-by: Surya Sashank Nistala <snistala@amazon.com> Co-authored-by: Surya Sashank Nistala <snistala@amazon.com>
1 parent d51b5a4 commit 3be4828

21 files changed

+477
-316
lines changed

‎src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,7 @@ public List<Setting<?>> getSettings() {
497497
SecurityAnalyticsSettings.BATCH_SIZE,
498498
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
499499
SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
500-
SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS
500+
SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN
501501
);
502502
}
503503

‎src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,11 @@
2121
import org.opensearch.securityanalytics.action.ListIOCsAction;
2222
import org.opensearch.securityanalytics.action.ListIOCsActionRequest;
2323
import org.opensearch.securityanalytics.action.ListIOCsActionResponse;
24-
import org.opensearch.securityanalytics.commons.model.STIX2;
25-
import org.opensearch.securityanalytics.model.STIX2IOC;
2624

2725
import java.io.IOException;
28-
import java.time.DateTimeException;
29-
import java.time.Instant;
3026
import java.util.List;
3127
import java.util.Locale;
3228

33-
import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias;
34-
3529
public class RestListIOCsAction extends BaseRestHandler {
3630
private static final Logger log = LogManager.getLogger(RestListIOCsAction.class);
3731

‎src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java‎

Lines changed: 29 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -10,33 +10,29 @@
1010
import org.apache.logging.log4j.Logger;
1111
import org.opensearch.OpenSearchException;
1212
import org.opensearch.action.DocWriteRequest;
13-
import org.opensearch.action.admin.indices.alias.Alias;
1413
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
1514
import org.opensearch.action.admin.indices.create.CreateIndexResponse;
16-
import org.opensearch.action.admin.indices.rollover.RolloverRequest;
17-
import org.opensearch.action.admin.indices.rollover.RolloverResponse;
1815
import org.opensearch.action.bulk.BulkRequest;
1916
import org.opensearch.action.bulk.BulkResponse;
2017
import org.opensearch.action.index.IndexRequest;
2118
import org.opensearch.action.support.GroupedActionListener;
2219
import org.opensearch.action.support.WriteRequest;
2320
import org.opensearch.client.Client;
24-
import org.opensearch.cluster.ClusterState;
2521
import org.opensearch.cluster.service.ClusterService;
2622
import org.opensearch.common.settings.Settings;
2723
import org.opensearch.common.util.io.Streams;
2824
import org.opensearch.common.xcontent.XContentFactory;
2925
import org.opensearch.core.action.ActionListener;
3026
import org.opensearch.core.xcontent.ToXContent;
3127
import org.opensearch.securityanalytics.commons.model.IOC;
28+
import org.opensearch.securityanalytics.commons.model.IOCType;
3229
import org.opensearch.securityanalytics.commons.model.UpdateAction;
3330
import org.opensearch.securityanalytics.commons.store.FeedStore;
3431
import org.opensearch.securityanalytics.model.STIX2IOC;
3532
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
3633
import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
3734
import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig;
3835
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
39-
import org.opensearch.securityanalytics.util.IndexUtils;
4036

4137
import java.io.ByteArrayOutputStream;
4238
import java.io.IOException;
@@ -56,7 +52,7 @@ public class STIX2IOCFeedStore implements FeedStore {
5652
public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*";
5753
public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID";
5854
public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER;
59-
public static final String IOC_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE;
55+
public static final String IOC_ALL_INDEX_PATTERN_BY_ID = IOC_INDEX_NAME_TEMPLATE + "-*";
6056
public static final String IOC_TIME_PLACEHOLDER = "TIME";
6157
public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;
6258

@@ -117,80 +113,36 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
117113
}
118114

119115
public void indexIocs(List<STIX2IOC> iocs) throws IOException {
120-
String iocAlias = getIocIndexAlias(saTifSourceConfig.getId());
121-
String iocPattern = getIocIndexRolloverPattern(saTifSourceConfig.getId());
116+
String newActiveIndex = getNewActiveIndex(saTifSourceConfig.getId());
117+
String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId());
122118

123-
if (iocIndexExists(iocAlias) == false) {
124-
initFeedIndex(iocAlias, iocPattern, ActionListener.wrap(
125-
r -> {
126-
saTifSourceConfig.getIocTypes().forEach(type -> {
127-
String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state());
128-
String lowerCaseType = type.toLowerCase(Locale.ROOT);
129-
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().putIfAbsent(lowerCaseType, new ArrayList<>());
130-
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(iocAlias);
131-
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex);
132-
});
133-
bulkIndexIocs(iocs, iocAlias);
134-
}, e-> {
135-
log.error("Failed to initialize the IOC index and save the IOCs", e);
136-
baseListener.onFailure(e);
137-
}
138-
));
139-
} else {
140-
rolloverIndex(iocAlias, iocPattern, ActionListener.wrap(
141-
r -> {
142-
saTifSourceConfig.getIocTypes().forEach(type -> {
143-
String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state());
144-
String lowerCaseType = type.toLowerCase(Locale.ROOT);
145-
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex);
146-
});
147-
bulkIndexIocs(iocs, iocAlias);
148-
}, e -> {
149-
log.error("Failed to rollover the IOC index and save the IOCs", e);
150-
baseListener.onFailure(e);
151-
}
152-
));
153-
}
154-
}
155-
156-
private void rolloverIndex(
157-
String alias,
158-
String pattern,
159-
ActionListener<RolloverResponse> listener
160-
) {
161-
if (clusterService.state().metadata().hasAlias(alias) == false) {
162-
listener.onFailure(new OpenSearchException("Alias not initialized"));
163-
return;
164-
}
165-
166-
RolloverRequest request = new RolloverRequest(alias, pattern);
167-
request.getCreateIndexRequest()
168-
.mapping(iocIndexMapping())
169-
.settings(Settings.builder().put("index.hidden", true).build());
170-
client.admin().indices().rolloverIndex(
171-
request,
172-
ActionListener.wrap(
173-
rolloverResponse -> {
174-
if (false == rolloverResponse.isRolledOver()) {
175-
log.info(alias + "not rolled over. Rollover condition status: " + rolloverResponse.getConditionStatus());
176-
listener.onFailure(new OpenSearchException(alias + "not rolled over. Rollover condition status: " + rolloverResponse.getConditionStatus()));
177-
} else {
178-
listener.onResponse(rolloverResponse);
179-
}
180-
}, e -> {
181-
log.error("rollover failed for alias [" + alias + "].");
182-
listener.onFailure(e);
119+
initFeedIndex(newActiveIndex, ActionListener.wrap(
120+
r -> {
121+
saTifSourceConfig.getIocTypes().forEach(type -> {
122+
IOCType iocType = IOCType.fromString(type);
123+
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
124+
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
125+
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
126+
listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType);
127+
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
128+
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex);
129+
listOfIocToIndexDetails.add(iocToIndexDetails);
183130
}
184-
)
185-
);
131+
});
132+
bulkIndexIocs(iocs, newActiveIndex);
133+
}, e-> {
134+
log.error("Failed to initialize the IOC index and save the IOCs", e);
135+
baseListener.onFailure(e);
136+
}
137+
));
186138
}
187139

188-
private void bulkIndexIocs(List<STIX2IOC> iocs, String iocAlias) throws IOException {
140+
private void bulkIndexIocs(List<STIX2IOC> iocs, String activeIndex) throws IOException {
189141
List<BulkRequest> bulkRequestList = new ArrayList<>();
190142
BulkRequest bulkRequest = new BulkRequest();
191143

192144
for (STIX2IOC ioc : iocs) {
193-
IndexRequest indexRequest = new IndexRequest(iocAlias)
145+
IndexRequest indexRequest = new IndexRequest(activeIndex)
194146
.id(StringUtils.isBlank(ioc.getId())? UUID.randomUUID().toString() : ioc.getId())
195147
.opType(DocWriteRequest.OpType.INDEX)
196148
.source(ioc.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS));
@@ -235,27 +187,20 @@ private void bulkIndexIocs(List<STIX2IOC> iocs, String iocAlias) throws IOExcept
235187
}
236188
}
237189

238-
public boolean iocIndexExists(String alias) {
239-
ClusterState clusterState = clusterService.state();
240-
return clusterState.metadata().hasAlias(alias);
190+
public static String getAllIocIndexPatternById(String sourceConfigId) {
191+
return IOC_ALL_INDEX_PATTERN_BY_ID.replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT));
241192
}
242193

243-
public static String getIocIndexAlias(String feedSourceConfigId) {
244-
return IOC_WRITE_INDEX_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT));
245-
}
246-
247-
public static String getIocIndexRolloverPattern(String feedSourceConfigId) {
194+
public static String getNewActiveIndex(String sourceConfigId) {
248195
return IOC_INDEX_PATTERN
249-
.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT))
196+
.replace(IOC_FEED_ID_PLACEHOLDER, sourceConfigId.toLowerCase(Locale.ROOT))
250197
.replace(IOC_TIME_PLACEHOLDER, Long.toString(Instant.now().toEpochMilli()));
251198
}
252199

253-
254-
public void initFeedIndex(String feedAliasName, String feedIndexName, ActionListener<CreateIndexResponse> listener) {
200+
public void initFeedIndex(String feedIndexName, ActionListener<CreateIndexResponse> listener) {
255201
var indexRequest = new CreateIndexRequest(feedIndexName)
256202
.mapping(iocIndexMapping())
257203
.settings(Settings.builder().put("index.hidden", true).build());
258-
indexRequest.alias(new Alias(feedAliasName)); // set the alias
259204
client.admin().indices().create(indexRequest, ActionListener.wrap(
260205
r -> {
261206
log.info("Created system index {}", feedIndexName);

‎src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ public static final List<Setting<?>> settings() {
230230
Setting.Property.NodeScope, Setting.Property.Dynamic
231231
);
232232

233-
public static final Setting<Integer> IOC_MAX_INDICES_PER_ALIAS = Setting.intSetting(
233+
public static final Setting<Integer> IOC_MAX_INDICES_PER_INDEX_PATTERN = Setting.intSetting(
234234
"plugins.security_analytics.ioc.max_indices_per_alias",
235235
30,
236236
1,

‎src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,36 @@
55

66
package org.opensearch.securityanalytics.threatIntel.common;
77

8+
import org.opensearch.securityanalytics.commons.model.IOCType;
89
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
910
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
1011
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
1112

1213
import java.util.ArrayList;
14+
import java.util.Arrays;
1315
import java.util.List;
16+
import java.util.stream.Collectors;
1417

1518
/**
1619
* Source config dto validator
1720
*/
1821
public class SourceConfigDtoValidator {
1922
public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) {
20-
2123
List<String> errorMsgs = new ArrayList<>();
24+
List<String> iocTypeEnumNames = Arrays.stream(IOCType.values())
25+
.map(Enum::name)
26+
.collect(Collectors.toList());
27+
2228
if (sourceConfigDto.getIocTypes().isEmpty()) {
2329
errorMsgs.add("Must specify at least one IOC type");
30+
} else {
31+
for (String s: sourceConfigDto.getIocTypes()) {
32+
if (false == iocTypeEnumNames.contains(s)) {
33+
errorMsgs.add("Invalid IOC type: " + s);
34+
}
35+
}
2436
}
37+
2538
switch (sourceConfigDto.getType()) {
2639
case IOC_UPLOAD:
2740
if (sourceConfigDto.isEnabled()) {

0 commit comments

Comments
 (0)