Skip to content

Commit c8ad00f

Browse files
jowg-amazonjowg-amazon
authored
Ioc upload integ tests and fix update (opensearch-project#1162)
* add tests and fix error msg Signed-off-by: Joanne Wang <jowg@amazon.com> * fix feeds store Signed-off-by: Joanne Wang <jowg@amazon.com> * fix msg Signed-off-by: Joanne Wang <jowg@amazon.com> * add return so flushIOCs isn't called on failure Signed-off-by: Joanne Wang <jowg@amazon.com> --------- Signed-off-by: Joanne Wang <jowg@amazon.com>
1 parent 105e590 commit c8ad00f

File tree

5 files changed

+492
-40
lines changed

5 files changed

+492
-40
lines changed

‎src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,15 +114,17 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
114114

115115
initFeedIndex(newActiveIndex, ActionListener.wrap(
116116
r -> {
117+
// reset the store configs
118+
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
119+
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().clear();
120+
}
121+
122+
// recreate the store configs
117123
saTifSourceConfig.getIocTypes().forEach(type -> {
118-
IOCType iocType = new IOCType(type);
119124
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
120-
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
121-
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
122-
listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType);
123125
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
124-
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex);
125-
listOfIocToIndexDetails.add(iocToIndexDetails);
126+
new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex);
127+
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails);
126128
}
127129
});
128130
bulkIndexIocs(iocs, newActiveIndex);

‎src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
123123
} catch (Exception e) {
124124
log.error("Failed to download IOCs.", e);
125125
listener.onFailure(e);
126+
return;
126127
}
127128

128129
try {

‎src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,15 @@
3939
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
4040
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
4141
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
42+
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
4243

4344
import java.time.Instant;
4445
import java.util.ArrayList;
4546
import java.util.HashSet;
4647
import java.util.LinkedHashMap;
4748
import java.util.LinkedHashSet;
4849
import java.util.List;
50+
import java.util.Locale;
4951
import java.util.Map;
5052
import java.util.Set;
5153
import java.util.SortedMap;
@@ -158,7 +160,7 @@ public void createIocAndTIFSourceConfig(
158160
));
159161
},
160162
e -> {
161-
log.error("Failed to download and save IOCs for source config [{}]", indexSaTifSourceConfigResponse.getId());
163+
log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
162164
saTifSourceConfigService.deleteTIFSourceConfig(indexSaTifSourceConfigResponse, ActionListener.wrap(
163165
deleteResponse -> {
164166
log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
@@ -204,15 +206,15 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
204206
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
205207
validStix2IocList.add(stix2IOC);
206208
} else {
207-
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
209+
log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}",
208210
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
209211
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
210212
);
211213
}
212214
}
213215
if (validStix2IocList.isEmpty()) {
214216
log.error("No supported IOCs to index");
215-
actionListener.onFailure(new OpenSearchStatusException("No compatible Iocs were uploaded for config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST));
217+
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)));
216218
return;
217219
}
218220
stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener);
@@ -274,14 +276,18 @@ public void updateIocAndTIFSourceConfig(
274276
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap(
275277
retrievedSaTifSourceConfig -> {
276278
if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) {
277-
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
278-
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()));
279+
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
280+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
281+
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()),
282+
RestStatus.BAD_REQUEST)));
279283
return;
280284
}
281285

282286
if (false == saTifSourceConfigDto.getType().equals(retrievedSaTifSourceConfig.getType())) {
283-
log.error("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
284-
listener.onFailure(new OpenSearchException("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()));
287+
log.error("Unable to update threat intel source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
288+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
289+
String.format(Locale.getDefault(), "Unable to update threat intel source config, type cannot change from %s to %s", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()),
290+
RestStatus.BAD_REQUEST)));
285291
return;
286292
}
287293

@@ -345,7 +351,6 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
345351

346352
saTifSourceConfigService.getClusterState(ActionListener.wrap(
347353
clusterStateResponse -> {
348-
List<String> iocTypes = updatedSaTifSourceConfig.getIocTypes();
349354
IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig();
350355
Set<String> activeIndices = new HashSet<>();
351356
Set<String> indicesToDelete = new HashSet<>();
@@ -388,11 +393,13 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
388393
), iocIndexPatterns.toArray(new String[0]));
389394
},
390395
e -> {
391-
log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId());
396+
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e);
392397
markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
393398
r -> {
394399
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
395-
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()));
400+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
401+
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()),
402+
e)));
396403
}, ex -> {
397404
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
398405
listener.onFailure(ex);
@@ -423,14 +430,18 @@ public void refreshTIFSourceConfig(
423430
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap(
424431
saTifSourceConfig -> {
425432
if (saTifSourceConfig.getType() == IOC_UPLOAD) {
426-
log.error("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
427-
listener.onFailure(new OpenSearchException("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD));
433+
log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
434+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
435+
String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD),
436+
RestStatus.BAD_REQUEST)));
428437
return;
429438
}
430439

431440
if (TIFJobState.AVAILABLE.equals(saTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(saTifSourceConfig.getState()) == false) {
432-
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
433-
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()));
441+
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
442+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
443+
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()),
444+
RestStatus.BAD_REQUEST)));
434445
return;
435446
}
436447

@@ -488,14 +499,16 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
488499
));
489500
}, downloadAndSaveIocsError -> {
490501
// Update source config as refresh failed
491-
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId());
502+
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId(), downloadAndSaveIocsError);
492503
markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
493504
r -> {
494-
log.debug("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
495-
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()));
496-
}, e -> {
505+
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
506+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
507+
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSourceConfig.getId()),
508+
downloadAndSaveIocsError)));
509+
}, ex -> {
497510
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
498-
listener.onFailure(e);
511+
listener.onFailure(ex);
499512
}
500513
));
501514
}));
@@ -528,7 +541,7 @@ public void deleteTIFSourceConfig(
528541
}, e -> {
529542
log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId);
530543
if (e instanceof IndexNotFoundException) {
531-
listener.onFailure(new OpenSearchException("Threat intel source config [{}] not found", saTifSourceConfigId));
544+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND)));
532545
} else {
533546
listener.onFailure(e);
534547
}
@@ -708,7 +721,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList
708721
}
709722
));
710723
}, e -> {
711-
log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId());
724+
log.error("Failed to delete IOC indices for threat intel source config [{}]", updateSaTifSourceConfigResponse.getId());
712725
listener.onFailure(e);
713726
}
714727
));

‎src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
import org.apache.logging.log4j.LogManager;
99
import org.apache.logging.log4j.Logger;
10-
import org.opensearch.OpenSearchException;
1110
import org.opensearch.OpenSearchStatusException;
1211
import org.opensearch.ResourceAlreadyExistsException;
1312
import org.opensearch.action.StepListener;
@@ -226,7 +225,7 @@ public void getTIFSourceConfig(
226225
client.get(getRequest, ActionListener.wrap(
227226
getResponse -> {
228227
if (!getResponse.isExists()) {
229-
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND)));
228+
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND)));
230229
return;
231230
}
232231
SATIFSourceConfig saTifSourceConfig = null;
@@ -238,7 +237,7 @@ public void getTIFSourceConfig(
238237
saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion());
239238
}
240239
if (saTifSourceConfig == null) {
241-
actionListener.onFailure(new OpenSearchException("No threat intel source config exists [{}]", tifSourceConfigId));
240+
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST)));
242241
} else {
243242
log.debug("Threat intel source config with id [{}] fetched", getResponse.getId());
244243
actionListener.onResponse(saTifSourceConfig);
@@ -258,7 +257,7 @@ public void searchTIFSourceConfigs(
258257

259258
// Check to make sure the job index exists
260259
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
261-
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
260+
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
262261
return;
263262
}
264263

@@ -350,7 +349,7 @@ public void deleteTIFSourceConfig(
350349
) {
351350
// check to make sure the job index exists
352351
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
353-
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
352+
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
354353
return;
355354
}
356355

@@ -410,7 +409,7 @@ private void deleteIocIndex(Set<String> indicesToDelete, Boolean backgroundJob,
410409
if (!response.isAcknowledged()) {
411410
log.error("Could not delete one or more IOC indices: " + index);
412411
if (backgroundJob == false) {
413-
listener.onFailure(new OpenSearchException("Could not delete one or more IOC indices: " + index));
412+
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR)));
414413
}
415414
} else {
416415
log.debug("Successfully deleted one or more IOC indices:" + index);

0 commit comments

Comments
 (0)