Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds support for uploading threat intelligence in Custom Format … #1493

Merged
merged 2 commits into from
Mar 14, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -173,6 +173,9 @@ dependencies {
compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}"
implementation "org.apache.commons:commons-csv:1.10.0"
implementation 'com.jayway.jsonpath:json-path:2.9.0'
implementation 'net.minidev:json-smart:2.5.2'
implementation 'net.minidev:accessors-smart:2.5.2'
compileOnly "com.google.guava:guava:32.1.3-jre"

// TODO uncomment once SA commons is published to maven central
Binary file modified security-analytics-commons-1.0.0.jar
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -12,14 +12,12 @@
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;

import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

/**
* A data transfer object for STIX2IOC containing additional details.
@@ -58,7 +56,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -89,7 +87,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
type = xcp.text();
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Original file line number Diff line number Diff line change
@@ -11,22 +11,17 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.securityanalytics.util.XContentUtils;

import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.UUID;

public class STIX2IOC extends STIX2 implements Writeable, ToXContentObject {
@@ -46,7 +41,7 @@ public STIX2IOC() {
public STIX2IOC(
String id,
String name,
IOCType type,
String type,
String value,
String severity,
Instant created,
@@ -86,7 +81,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
this(
sin.readString(), // id
sin.readString(), // name
new IOCType(sin.readString()), // type
sin.readString(), // type
sin.readString(), // value
sin.readString(), // severity
sin.readInstant(), // created
@@ -186,7 +181,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -204,26 +199,27 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws

switch (fieldName) {
case NAME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
Copy link
Collaborator

@AWSHurneyt AWSHurneyt Mar 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just curious. I noticed that the STIX2IOCDto class is using a getString helper function when parsing these fields. Could you clarify why we don't use something similar here?

break;
}
name = xcp.text();
break;
case TYPE_FIELD:
String typeString = xcp.text();
try {
type = new IOCType(typeString);
} catch (Exception e) {
String error = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ",
typeString,
id
);
logger.error(error, e);
throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
type = xcp.text();
break;
case VALUE_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
value = xcp.text();
break;
case SEVERITY_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
severity = xcp.text();
break;
case CREATED_FIELD:
@@ -255,6 +251,9 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}
break;
case DESCRIPTION_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
description = xcp.text();
break;
case LABELS_FIELD:
@@ -267,12 +266,21 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}
break;
case SPEC_VERSION_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
specVersion = xcp.text();
break;
case FEED_ID_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
feedId = xcp.text();
break;
case FEED_NAME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
feedName = xcp.text();
break;
default:
@@ -305,9 +313,6 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
public void validate() throws IllegalArgumentException {
if (super.getType() == null) {
throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
} else if (!IOCType.supportedType(super.getType().toString())) {
logger.debug("Unsupported IOCType: {}", super.getType().toString());
throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
}

if (super.getValue() == null || super.getValue().isEmpty()) {
Original file line number Diff line number Diff line change
@@ -10,14 +10,11 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.io.IOException;
import java.time.Instant;
@@ -32,7 +29,7 @@ public class STIX2IOCDto implements Writeable, ToXContentObject {

private String id;
private String name;
private IOCType type;
private String type;
private String value;
private String severity;
private Instant created;
@@ -50,7 +47,7 @@ public STIX2IOCDto() {}
public STIX2IOCDto(
String id,
String name,
IOCType type,
String type,
String value,
String severity,
Instant created,
@@ -149,7 +146,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -167,37 +164,24 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr

switch (fieldName) {
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
}
id = getString(xcp, id);
break;
case STIX2IOC.VERSION_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
version = xcp.longValue();
}
break;
case STIX2.NAME_FIELD:
name = xcp.text();
name = getString(xcp, name);
break;
case STIX2.TYPE_FIELD:
String typeString = xcp.text();
try {
type = new IOCType(typeString);
} catch (Exception e) {
String error = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ",
typeString,
id
);
logger.error(error, e);
throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
}
type = getString(xcp, type);
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
value = getString(xcp, value);
break;
case STIX2.SEVERITY_FIELD:
severity = xcp.text();
severity = getString(xcp, severity);
break;
case STIX2.CREATED_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
@@ -228,7 +212,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}
break;
case STIX2.DESCRIPTION_FIELD:
description = xcp.text();
description = getString(xcp, description);
break;
case STIX2.LABELS_FIELD:
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp);
@@ -240,13 +224,13 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}
break;
case STIX2.SPEC_VERSION_FIELD:
specVersion = xcp.text();
specVersion = getString(xcp, specVersion);
break;
case STIX2IOC.FEED_ID_FIELD:
feedId = xcp.text();
feedId = getString(xcp, feedId);
break;
case STIX2IOC.FEED_NAME_FIELD:
feedName = xcp.text();
feedName = getString(xcp, feedName);
break;
default:
xcp.skipChildren();
@@ -270,6 +254,14 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
);
}

private static String getString(XContentParser xcp, final String currVal) throws IOException {
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
return xcp.text();
} else {
return currVal;
}
}

public String getId() {
return id;
}
@@ -286,11 +278,11 @@ public void setName(String name) {
this.name = name;
}

public IOCType getType() {
public String getType() {
return type;
}

public void setType(IOCType type) {
public void setType(String type) {
this.type = type;
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package org.opensearch.securityanalytics.services;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.threatIntel.model.JsonPathIocSchema;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.service.JsonPathIocSchemaThreatIntelHandler;

import java.io.InputStream;
import java.util.List;
import java.util.function.Consumer;

/**
* An implementation of InputCodec used to parse input stream using JsonPath notations from {@link JsonPathIocSchema} and build a list of {@link STIX2IOC} objects
*/
public class JsonPathAwareInputCodec implements InputCodec<STIX2IOC> {
private static final Logger logger = LogManager.getLogger(JsonPathAwareInputCodec.class);
private final SATIFSourceConfig satifSourceConfig;

public JsonPathAwareInputCodec(SATIFSourceConfig satifSourceConfig) {
this.satifSourceConfig = satifSourceConfig;
}

@Override
public void parse(final InputStream inputStream, final Consumer<STIX2IOC> consumer) {
try {
List<STIX2IOC> stix2IOCS = JsonPathIocSchemaThreatIntelHandler.parseCustomSchema(
(JsonPathIocSchema) satifSourceConfig.getIocSchema(), inputStream, satifSourceConfig.getName(), satifSourceConfig.getId());
stix2IOCS.forEach(ioc -> {
try {
consumer.accept(ioc);
} catch (Exception e) {
logger.error(String.format("Error while indexing STIX2Ioc - type [%s], value [%s]"), e);
}
});
} catch (Exception e) {
logger.error(String.format("Error while downloading and indexing STIX2Ioc"), e);
}
}
}
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@
import org.opensearch.securityanalytics.commons.model.FeedConfiguration;
import org.opensearch.securityanalytics.commons.model.FeedLocation;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import software.amazon.awssdk.services.s3.S3Client;

import java.util.List;
@@ -35,23 +36,47 @@ public STIX2IOCConnectorFactory(final InputCodecFactory inputCodecFactory, final
protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration) {
final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
logger.debug("FeedLocation: {}", feedLocation);
switch(feedLocation) {
case S3: return createS3Connector(feedConfiguration);
default: throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);
switch (feedLocation) {
case S3:
return createS3Connector(feedConfiguration, null);
default:
throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);
}
}

private S3Connector<STIX2> createS3Connector(final FeedConfiguration feedConfiguration) {
protected Connector<STIX2> doCreate(FeedConfiguration feedConfiguration, SATIFSourceConfig satifSourceConfig) {
final FeedLocation feedLocation = FeedLocation.fromFeedConfiguration(feedConfiguration);
logger.debug("FeedLocation: {}", feedLocation);
switch (feedLocation) {
case S3:
return createS3Connector(feedConfiguration, satifSourceConfig);
default:
throw new IllegalArgumentException("Unsupported feedLocation: " + feedLocation);
}
}

private S3Connector<STIX2> createS3Connector(final FeedConfiguration feedConfiguration, SATIFSourceConfig satifSourceConfig) {
final InputCodec inputCodec = getInputCodec(feedConfiguration, satifSourceConfig);
final S3ConnectorConfig s3ConnectorConfig = feedConfiguration.getS3ConnectorConfig();
final S3Client s3Client = s3ClientFactory.create(s3ConnectorConfig.getRoleArn(), s3ConnectorConfig.getRegion());
final InputCodec inputCodec = inputCodecFactory.create(feedConfiguration.getIocSchema().getModelClass(), feedConfiguration.getInputCodecSchema());
return new S3Connector<>(s3ConnectorConfig, s3Client, inputCodec);
}

public S3Connector<STIX2> createAmazonS3Connector(final FeedConfiguration feedConfiguration, List<String> clusterTuple) {
private InputCodec getInputCodec(FeedConfiguration feedConfiguration, SATIFSourceConfig satifSourceConfig) {
final InputCodec inputCodec;
if (satifSourceConfig != null && satifSourceConfig.getIocSchema() != null) {
logger.info("Parsing custom schema JSON from S3 for threat intel source [{}]", satifSourceConfig.getName());
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nitpick: Would using the source config ID make more sense for this log? Or do we have validation in place to prevent customers from creating configs with the same name?

inputCodec = new JsonPathAwareInputCodec(satifSourceConfig);
} else {
inputCodec = inputCodecFactory.create(feedConfiguration.getIocSchema().getModelClass(), feedConfiguration.getInputCodecSchema());
}
return inputCodec;
}

public S3Connector<STIX2> createAmazonS3Connector(final FeedConfiguration feedConfiguration, List<String> clusterTuple, SATIFSourceConfig satifSourceConfig) {
final InputCodec inputCodec = getInputCodec(feedConfiguration, satifSourceConfig);
final S3ConnectorConfig s3ConnectorConfig = feedConfiguration.getS3ConnectorConfig();
final AmazonS3 s3Client = s3ClientFactory.createAmazonS3(s3ConnectorConfig.getRoleArn(), s3ConnectorConfig.getRegion(), clusterTuple);
final InputCodec inputCodec = inputCodecFactory.create(feedConfiguration.getIocSchema().getModelClass(), feedConfiguration.getInputCodecSchema());
return new S3Connector<>(s3ConnectorConfig, s3Client, inputCodec);
}
}
Original file line number Diff line number Diff line change
@@ -14,10 +14,13 @@
import org.opensearch.securityanalytics.commons.model.UpdateAction;
import org.opensearch.securityanalytics.commons.model.UpdateType;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -27,11 +30,15 @@ public class STIX2IOCConsumer implements Consumer<STIX2> {
private final LinkedBlockingQueue<STIX2IOC> queue;
private final STIX2IOCFeedStore feedStore;
private final UpdateType updateType;
private final SATIFSourceConfig saTifSourceConfig;
private final Set<String> iocTypes;

public STIX2IOCConsumer(final int batchSize, final STIX2IOCFeedStore feedStore, final UpdateType updateType) {
public STIX2IOCConsumer(final int batchSize, final STIX2IOCFeedStore feedStore, final UpdateType updateType, SATIFSourceConfig saTifSourceConfig) {
this.queue = new LinkedBlockingQueue<>(batchSize);
this.feedStore = feedStore;
this.updateType = updateType;
this.saTifSourceConfig = saTifSourceConfig;
this.iocTypes = new HashSet<>();
}

@Override
@@ -41,16 +48,7 @@ public void accept(final STIX2 ioc) {
feedStore.getSaTifSourceConfig().getId(),
feedStore.getSaTifSourceConfig().getName()
);

// If the IOC received is not a type listed for the config, do not add it to the queue
if (!feedStore.getSaTifSourceConfig().getIocTypes().contains(stix2IOC.getType().toString())) {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().toString(), feedStore.getSaTifSourceConfig().getId(),
stix2IOC.getId(), stix2IOC.getType(), stix2IOC.getValue()
);
return;
}

iocTypes.add(ioc.getType());
if (queue.offer(stix2IOC)) {
return;
}
@@ -68,6 +66,7 @@ public void flushIOCs() {
queue.drainTo(iocsToFlush);

final Map<IOC, UpdateAction> iocToActions = buildIOCToActions(iocsToFlush);
saTifSourceConfig.setIocTypes(new ArrayList<>(iocTypes));
feedStore.storeIOCs(iocToActions);
}

Original file line number Diff line number Diff line change
@@ -27,7 +27,6 @@
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.securityanalytics.commons.model.IOC;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.UpdateAction;
import org.opensearch.securityanalytics.commons.store.FeedStore;
import org.opensearch.securityanalytics.model.STIX2IOC;
@@ -224,7 +223,7 @@ private void initSourceConfigIndexes(StepListener<Void> stepListener) {
saTifSourceConfig.getIocTypes().forEach(type -> {
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex);
new DefaultIocStoreConfig.IocToIndexDetails(type, iocIndexPattern, newActiveIndex);
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails);
}
});
Original file line number Diff line number Diff line change
@@ -149,9 +149,9 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
return;
}

Connector<STIX2> s3Connector = constructS3Connector(s3ConnectorConfig);
Connector<STIX2> s3Connector = constructS3Connector(s3ConnectorConfig, saTifSourceConfig);
STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener);
STIX2IOCConsumer consumer = new STIX2IOCConsumer(batchSize, feedStore, UpdateType.REPLACE);
STIX2IOCConsumer consumer = new STIX2IOCConsumer(batchSize, feedStore, UpdateType.REPLACE, saTifSourceConfig);

Instant startTime = Instant.now();
Instant endTime;
@@ -226,7 +226,7 @@ public void testS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener

private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionListener<TestS3ConnectionResponse> listener) {
try {
S3Connector<STIX2> connector = (S3Connector<STIX2>) constructS3Connector(s3ConnectorConfig);
S3Connector<STIX2> connector = (S3Connector<STIX2>) constructS3Connector(s3ConnectorConfig, null);
HeadObjectResponse response = connector.testS3Connection(s3ConnectorConfig);
listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(response.sdkHttpResponse().statusCode()), ""));
} catch (NoSuchKeyException noSuchKeyException) {
@@ -251,7 +251,7 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL

private void testAmazonS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener<TestS3ConnectionResponse> listener) {
try {
S3Connector<STIX2> connector = (S3Connector<STIX2>) constructS3Connector(s3ConnectorConfig);
S3Connector<STIX2> connector = (S3Connector<STIX2>) constructS3Connector(s3ConnectorConfig, null);
boolean response = connector.testAmazonS3Connection(s3ConnectorConfig);
listener.onResponse(new TestS3ConnectionResponse(response ? RestStatus.OK : RestStatus.FORBIDDEN, ""));
} catch (AmazonServiceException e) {
@@ -268,22 +268,29 @@ private void testAmazonS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionL
}
}

private Connector<STIX2> constructS3Connector(S3ConnectorConfig s3ConnectorConfig) {
FeedConfiguration feedConfiguration = new FeedConfiguration(IOCSchema.STIX2, InputCodecSchema.ND_JSON, s3ConnectorConfig);
private Connector<STIX2> constructS3Connector(S3ConnectorConfig s3ConnectorConfig, SATIFSourceConfig saTifSourceConfig) {
FeedConfiguration feedConfiguration;
if(saTifSourceConfig != null && saTifSourceConfig.getIocSchema() != null) {
feedConfiguration = new FeedConfiguration(IOCSchema.STIX2, InputCodecSchema.ND_JSON, s3ConnectorConfig);
} else {
feedConfiguration = new FeedConfiguration(IOCSchema.STIX2, InputCodecSchema.ND_JSON, s3ConnectorConfig);
}

if (internalAuthEndpoint.isEmpty()) {
return constructS3ClientConnector(feedConfiguration);
return constructS3ClientConnector(feedConfiguration, saTifSourceConfig);
} else {
return constructAmazonS3Connector(feedConfiguration);

return constructAmazonS3Connector(feedConfiguration, saTifSourceConfig);
}
}

private Connector<STIX2> constructS3ClientConnector(FeedConfiguration feedConfiguration) {
return connectorFactory.doCreate(feedConfiguration);
private Connector<STIX2> constructS3ClientConnector(FeedConfiguration feedConfiguration, SATIFSourceConfig saTifSourceConfig) {
return connectorFactory.doCreate(feedConfiguration, saTifSourceConfig);
}

private Connector<STIX2> constructAmazonS3Connector(FeedConfiguration feedConfiguration) {
private Connector<STIX2> constructAmazonS3Connector(FeedConfiguration feedConfiguration, SATIFSourceConfig saTifSourceConfig) {
List<String> clusterTuple = List.of(clusterService.getClusterName().value().split(":"));
return connectorFactory.createAmazonS3Connector(feedConfiguration, clusterTuple);
return connectorFactory.createAmazonS3Connector(feedConfiguration, clusterTuple, saTifSourceConfig);
}

private S3ConnectorConfig constructS3ConnectorConfig(SATIFSourceConfig saTifSourceConfig) {
@@ -373,7 +380,7 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator<CSVRecord> iterator, SA
STIX2IOC stix2IOC = new STIX2IOC(
UUID.randomUUID().toString(),
UUID.randomUUID().toString(),
iocType == null ? new IOCType(IOCType.IPV4_TYPE) : new IOCType(iocType),
iocType,
iocValue,
"high",
now,
Original file line number Diff line number Diff line change
@@ -234,7 +234,7 @@ public static final List<Setting<?>> settings() {

public static final Setting<Integer> IOC_MAX_INDICES_PER_INDEX_PATTERN = Setting.intSetting(
"plugins.security_analytics.ioc.max_indices_per_alias",
30,
2,
1,
Setting.Property.NodeScope, Setting.Property.Dynamic
);
Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@
import org.opensearch.commons.alerting.model.Table;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.securityanalytics.commons.model.IOCType;

import java.io.IOException;
import java.util.List;
@@ -63,18 +62,6 @@ public ActionRequestValidationException validate() {
} else if (table.getSize() < 0 || table.getSize() > 10000) {
validationException = ValidateActions
.addValidationError(String.format("size param must be between 0 and 10,000."), validationException);
} else {
for (String type : types) {
if (!ALL_TYPES_FILTER.equalsIgnoreCase(type)) {
try {
IOCType.fromString(type);
} catch (IllegalArgumentException e) {
validationException = ValidateActions
.addValidationError(String.format("Unrecognized [%s] param.", TYPE_FIELD), validationException);
break;
}
}
}
}
return validationException;
}
Original file line number Diff line number Diff line change
@@ -5,8 +5,9 @@

package org.opensearch.securityanalytics.threatIntel.common;

import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.threatIntel.model.CustomSchemaIocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.JsonPathIocSchema;
import org.opensearch.securityanalytics.threatIntel.model.S3Source;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
@@ -15,6 +16,8 @@
import java.util.List;
import java.util.regex.Pattern;

import static org.apache.logging.log4j.util.Strings.isBlank;

/**
* Source config dto validator
*/
@@ -49,16 +52,6 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
errorMsgs.add("Source must not be empty");
}

if (sourceConfigDto.getIocTypes() == null || sourceConfigDto.getIocTypes().isEmpty()) {
errorMsgs.add("Must specify at least one IOC type");
} else {
for (String s: sourceConfigDto.getIocTypes()) {
if (!IOCType.supportedType(s)) {
errorMsgs.add("Invalid IOC type: " + s);
}
}
}

if (sourceConfigDto.getType() == null) {
errorMsgs.add("Type must not be empty");
} else {
@@ -70,10 +63,23 @@ public List<String> validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto
if (sourceConfigDto.getSchedule() != null) {
errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type");
}
if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) {
errorMsgs.add("Source must be IOC_UPLOAD type");
if (sourceConfigDto.getSource() != null &&
(sourceConfigDto.getSource() instanceof IocUploadSource == false
&& sourceConfigDto.getSource() instanceof CustomSchemaIocUploadSource == false)) {
errorMsgs.add("Source must be IOC_UPLOAD or custom_schema_ioc_upload type");
}
if(sourceConfigDto.getSource() instanceof CustomSchemaIocUploadSource) {
if(sourceConfigDto.getIocSchema() == null || sourceConfigDto.getIocSchema() instanceof JsonPathIocSchema == false) {
errorMsgs.add("Ioc Schema must be a set of valid json paths for extracting ioc type, ioc value and other fields");

}
if(isBlank(((CustomSchemaIocUploadSource) sourceConfigDto.getSource()).getIocs())) {
errorMsgs.add("Iocs must not be blank for custom_schema_ioc_upload type");
}
}
if (sourceConfigDto.getSource() instanceof IocUploadSource && ((IocUploadSource) sourceConfigDto.getSource()).getIocs() == null) {
if (sourceConfigDto.getSource() instanceof IocUploadSource
&& ((IocUploadSource) sourceConfigDto.getSource()).getIocs() == null
&& isBlank(((CustomSchemaIocUploadSource) sourceConfigDto.getSource()).getIocs())) {
errorMsgs.add("Ioc list must include at least one ioc");
}
break;
Original file line number Diff line number Diff line change
@@ -12,15 +12,4 @@ public enum SourceConfigType {
S3_CUSTOM,
IOC_UPLOAD,
URL_DOWNLOAD

// LICENSED,
//
// OPEN_SOURCED,
//
// INTERNAL,
//
// DEFAULT_OPEN_SOURCED,
//
// EXTERNAL_LICENSED,

}
Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ public void scanIoCs(IocScanContext<Data> iocScanContext,
long startTime = System.currentTimeMillis();
IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext);
if (iocLookupDtos.getIocsPerIocTypeMap().isEmpty()) {
log.error("Threat intel monitor {}: Unexpected scenario that non-zero number of docs are fetched from indices containing iocs but iocs-per-type map constructed is empty",
log.error("Threat intel monitor fanout {}: Unexpected scenario that non-zero number of docs are fetched from indices containing iocs but iocs-per-type map constructed is empty",
iocScanContext.getMonitor().getId()
);
scanCallback.accept(Collections.emptyList(), null);
@@ -50,7 +50,10 @@ public void scanIoCs(IocScanContext<Data> iocScanContext,
BiConsumer<List<STIX2IOC>, Exception> iocScanResultConsumer = (List<STIX2IOC> maliciousIocs, Exception e) -> {
long scanEndTime = System.currentTimeMillis();
long timeTaken = scanEndTime - startTime;
log.debug("Threat intel monitor {}: scan time taken is {}", monitor.getId(), timeTaken);
if(maliciousIocs != null) {
log.info("Threat intel monitor fanout : {} malicious iocs found in scan", maliciousIocs.size());
}
log.info("Threat intel monitor {}: scan time taken is {} millis", monitor.getId(), timeTaken);
if (e == null) {
createIocFindings(maliciousIocs, iocLookupDtos.iocValueToDocIdMap, iocScanContext,
(iocFindings, e1) -> {
@@ -126,7 +129,7 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
Map<String, Set<String>> docIdToIocsMap = new HashMap<>();
for (Data datum : data) {
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
String iocType = iocTypeToIndexFieldMapping.getIocType();
String concreteIndex = getIndexName(datum);
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) {
// if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
@@ -169,6 +172,7 @@ private void createIocFindings(List<STIX2IOC> iocs,
IocScanContext iocScanContext,
BiConsumer<List<IocFinding>, Exception> callback) {
try {
log.info("Threat intel monitor fanout:creating findings for [{}] iocs", iocs.size());
Instant timestamp = Instant.now();
Monitor monitor = iocScanContext.getMonitor();
// Map to collect unique IocValue with their respective FeedIds
@@ -177,7 +181,7 @@ private void createIocFindings(List<STIX2IOC> iocs,
for (STIX2IOC ioc : iocs) {
String iocValue = ioc.getValue();
if (false == iocValueToType.containsKey(iocValue))
iocValueToType.put(iocValue, ioc.getType().toString());
iocValueToType.put(iocValue, ioc.getType());
iocValueToFeedIds
.computeIfAbsent(iocValue, k -> new HashSet<>())
.add(new IocWithFeeds(ioc.getId(), ioc.getFeedId(), ioc.getFeedName(), "")); //todo figure how to store index
Original file line number Diff line number Diff line change
@@ -254,23 +254,25 @@ void matchAgainstThreatIntelAndReturnMaliciousIocs(
Monitor monitor,
BiConsumer<List<STIX2IOC>, Exception> callback,
Map<String, List<String>> iocTypeToIndices) {
iocsPerType.forEach((s, strings) -> log.info("Threat intel monitor fanout : {} iocs to scan for ioc type {}", strings.size(), s));
long startTime = System.currentTimeMillis();
int numIocs = iocsPerType.values().stream().mapToInt(Set::size).sum();
GroupedActionListener<SearchHitsOrException> groupedListenerForAllIocTypes = getGroupedListenerForIocScanFromAllIocTypes(iocsPerType, monitor, callback, startTime, numIocs);
for (String iocType : iocsPerType.keySet()) {
List<String> indices = iocTypeToIndices.get(iocType);
iocsPerType.forEach((s, strings) -> log.info("Threat intel monitor fanout : {} iocs to scan for ioc type {}", strings.size(), s));
Set<String> iocs = iocsPerType.get(iocType);
if (iocTypeToIndices.containsKey(iocType.toLowerCase())) {
if (iocTypeToIndices.containsKey(iocType)) {
if (indices.isEmpty()) {
log.debug(
"Threat intel monitor {} : No ioc indices of type {} found so no scan performed.",
log.info(
"Threat intel monitor fanout {} : No ioc indices of type {} found so no scan performed.",
monitor.getId(),
iocType
);
groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null));
} else if (iocs.isEmpty()) {
log.debug(
"Threat intel monitor {} : No iocs of type {} found in user data so no scan performed.",
log.info(
"Threat intel monitor fanout {} : No iocs of type {} found in user data so no scan performed.",
monitor.getId(),
iocType
);
@@ -279,6 +281,7 @@ void matchAgainstThreatIntelAndReturnMaliciousIocs(
performScanForMaliciousIocsPerIocType(indices, iocs, monitor, iocType, groupedListenerForAllIocTypes);
}
} else {
iocsPerType.forEach((s, strings) -> log.info("Threat intel monitor fanout : No ioc indices found for type {}. Not performing search.", iocType));
groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), null));
}
}
@@ -338,7 +341,7 @@ private void performScanForMaliciousIocsPerIocType(
GroupedActionListener<SearchHitsOrException> perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener, maxTerms);
List<String> iocList = new ArrayList<>(iocs);
int totalIocs = iocList.size();

log.info("Threat intel monitor fanout : performScanForMaliciousIocsPerIocType for {} iocs of type {}", totalIocs, iocType);
for (int start = 0; start < totalIocs; start += maxTerms) {
int end = Math.min(start + maxTerms, totalIocs);
List<String> iocsSublist = iocList.subList(start, end);
@@ -362,9 +365,12 @@ private void performScanForMaliciousIocsPerIocType(
);
}
}

log.info("Threat intel monitor fanout : performScanForMaliciousIocsPerIocType for {} iocs of type {}.SearchResponse {}", totalIocs, iocType, searchResponse);
perIocTypeListener.onResponse(new SearchHitsOrException(
searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ?
emptyList() : Arrays.asList(searchResponse.getHits().getHits()), null));

},
e -> {
log.error(() -> new ParameterizedMessage("Threat intel monitor {} scan with {} user data indicators failed for ioc Type {}",
@@ -384,8 +390,9 @@ private static SearchRequest getSearchRequestForIocType(List<String> indices, St
// add the iocs sublist
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist));
// add ioc type filter
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase(Locale.ROOT)));
boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType));
searchRequest.source().query(boolQueryBuilder);
log.info("Threat intel monitor fanout : searchRequest for ioc type {} is {}", iocType, searchRequest);
return searchRequest;
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package org.opensearch.securityanalytics.threatIntel.model;

import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;

public class CustomSchemaIocUploadSource extends Source implements Writeable, ToXContent {
public static final String IOCS_FIELD = "iocs";
public static final String FILE_NAME_FIELD = "file_name";
private String fileName;
private String iocs;

public CustomSchemaIocUploadSource(String fileName, String iocs) {
this.fileName = fileName;
this.iocs = iocs;
}

public CustomSchemaIocUploadSource(StreamInput sin) throws IOException {
this (
sin.readOptionalString(), // file name
sin.readOptionalString() // iocs
);
}

public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(fileName);
out.writeOptionalString(iocs);
}

public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject(CUSTOM_SCHEMA_IOC_UPLOAD_FIELD);
if (fileName != null) {
builder.field(FILE_NAME_FIELD, fileName);
}
if(iocs != null) {
builder.field(IOCS_FIELD, iocs);
}
builder.endObject();
builder.endObject();
return builder;
}

@Override
String name() {
return CUSTOM_SCHEMA_IOC_UPLOAD_FIELD;
}

public static CustomSchemaIocUploadSource parse(XContentParser xcp) throws IOException {
String fileName = null;
String iocs = null;

while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();
switch (fieldName) {
case FILE_NAME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
fileName = null;
} else {
fileName = xcp.text();
}
break;
case IOCS_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
iocs = null;
} else {
iocs = xcp.text();
}
break;
default:
break;
}
}
return new CustomSchemaIocUploadSource(fileName, iocs);
}

public String getIocs() {
return iocs;
}

public void setIocs(String iocs) {
this.iocs = iocs;
}

public String getFileName() {
return fileName;
}
}
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;

import java.io.IOException;
import java.util.ArrayList;
@@ -90,19 +89,19 @@ public static class IocToIndexDetails implements Writeable, ToXContent {
public static final String IOC_TYPE_FIELD = "ioc_type";
public static final String INDEX_PATTERN_FIELD = "index_pattern";
public static final String ACTIVE_INDEX_FIELD = "active_index";
private final IOCType iocType;
private final String iocType;
private final String indexPattern;
private final String activeIndex;

public IocToIndexDetails(IOCType iocType, String indexPattern, String activeIndex) {
public IocToIndexDetails(String iocType, String indexPattern, String activeIndex) {
this.iocType = iocType;
this.indexPattern = indexPattern;
this.activeIndex = activeIndex;
}

public IocToIndexDetails(StreamInput sin) throws IOException {
this(
new IOCType(sin.readString()),
sin.readString(),
sin.readString(),
sin.readString()
);
@@ -124,7 +123,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}

public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
IOCType iocType = null;
String iocType = null;
String indexPattern = null;
String activeIndex = null;

@@ -135,7 +134,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {

switch (fieldName) {
case IOC_TYPE_FIELD:
iocType = toIocType(xcp.text());
iocType = xcp.text();
break;
case INDEX_PATTERN_FIELD:
indexPattern = xcp.text();
@@ -150,16 +149,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
return new IocToIndexDetails(iocType, indexPattern, activeIndex);
}

public static IOCType toIocType(String name) {
try {
return new IOCType(name);
} catch (IllegalArgumentException e) {
log.error("Invalid Ioc type, cannot be parsed.", e);
return null;
}
}

public IOCType getIocType() {
public String getIocType() {
return iocType;
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package org.opensearch.securityanalytics.threatIntel.model;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;

import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken;

/**
* Stores the schema defined by users who upload threat intelligence in a custom format.
*/
public abstract class IocSchema<Notation> implements Writeable, ToXContentObject {
private static final Logger log = LogManager.getLogger(IocSchema.class);
abstract String getFormat(); // data format like json, xml, csv etc.

abstract Notation getId();

abstract Notation getName();

abstract Notation getType();

abstract Notation getValue();

abstract Notation getSeverity();

abstract Notation getCreated();

abstract Notation getModified();

abstract Notation getDescription();

abstract Notation getLabels();

abstract Notation getSpecVersion();

static JsonPathIocSchema readFrom(StreamInput sin) throws IOException {
String format = sin.readString();
switch (format) {
case JsonPathIocSchema.JSON_PATH_DATA_FORMAT:
return new JsonPathIocSchema(sin);
default:
throw new IllegalStateException("Unexpected ioc schema format [" + format + "] found while reading parse stream");
}
}

static IocSchema parse(XContentParser xcp) throws IOException {
IocSchema schema = null;
ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();
switch (fieldName) {
case JsonPathIocSchema.JSON_PATH_DATA_FORMAT:
schema = JsonPathIocSchema.parse(xcp);
break;
default:
String errorMessage = String.format("Unexpected ioc schema format [%s] found while parsing", fieldName);
log.error(errorMessage);
throw new IllegalStateException(errorMessage);
}
}
return schema;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
package org.opensearch.securityanalytics.threatIntel.model;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;

import java.io.IOException;

/**
* Stores the schema defined by users who upload threat intelligence in a custom format.
* Each field is defined and extracted using {@link com.jayway.jsonpath.JsonPath} annotation.
* Each field is of type {@link JsonPathSchemaField}
*/
public class JsonPathIocSchema extends IocSchema<JsonPathSchemaField> {
private static final Logger log = LogManager.getLogger(JsonPathIocSchema.class);
public static final String FIELD_ID = "id";
public static final String FIELD_NAME = "name";
public static final String FIELD_TYPE = "type";
public static final String FIELD_VALUE = "value";
public static final String FIELD_SEVERITY = "severity";
public static final String FIELD_CREATED = "created";
public static final String FIELD_MODIFIED = "modified";
public static final String FIELD_DESCRIPTION = "description";
public static final String FIELD_LABELS = "labels";
public static final String FIELD_SPEC_VERSION = "spec_version";
public static final String JSON_PATH_DATA_FORMAT = "json_path_schema";

private final JsonPathSchemaField id;
private final JsonPathSchemaField name;
private final JsonPathSchemaField type;
private final JsonPathSchemaField value;
private final JsonPathSchemaField severity;
private final JsonPathSchemaField created;
private final JsonPathSchemaField modified;
private final JsonPathSchemaField description;
private final JsonPathSchemaField labels;
private final JsonPathSchemaField specVersion;

public JsonPathIocSchema(JsonPathSchemaField id, JsonPathSchemaField name, JsonPathSchemaField type, JsonPathSchemaField value, JsonPathSchemaField severity,
JsonPathSchemaField created, JsonPathSchemaField modified, JsonPathSchemaField description, JsonPathSchemaField labels,
JsonPathSchemaField specVersion) {
this.id = id;
this.name = name;
this.type = type;
this.value = value;
this.severity = severity;
this.created = created;
this.modified = modified;
this.description = description;
this.labels = labels;
this.specVersion = specVersion;
}

public JsonPathIocSchema(StreamInput in) throws IOException {
this(
readOptionalSchemaField(in), //id
readOptionalSchemaField(in), //name
readOptionalSchemaField(in), //type
readOptionalSchemaField(in), //value
readOptionalSchemaField(in), //severity
readOptionalSchemaField(in), //created
readOptionalSchemaField(in), //modified
readOptionalSchemaField(in), //description
readOptionalSchemaField(in), //labels
readOptionalSchemaField(in) //specVersion
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
writeOptionalSchemaField(id, out);
writeOptionalSchemaField(name, out);
writeOptionalSchemaField(type, out);
writeOptionalSchemaField(value, out);
writeOptionalSchemaField(severity, out);
writeOptionalSchemaField(created, out);
writeOptionalSchemaField(modified, out);
writeOptionalSchemaField(description, out);
writeOptionalSchemaField(labels, out);
writeOptionalSchemaField(specVersion, out);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject(JSON_PATH_DATA_FORMAT);
jsonPathSchemaFieldToXcontent(builder, params, id, FIELD_ID);
jsonPathSchemaFieldToXcontent(builder, params, name, FIELD_NAME);
jsonPathSchemaFieldToXcontent(builder, params, type, FIELD_TYPE);
jsonPathSchemaFieldToXcontent(builder, params, value, FIELD_VALUE);
jsonPathSchemaFieldToXcontent(builder, params, severity, FIELD_SEVERITY);
jsonPathSchemaFieldToXcontent(builder, params, created, FIELD_CREATED);
jsonPathSchemaFieldToXcontent(builder, params, modified, FIELD_MODIFIED);
jsonPathSchemaFieldToXcontent(builder, params, description, FIELD_DESCRIPTION);
jsonPathSchemaFieldToXcontent(builder, params, labels, FIELD_LABELS);
jsonPathSchemaFieldToXcontent(builder, params, specVersion, FIELD_SPEC_VERSION);
builder.endObject();
return builder.endObject();
}

// performs null check before converting to Xcontent
private void jsonPathSchemaFieldToXcontent(XContentBuilder builder, Params params, JsonPathSchemaField jsonPathSchemaField, String fieldName) throws IOException {
if (jsonPathSchemaField != null) {
builder.field(fieldName, jsonPathSchemaField);
}
}

public static JsonPathIocSchema parse(XContentParser parser) throws IOException {
JsonPathSchemaField idPath = null;
JsonPathSchemaField namePath = null;
JsonPathSchemaField typePath = null;
JsonPathSchemaField valuePath = null;
JsonPathSchemaField severityPath = null;
JsonPathSchemaField createdPath = null;
JsonPathSchemaField modifiedPath = null;
JsonPathSchemaField descriptionPath = null;
JsonPathSchemaField labelsPath = null;
JsonPathSchemaField specVersionPath = null;

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = parser.currentName();
parser.nextToken();

switch (fieldName) {
case FIELD_ID:
idPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_NAME:
namePath = JsonPathSchemaField.parse(parser);
break;
case FIELD_TYPE:
typePath = JsonPathSchemaField.parse(parser);
break;
case FIELD_VALUE:
valuePath = JsonPathSchemaField.parse(parser);
break;
case FIELD_SEVERITY:
severityPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_CREATED:
createdPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_MODIFIED:
modifiedPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_DESCRIPTION:
descriptionPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_LABELS:
labelsPath = JsonPathSchemaField.parse(parser);
break;
case FIELD_SPEC_VERSION:
specVersionPath = JsonPathSchemaField.parse(parser);
break;
default:
parser.skipChildren();
}
}

return new JsonPathIocSchema(
idPath, namePath, typePath, valuePath,
severityPath, createdPath, modifiedPath,
descriptionPath, labelsPath, specVersionPath
);
}

public JsonPathSchemaField getId() {
return id;
}

public JsonPathSchemaField getName() {
return name;
}

public JsonPathSchemaField getType() {
return type;
}

public JsonPathSchemaField getValue() {
return value;
}

public JsonPathSchemaField getSeverity() {
return severity;
}

public JsonPathSchemaField getCreated() {
return created;
}

public JsonPathSchemaField getModified() {
return modified;
}

public JsonPathSchemaField getDescription() {
return description;
}

public JsonPathSchemaField getLabels() {
return labels;
}

public JsonPathSchemaField getSpecVersion() {
return specVersion;
}

@Override
public String getFormat() {
return JSON_PATH_DATA_FORMAT;
}

private static void writeOptionalSchemaField(JsonPathSchemaField jsonPathSchemaField, StreamOutput out) throws IOException {
if (jsonPathSchemaField == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
jsonPathSchemaField.writeTo(out);
}
}

private static JsonPathSchemaField readOptionalSchemaField(StreamInput in) throws IOException {
return in.readBoolean() ? new JsonPathSchemaField(in) : null;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package org.opensearch.securityanalytics.threatIntel.model;

import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;

import java.io.IOException;

/**
* Encapsulates data required to extract value for a field from data based on schema
*/
public class JsonPathSchemaField implements Writeable, ToXContentObject {
public static final String JSON_PATH_FIELD = "json_path";

private final String jsonPath;

public JsonPathSchemaField(String jsonPath) {
this.jsonPath = jsonPath;
}

public JsonPathSchemaField(StreamInput in) throws IOException {
this(in.readString());
}

public static JsonPathSchemaField parse(XContentParser xcp) throws IOException {
String jsonPath1 = "";
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
String fieldName = xcp.currentName();
xcp.nextToken();

switch (fieldName) {
case JSON_PATH_FIELD:
jsonPath1 = xcp.text();
break;
default:
xcp.skipChildren();
}
}
return new JsonPathSchemaField(jsonPath1);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(jsonPath);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(JSON_PATH_FIELD, jsonPath);
return builder.endObject();
}

public String getJsonPath() {
return jsonPath;
}
}
Original file line number Diff line number Diff line change
@@ -68,6 +68,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ
public static final String ENABLED_FIELD = "enabled";
public static final String IOC_STORE_FIELD = "ioc_store_config";
public static final String IOC_TYPES_FIELD = "ioc_types";
public static final String IOC_SCHEMA_FIELD = "ioc_schema";

private String id;
private Long version;
@@ -89,10 +90,12 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ
private IocStoreConfig iocStoreConfig;
private List<String> iocTypes;
private final boolean enabledForScan;
private final IocSchema iocSchema;

public SATIFSourceConfig(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source,
Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser,
boolean isEnabled, IocStoreConfig iocStoreConfig, List<String> iocTypes, boolean enabledForScan) {
boolean isEnabled, IocStoreConfig iocStoreConfig, List<String> iocTypes, boolean enabledForScan,
IocSchema iocSchema) {
this.id = id == null ? UUIDs.base64UUID() : id;
this.version = version != null ? version : NO_VERSION;
this.name = name;
@@ -121,6 +124,7 @@ public SATIFSourceConfig(String id, Long version, String name, String format, So
this.isEnabled = isEnabled;
this.iocStoreConfig = iocStoreConfig != null ? iocStoreConfig : newIocStoreConfig("default");
this.iocTypes = iocTypes;
this.iocSchema = iocSchema;
}

public SATIFSourceConfig(StreamInput sin) throws IOException {
@@ -144,7 +148,9 @@ public SATIFSourceConfig(StreamInput sin) throws IOException {
sin.readBoolean(), // is enabled
IocStoreConfig.readFrom(sin), // ioc map store
sin.readStringList(), // ioc types
sin.readBoolean() // enabled for scan
sin.readBoolean(), // enabled for scan
sin.readBoolean() ? IocSchema.readFrom(sin) : null

);
}

@@ -186,6 +192,12 @@ public void writeTo(final StreamOutput out) throws IOException {
iocStoreConfig.writeTo(out);
out.writeStringCollection(iocTypes);
out.writeBoolean(enabledForScan);
if(iocSchema != null) {
out.writeBoolean(true);
iocSchema.writeTo(out);
} else {
out.writeBoolean(false);
}
}

@Override
@@ -208,6 +220,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
} else {
builder.field(SOURCE_FIELD, source);
}
if (iocSchema == null) {
builder.nullField(IOC_SCHEMA_FIELD);
} else {
builder.field(IOC_SCHEMA_FIELD, iocSchema);
}

if (createdAt == null) {
builder.nullField(CREATED_AT_FIELD);
@@ -293,6 +310,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio
boolean enabledForScan = true;
IocStoreConfig iocStoreConfig = null;
List<String> iocTypes = new ArrayList<>();
IocSchema iocSchema = null;

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
@@ -357,6 +375,13 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio
source = Source.parse(xcp);
}
break;
case IOC_SCHEMA_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
iocSchema = null;
} else {
iocSchema = IocSchema.parse(xcp);
}
break;
case ENABLED_TIME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
enabledTime = null;
@@ -465,7 +490,8 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio
isEnabled,
iocStoreConfig,
iocTypes,
enabledForScan
enabledForScan,
iocSchema
);
}

@@ -677,4 +703,8 @@ public void setIocTypes(List<String> iocTypes) {
public boolean isEnabledForScan() {
return this.enabledForScan;
}

public IocSchema getIocSchema() {
return iocSchema;
}
}
Original file line number Diff line number Diff line change
@@ -10,7 +10,6 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchException;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.common.UUIDs;
import org.opensearch.commons.authuser.User;
@@ -25,8 +24,6 @@
import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
import org.opensearch.jobscheduler.spi.schedule.Schedule;
import org.opensearch.jobscheduler.spi.schedule.ScheduleParser;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.model.STIX2IOCDto;
import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType;
import org.opensearch.securityanalytics.threatIntel.common.RefreshType;
import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
@@ -38,7 +35,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;

/**
* Implementation of TIF Config Dto to store the source configuration metadata as DTO object
@@ -72,6 +68,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou
public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user";
public static final String ENABLED_FIELD = "enabled";
public static final String IOC_TYPES_FIELD = "ioc_types";
public static final String IOC_SCHEMA_FIELD = "ioc_schema";

private String id;
private Long version;
@@ -92,6 +89,7 @@ public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSou
private Boolean isEnabled;
private List<String> iocTypes;
private final boolean enabledForScan;
private final IocSchema iocSchema;

public SATIFSourceConfigDto(SATIFSourceConfig saTifSourceConfig) {
this.id = saTifSourceConfig.getId();
@@ -113,17 +111,12 @@ public SATIFSourceConfigDto(SATIFSourceConfig saTifSourceConfig) {
this.isEnabled = saTifSourceConfig.isEnabled();
this.iocTypes = saTifSourceConfig.getIocTypes();
this.enabledForScan = saTifSourceConfig.isEnabledForScan();
}

private List<STIX2IOCDto> convertToIocDtos(List<STIX2IOC> stix2IocList) {
return stix2IocList.stream()
.map(STIX2IOCDto::new)
.collect(Collectors.toList());
this.iocSchema = saTifSourceConfig.getIocSchema();
}

public SATIFSourceConfigDto(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source,
Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser,
boolean isEnabled, List<String> iocTypes, boolean enabledForScan) {
boolean isEnabled, List<String> iocTypes, boolean enabledForScan, IocSchema iocSchema) {
this.id = id == null ? UUIDs.base64UUID() : id;
this.version = version != null ? version : NO_VERSION;
this.name = name;
@@ -133,7 +126,7 @@ public SATIFSourceConfigDto(String id, Long version, String name, String format,
this.createdByUser = createdByUser;
this.source = source;
this.createdAt = createdAt != null ? createdAt : Instant.now();

this.iocSchema = iocSchema;
if (isEnabled && enabledTime == null) {
this.enabledTime = Instant.now();
} else if (!isEnabled) {
@@ -173,7 +166,8 @@ public SATIFSourceConfigDto(StreamInput sin) throws IOException {
sin.readBoolean() ? new User(sin) : null, // last refreshed user
sin.readBoolean(), // is enabled
sin.readStringList(), // ioc types
sin.readBoolean()
sin.readBoolean(),
sin.readBoolean() ? IocSchema.readFrom(sin) : null
);
}

@@ -211,6 +205,13 @@ public void writeTo(final StreamOutput out) throws IOException {
out.writeBoolean(isEnabled);
out.writeStringCollection(iocTypes);
out.writeBoolean(enabledForScan);
out.writeBoolean(iocSchema != null);
if (iocSchema != null) {
out.writeBoolean(true);
iocSchema.writeTo(out);
} else {
out.writeBoolean(false);
}
}

@Override
@@ -239,6 +240,12 @@ public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException
builder.field(SOURCE_FIELD, source);
}

if (iocSchema == null) {
builder.nullField(IOC_SCHEMA_FIELD);
} else {
builder.field(IOC_SCHEMA_FIELD, iocSchema);
}

if (createdAt == null) {
builder.nullField(CREATED_AT_FIELD);
} else {
@@ -317,6 +324,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver
boolean isEnabled = true;
List<String> iocTypes = new ArrayList<>();
boolean enabledForScan = true;
IocSchema iocSchema = null;

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
@@ -377,6 +385,13 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver
source = Source.parse(xcp);
}
break;
case IOC_SCHEMA_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
iocSchema = null;
} else {
iocSchema = IocSchema.parse(xcp);
}
break;
case ENABLED_TIME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
enabledTime = null;
@@ -477,7 +492,8 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver
lastRefreshedUser,
isEnabled,
iocTypes,
enabledForScan
enabledForScan,
iocSchema
);
}

@@ -642,6 +658,10 @@ public boolean isEnabled() {
return this.isEnabled;
}

public IocSchema getIocSchema() {
return iocSchema;
}

/**
* Enable auto update of threat intel feed data
*/
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ public abstract class Source {
abstract String name();
public static final String S3_FIELD = "s3";
public static final String IOC_UPLOAD_FIELD = "ioc_upload";
public static final String CUSTOM_SCHEMA_IOC_UPLOAD_FIELD = "custom_schema_ioc_upload";
public static final String URL_DOWNLOAD_FIELD = "url_download";

static Source readFrom(StreamInput sin) throws IOException {
@@ -31,6 +32,8 @@ static Source readFrom(StreamInput sin) throws IOException {
return new S3Source(sin);
case IOC_UPLOAD:
return new IocUploadSource(sin);
case CUSTOM_SCHEMA_IOC_UPLOAD:
return new CustomSchemaIocUploadSource(sin);
case URL_DOWNLOAD:
return new UrlDownloadSource(sin);
default:
@@ -52,6 +55,9 @@ public static Source parse(XContentParser xcp) throws IOException {
case IOC_UPLOAD_FIELD:
source = IocUploadSource.parse(xcp);
break;
case CUSTOM_SCHEMA_IOC_UPLOAD_FIELD:
source = CustomSchemaIocUploadSource.parse(xcp);
break;
case URL_DOWNLOAD_FIELD:
source = UrlDownloadSource.parse(xcp);
break;
@@ -73,7 +79,9 @@ enum Type {

IOC_UPLOAD(),

URL_DOWNLOAD();
URL_DOWNLOAD(),

CUSTOM_SCHEMA_IOC_UPLOAD();

@Override
public String toString() {
Original file line number Diff line number Diff line change
@@ -197,6 +197,8 @@ private void onGetIocTypeToIndices(Map<String, List<String>> iocTypeToIndicesMap
remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(),
clusterService,
indexNameExpressionResolver);
log.debug("Threat intel monitor fanout - Submitting following [{}] records's fields for scan", hits.size());

saIoCScanService.scanIoCs(new IocScanContext<>(
request.getMonitor(),
request.getMonitorMetadata(),
Original file line number Diff line number Diff line change
@@ -145,7 +145,8 @@ public void onFailure(Exception e) {
null,
true,
List.of(iocType),
true
true,
null
),
null,
RestRequest.Method.POST,

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -34,9 +34,11 @@
import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType;
import org.opensearch.securityanalytics.threatIntel.common.TIFJobState;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.model.CustomSchemaIocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig;
import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig;
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.JsonPathIocSchema;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource;
@@ -54,16 +56,17 @@
import java.util.SortedMap;
import java.util.stream.Collectors;

import static org.apache.logging.log4j.util.Strings.isBlank;
import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD;
import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.URL_DOWNLOAD;
import static org.opensearch.securityanalytics.threatIntel.service.JsonPathIocSchemaThreatIntelHandler.parseCustomSchema;

/**
* Service class for threat intel feed source config object
*/
public class SATIFSourceConfigManagementService {
private static final Logger log = LogManager.getLogger(SATIFSourceConfigManagementService.class);
private final SATIFSourceConfigService saTifSourceConfigService;
private final TIFLockService lockService; //TODO: change to js impl lock
private final STIX2IOCFetchService stix2IOCFetchService;
private final NamedXContentRegistry xContentRegistry;
private final ClusterService clusterService;
@@ -84,7 +87,6 @@ public SATIFSourceConfigManagementService(
final ClusterService clusterService
) {
this.saTifSourceConfigService = saTifSourceConfigService;
this.lockService = lockService;
this.stix2IOCFetchService = stix2IOCFetchService;
this.xContentRegistry = xContentRegistry;
this.clusterService = clusterService;
@@ -202,28 +204,68 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
stix2IOCFetchService.downloadFromUrlAndIndexIOCs(saTifSourceConfig, actionListener);
break;
case IOC_UPLOAD:
List<STIX2IOC> validStix2IocList = new ArrayList<>();
// If the IOC received is not a type listed for the config, do not add it to the queue
for (STIX2IOC stix2IOC : stix2IOCList) {
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
validStix2IocList.add(stix2IOC);
} else {
log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
if(saTifSourceConfig.getSource() instanceof IocUploadSource) {
saveLocalUploadedIocs(saTifSourceConfig, stix2IOCList, actionListener);
} else if(saTifSourceConfig.getIocSchema() != null) {
try {
validateCustomSchemaIocUploadInput(saTifSourceConfig);
CustomSchemaIocUploadSource customSchemaIocUploadSource = (CustomSchemaIocUploadSource) saTifSourceConfig.getSource();
stix2IOCList = parseCustomSchema((JsonPathIocSchema) saTifSourceConfig.getIocSchema(),
customSchemaIocUploadSource.getIocs(),
saTifSourceConfig.getName(),
saTifSourceConfig.getId()
);
saveLocalUploadedIocs(saTifSourceConfig, stix2IOCList, actionListener);
} catch (Exception e) {
log.error(String.format("Failed to parse and save %s ioc_upload", saTifSourceConfig.getName()), e);
actionListener.onFailure(e);
}
} else {
String errorMessage = String.format("Threat intel source config [{}] doesn't contain a valid source of iocs", saTifSourceConfig.getName());
log.error(errorMessage);
actionListener.onFailure(new IllegalArgumentException(errorMessage));
}
if (validStix2IocList.isEmpty()) {
log.error("No supported IOCs to index");
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)));
return;
}
stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener);
break;
}
}

private static void validateCustomSchemaIocUploadInput(SATIFSourceConfig saTifSourceConfig) {
CustomSchemaIocUploadSource source = (CustomSchemaIocUploadSource) saTifSourceConfig.getSource();
if (isBlank(source.getIocs())) {
log.error("Ioc Schema set as null when creating {} source config name {}.",
saTifSourceConfig.getType(), saTifSourceConfig.getName()
);
throw new IllegalArgumentException(String.format(saTifSourceConfig.getName(), "Iocs cannot be empty when creating/updating %s source config."));

}
if (saTifSourceConfig.getIocSchema() == null) {
log.error("Ioc Schema set as null when creating {} source config [{}].",
saTifSourceConfig.getType(), saTifSourceConfig.getName()
);
throw new IllegalArgumentException(String.format("Iocs cannot be null or empty when creating %s source config.", saTifSourceConfig.getName()));
}
JsonPathIocSchema iocSchema = (JsonPathIocSchema) saTifSourceConfig.getIocSchema();
if (iocSchema.getValue() == null || isBlank(iocSchema.getValue().getJsonPath())
|| iocSchema.getType() == null || isBlank(iocSchema.getType().getJsonPath())
) {
log.error("Custom Format Ioc Schema is missing the json path notation to extract ioc 'value' and/or" +
"ioc 'type' when parsing indicators from custom format threat intel source {}.",
saTifSourceConfig.getName()
);
throw new IllegalArgumentException(String.format("Custom Ioc Schema jsonPath notation for ioc 'value' and/or ioc 'type' cannot be blank in source [%s]", saTifSourceConfig.getName()));
}
}

private void saveLocalUploadedIocs(SATIFSourceConfig saTifSourceConfig, List<STIX2IOC> stix2IOCList, ActionListener<STIX2IOCFetchService.STIX2IOCFetchResponse> actionListener) {
if (stix2IOCList.isEmpty()) {
log.error("No supported IOCs to index");
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)));
return;
}
saTifSourceConfig.setIocTypes(new ArrayList<>(stix2IOCList.stream().map(STIX2IOC::getType).collect(Collectors.toSet())));
stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, stix2IOCList, actionListener);
}

public void getTIFSourceConfig(
final String saTifSourceConfigId,
final ActionListener<SATIFSourceConfigDto> listener
@@ -304,7 +346,8 @@ public void updateIocAndTIFSourceConfig(
isEnabled,
retrievedSaTifSourceConfig.getIocStoreConfig(),
retrievedSaTifSourceConfig.getIocTypes(),
saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan
saTifSourceConfigDto.isEnabledForScan(), // update only enabled_for_scan
saTifSourceConfigDto.getIocSchema()
);
internalUpdateTIFSourceConfig(config, ActionListener.wrap(
r -> {
@@ -401,10 +444,10 @@ public void refreshTIFSourceConfig(
) {
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap(
saTifSourceConfig -> {
if (saTifSourceConfig.getType() == IOC_UPLOAD) {
log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
if (IOC_UPLOAD.equals(saTifSourceConfig.getType()) ) {
log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), saTifSourceConfig.getType());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD),
String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), saTifSourceConfig.getType()),
RestStatus.BAD_REQUEST)));
return;
}
@@ -760,7 +803,8 @@ public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto saTifSourceCo
saTifSourceConfigDto.isEnabled(),
iocStoreConfig,
new ArrayList<>(iocTypes),
saTifSourceConfigDto.isEnabledForScan()
saTifSourceConfigDto.isEnabledForScan(),
saTifSourceConfigDto.getIocSchema()
);
}

@@ -787,7 +831,8 @@ private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSour
saTifSourceConfig.isEnabled(),
saTifSourceConfig.getIocStoreConfig(),
saTifSourceConfig.getIocTypes(),
saTifSourceConfigDto.isEnabledForScan()
saTifSourceConfigDto.isEnabledForScan(),
saTifSourceConfigDto.getIocSchema()
);
}
if (false == saTifSourceConfig.getSource().getClass().equals(saTifSourceConfigDto.getSource().getClass())) {
@@ -815,7 +860,8 @@ private SATIFSourceConfig updateSaTifSourceConfig(SATIFSourceConfigDto saTifSour
saTifSourceConfigDto.isEnabled(),
saTifSourceConfig.getIocStoreConfig(),
new ArrayList<>(iocTypes),
saTifSourceConfigDto.isEnabledForScan()
saTifSourceConfigDto.isEnabledForScan(),
saTifSourceConfigDto.getIocSchema()
);
}

Original file line number Diff line number Diff line change
@@ -46,7 +46,6 @@
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.fetch.subphase.FetchSourceContext;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction;
import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
@@ -174,7 +173,8 @@ private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig saTif
saTifSourceConfig.isEnabled(),
saTifSourceConfig.getIocStoreConfig(),
saTifSourceConfig.getIocTypes(),
saTifSourceConfig.isEnabledForScan()
saTifSourceConfig.isEnabledForScan(),
saTifSourceConfig.getIocSchema()
);
}

@@ -635,7 +635,7 @@ public void getIocTypeToIndices(ActionListener<Map<String, List<String>>> listen
DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig();
for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails : iocStoreConfig.getIocToIndexDetails()) {
String activeIndex = iocToindexDetails.getActiveIndex();
IOCType iocType = iocToindexDetails.getIocType();
String iocType = iocToindexDetails.getIocType();
List<String> strings = cumulativeIocTypeToIndices.computeIfAbsent(iocType.toString(), k -> new ArrayList<>());
strings.add(activeIndex);
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package org.opensearch.securityanalytics.threatIntel.util;

import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.commons.alerting.model.Alert;
import org.opensearch.commons.alerting.model.Monitor;
import org.opensearch.commons.alerting.model.Trigger;
@@ -10,12 +8,8 @@
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.TermQueryBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;
import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert;
@@ -175,7 +169,7 @@ public static ArrayList<IocFinding> getTriggerMatchedFindings(List<IocFinding> i
boolean iocTypeConditionMatch = false;
if (threatIntelTrigger.getIocTypes() == null || threatIntelTrigger.getIocTypes().isEmpty()) {
iocTypeConditionMatch = true;
} else if (threatIntelTrigger.getIocTypes().contains(iocFinding.getIocType().toLowerCase())) {
} else if (threatIntelTrigger.getIocTypes().contains(iocFinding.getIocType())) {
iocTypeConditionMatch = true;
}
boolean dataSourcesConditionMatch = false;
Original file line number Diff line number Diff line change
@@ -83,7 +83,10 @@
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
@@ -2345,4 +2348,15 @@ public static class LogIndices {
public String appLogsIndex;
public String s3AccessLogsIndex;
}

public String readResource(String name) throws IOException {
try (InputStream inputStream = SecurityAnalyticsPlugin.class.getClassLoader().getResourceAsStream(name)) {
if (inputStream == null) {
throw new IOException("Resource not found: " + name);
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
return reader.lines().collect(Collectors.joining("\n"));
}
}
}
}
Original file line number Diff line number Diff line change
@@ -2897,7 +2897,8 @@ public static SATIFSourceConfigDto randomSATIFSourceConfigDto(
lastRefreshedUser,
isEnabled,
iocTypes,
true
true,
null
);
}

@@ -2961,7 +2962,7 @@ public static SATIFSourceConfig randomSATIFSourceConfig(
schedule = new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS);
}
if (iocStoreConfig == null) {
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex")));
iocStoreConfig = new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.DOMAIN_NAME_TYPE, "indexPattern", "writeIndex")));
}
if (iocTypes == null) {
iocTypes = List.of("ip");
@@ -2987,7 +2988,8 @@ public static SATIFSourceConfig randomSATIFSourceConfig(
isEnabled,
iocStoreConfig,
iocTypes,
true
true,
null
);
}
}
Original file line number Diff line number Diff line change
@@ -54,7 +54,8 @@ public void testStreamInOut() throws IOException {
null,
false,
iocTypes,
true
true,
null
);

SAGetTIFSourceConfigResponse response = new SAGetTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto);
Original file line number Diff line number Diff line change
@@ -56,18 +56,18 @@ public void testValidateSourceConfigPostRequest() {
null,
false,
null,
true
true,
null
);
String id = saTifSourceConfigDto.getId();
SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, RestRequest.Method.POST, saTifSourceConfigDto);
Assert.assertNotNull(request);

ActionRequestValidationException exception = request.validate();
assertEquals(5, exception.validationErrors().size());
assertEquals(4, exception.validationErrors().size());
assertTrue(exception.validationErrors().contains("Name must not be empty"));
assertTrue(exception.validationErrors().contains("Format must not be empty"));
assertTrue(exception.validationErrors().contains("Source must not be empty"));
assertTrue(exception.validationErrors().contains("Must specify at least one IOC type"));
assertTrue(exception.validationErrors().contains("Type must not be empty"));
}
}
Original file line number Diff line number Diff line change
@@ -50,7 +50,8 @@ public void testIndexTIFSourceConfigPostResponse() throws IOException {
null,
false,
iocTypes,
true
true,
null
);

SAIndexTIFSourceConfigResponse response = new SAIndexTIFSourceConfigResponse(saTifSourceConfigDto.getId(), saTifSourceConfigDto.getVersion(), RestStatus.OK, saTifSourceConfigDto);
Original file line number Diff line number Diff line change
@@ -1549,17 +1549,6 @@ private void deleteDatastream(String datastreamName) throws IOException {

private final String DNS_MAPPINGS = "OSMapping/dns_logtype.json";

private String readResource(String name) throws IOException {
try (InputStream inputStream = SecurityAnalyticsPlugin.class.getClassLoader().getResourceAsStream(name)) {
if (inputStream == null) {
throw new IOException("Resource not found: " + name);
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
return reader.lines().collect(Collectors.joining("\n"));
}
}
}

public void testReadResource() throws IOException {
String content = readResource(DNS_SAMPLE);
assertTrue(content.contains("query_type"));
Original file line number Diff line number Diff line change
@@ -61,7 +61,8 @@ public void testParseFunctionWithNullValues() throws IOException {
null,
true,
List.of("ip"),
true
true,
null
);
String json = toJsonString(saTifSourceConfigDto);
SATIFSourceConfigDto newSaTifSourceConfigDto = SATIFSourceConfigDto.parse(getParser(json), saTifSourceConfigDto.getId(), null);
Original file line number Diff line number Diff line change
@@ -62,9 +62,10 @@ public void testParseFunctionWithNullValues() throws IOException {
null,
null,
true,
new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))),
new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(IOCType.DOMAIN_NAME_TYPE, "indexPattern", "writeIndex"))),
List.of("ip"),
true
true,
null
);
String json = toJsonString(saTifSourceConfig);
SATIFSourceConfig newSaTifSourceConfig = SATIFSourceConfig.parse(getParser(json), saTifSourceConfig.getId(), null);
Original file line number Diff line number Diff line change
@@ -7,9 +7,7 @@

import org.opensearch.common.io.stream.BytesStreamOutput;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.test.OpenSearchTestCase;

import java.io.IOException;
@@ -36,25 +34,16 @@ public void testParseFunction() throws IOException {
assertEqualIocDtos(ioc, newIoc);
}

public void testParseFunction_invalidType() throws IOException {
public void testParseFunction_customType() throws IOException {
// Execute test case for each IOCType
for (String type : IOCType.types) {
STIX2IOCDto ioc = randomIocDto(new IOCType(type));
STIX2IOCDto ioc = randomIocDto(type);
String json = toJsonString(ioc);

// Replace the IOCType with a fake type
String fakeType = "fake" + type;
final String invalidJson = json.replace(type, fakeType);

SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOCDto.parse(parser(invalidJson), ioc.getId(), ioc.getVersion()));
assertEquals(RestStatus.BAD_REQUEST, exception.status());

String expectedError = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ",
fakeType,
ioc.getId()
);
assertTrue(exception.getMessage().contains(expectedError));
STIX2IOCDto.parse(parser(invalidJson), ioc.getId(), ioc.getVersion());
}
}
}
Original file line number Diff line number Diff line change
@@ -36,25 +36,18 @@ public void testParseFunction() throws IOException {
assertEqualIOCs(ioc, newIoc);
}

public void testParseFunction_invalidType() throws IOException {
public void testParseFunction_customType() throws IOException {
// Execute test case for each IOCType
for (String type : IOCType.types) {
STIX2IOC ioc = randomIOC(new IOCType(type));
STIX2IOC ioc = randomIOC(type);
String json = toJsonString(ioc);

// Replace the IOCType with a fake type
String fakeType = "fake" + type;
final String invalidJson = json.replace(type, fakeType);

SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOC.parse(parser(invalidJson), ioc.getId(), ioc.getVersion()));
assertEquals(RestStatus.BAD_REQUEST, exception.status());

String expectedError = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ",
fakeType,
ioc.getId()
);
assertTrue(exception.getMessage().contains(expectedError));
STIX2IOC parsedIoc = STIX2IOC.parse(parser(invalidJson), ioc.getId(), ioc.getVersion());
assertEquals(parsedIoc.getType(), fakeType);
}
}
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -56,7 +56,7 @@ public void testListIOCsWithNoFindingsIndex() throws IOException {
new STIX2IOCDto(
iocId,
iocId + "-name",
new IOCType(IOCType.IPV4_TYPE),
IOCType.IPV4_TYPE,
"ipv4value" + i,
"severity",
null,
@@ -91,7 +91,8 @@ public void testListIOCsWithNoFindingsIndex() throws IOException {
null,
false,
List.of(IOCType.IPV4_TYPE),
true
true,
null
);

// Create the IOC system indexes using IOC_UPLOAD config
@@ -139,7 +140,7 @@ public void testListIOCsBySearchString() throws IOException {
new STIX2IOCDto(
"id1",
searchString,
new IOCType(IOCType.IPV4_TYPE),
IOCType.IPV4_TYPE,
"ipv4value",
"severity",
null,
@@ -155,7 +156,7 @@ public void testListIOCsBySearchString() throws IOException {
new STIX2IOCDto(
"id2",
TestHelpers.randomLowerCaseString(),
new IOCType(IOCType.IPV4_TYPE),
IOCType.IPV4_TYPE,
searchString,
"severity",
null,
@@ -171,7 +172,7 @@ public void testListIOCsBySearchString() throws IOException {
new STIX2IOCDto(
"id3",
"name",
new IOCType(IOCType.IPV4_TYPE),
IOCType.IPV4_TYPE,
"ipv4value",
"severity",
null,
@@ -204,7 +205,7 @@ public void testListIOCsBySearchString() throws IOException {
null,
false,
List.of(IOCType.IPV4_TYPE),
true
true, null
);

// Create the IOC system indexes using IOC_UPLOAD config
@@ -242,7 +243,7 @@ public void testListIOCsNumFindings() throws Exception {
new STIX2IOCDto(
iocId,
iocId + "-name",
new IOCType(IOCType.IPV4_TYPE),
IOCType.IPV4_TYPE,
"ipv4value",
"severity",
null,
@@ -284,7 +285,8 @@ public void testListIOCsNumFindings() throws Exception {
null,
false,
List.of(IOCType.IPV4_TYPE),
true
true,
null
);

// Create the IOC system indexes using IOC_UPLOAD config
Original file line number Diff line number Diff line change
@@ -169,7 +169,7 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int

// Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist.
int numOfIOCs = 1;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(IOCType.IPV4_TYPE));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -199,7 +199,8 @@ public void testCreateSATIFSourceConfigAndVerifyJobRan() throws IOException, Int
null,
true,
iocTypes,
true
true,
null
);
Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(201, response.getStatusLine().getStatusCode());
@@ -242,7 +243,7 @@ public void testGetSATIFSourceConfigById() throws IOException {

// Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist.
int numOfIOCs = 1;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.HASHES_TYPE)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(IOCType.HASHES_TYPE));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -272,7 +273,8 @@ public void testGetSATIFSourceConfigById() throws IOException {
null,
true,
iocTypes,
true
true,
null
);

Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
@@ -310,7 +312,7 @@ public void testDeleteSATIFSourceConfig() throws IOException {

// Generate test IOCs, and upload them to S3 to create the bucket object. Feed creation fails if the bucket object doesn't exist.
int numOfIOCs = 1;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(IOCType.IPV4_TYPE));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -340,7 +342,7 @@ public void testDeleteSATIFSourceConfig() throws IOException {
null,
true,
iocTypes,
true
true, null
);

Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
@@ -383,7 +385,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept
for (String type : IOCType.types) {
// Generate test IOCs, and upload them to S3
int numOfIOCs = 5;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(type)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(type));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated for type: " + type, numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -413,7 +415,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept
null,
true,
iocTypes,
true
true, null
);

// Confirm test feed was created successfully
@@ -515,8 +517,8 @@ public void testRetrieveMultipleIOCTypesSuccessfully() throws IOException, Inter
null,
true,
IOCType.types,
true
);
true, null
);

// Confirm test feed was created successfully
Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
@@ -584,7 +586,7 @@ public void testWithValidAndInvalidIOCTypes() throws IOException {

// Generate test IOCs, and upload them to S3
int numOfIOCs = 5;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(IOCType.IPV4_TYPE));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -620,8 +622,8 @@ public void testWithValidAndInvalidIOCTypes() throws IOException {
null,
true,
iocTypes,
true
);
true, null
);

Exception exception = assertThrows(ResponseException.class, () ->
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto))
@@ -638,7 +640,7 @@ public void testWithInvalidIOCTypes() throws IOException {

// Generate test IOCs, and upload them to S3
int numOfIOCs = 5;
stix2IOCGenerator = new STIX2IOCGenerator(List.of(new IOCType(IOCType.IPV4_TYPE)));
stix2IOCGenerator = new STIX2IOCGenerator(List.of(IOCType.IPV4_TYPE));
s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator);
assertEquals("Incorrect number of test IOCs generated.", numOfIOCs, stix2IOCGenerator.getIocs().size());

@@ -671,8 +673,8 @@ public void testWithInvalidIOCTypes() throws IOException {
null,
true,
iocTypes,
true
);
true, null
);

Exception exception = assertThrows(ResponseException.class, () ->
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto))
@@ -723,8 +725,8 @@ public void testWithNoIOCsToDownload() {
null,
true,
iocTypes,
true
);
true, null
);

Exception exception = assertThrows(ResponseException.class, () ->
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto))
@@ -777,7 +779,8 @@ public void testWhenBucketObjectDoesNotExist() throws IOException {
null,
true,
iocTypes,
true
true,
null
);

try {
@@ -840,7 +843,8 @@ public void testWhenRoleArnIsEmpty() throws IOException {
null,
true,
iocTypes,
true
true,
null
);

Exception exception = assertThrows(ResponseException.class, () ->
Loading