Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds support for uploading threat intelligence in Custom Format … #1493

Merged
merged 2 commits into from
Mar 14, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -173,6 +173,9 @@ dependencies {
compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}"
implementation "org.apache.commons:commons-csv:1.10.0"
implementation 'com.jayway.jsonpath:json-path:2.9.0'
implementation 'net.minidev:json-smart:2.5.2'
implementation 'net.minidev:accessors-smart:2.5.2'
compileOnly "com.google.guava:guava:32.1.3-jre"

// TODO uncomment once SA commons is published to maven central
Binary file modified security-analytics-commons-1.0.0.jar
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -12,14 +12,12 @@
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;

import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

/**
* A data transfer object for STIX2IOC containing additional details.
@@ -58,7 +56,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -89,7 +87,7 @@ public static DetailedSTIX2IOCDto parse(XContentParser xcp, String id, Long vers
name = xcp.text();
break;
case STIX2.TYPE_FIELD:
type = new IOCType(xcp.text().toLowerCase(Locale.ROOT));
type = xcp.text();
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
Original file line number Diff line number Diff line change
@@ -11,22 +11,17 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.securityanalytics.util.XContentUtils;

import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.UUID;

public class STIX2IOC extends STIX2 implements Writeable, ToXContentObject {
@@ -46,7 +41,7 @@ public STIX2IOC() {
public STIX2IOC(
String id,
String name,
IOCType type,
String type,
String value,
String severity,
Instant created,
@@ -86,7 +81,7 @@ public STIX2IOC(StreamInput sin) throws IOException {
this(
sin.readString(), // id
sin.readString(), // name
new IOCType(sin.readString()), // type
sin.readString(), // type
sin.readString(), // value
sin.readString(), // severity
sin.readInstant(), // created
@@ -186,7 +181,7 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -204,26 +199,27 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws

switch (fieldName) {
case NAME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
Copy link
Collaborator

@AWSHurneyt AWSHurneyt Mar 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just curious. I noticed that the STIX2IOCDto class is using a getString helper function when parsing these fields. Could you clarify why we don't use something similar here?

break;
}
name = xcp.text();
break;
case TYPE_FIELD:
String typeString = xcp.text();
try {
type = new IOCType(typeString);
} catch (Exception e) {
String error = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ",
typeString,
id
);
logger.error(error, e);
throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
type = xcp.text();
break;
case VALUE_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
value = xcp.text();
break;
case SEVERITY_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
severity = xcp.text();
break;
case CREATED_FIELD:
@@ -255,6 +251,9 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}
break;
case DESCRIPTION_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
description = xcp.text();
break;
case LABELS_FIELD:
@@ -267,12 +266,21 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
}
break;
case SPEC_VERSION_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
specVersion = xcp.text();
break;
case FEED_ID_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
feedId = xcp.text();
break;
case FEED_NAME_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
break;
}
feedName = xcp.text();
break;
default:
@@ -305,9 +313,6 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws
public void validate() throws IllegalArgumentException {
if (super.getType() == null) {
throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD));
} else if (!IOCType.supportedType(super.getType().toString())) {
logger.debug("Unsupported IOCType: {}", super.getType().toString());
throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD));
}

if (super.getValue() == null || super.getValue().isEmpty()) {
Original file line number Diff line number Diff line change
@@ -10,14 +10,11 @@
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.commons.model.STIX2;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.io.IOException;
import java.time.Instant;
@@ -32,7 +29,7 @@ public class STIX2IOCDto implements Writeable, ToXContentObject {

private String id;
private String name;
private IOCType type;
private String type;
private String value;
private String severity;
private Instant created;
@@ -50,7 +47,7 @@ public STIX2IOCDto() {}
public STIX2IOCDto(
String id,
String name,
IOCType type,
String type,
String value,
String severity,
Instant created,
@@ -149,7 +146,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}

String name = null;
IOCType type = null;
String type = null;
String value = null;
String severity = null;
Instant created = null;
@@ -167,37 +164,24 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr

switch (fieldName) {
case STIX2.ID_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
id = xcp.text();
}
id = getString(xcp, id);
break;
case STIX2IOC.VERSION_FIELD:
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
version = xcp.longValue();
}
break;
case STIX2.NAME_FIELD:
name = xcp.text();
name = getString(xcp, name);
break;
case STIX2.TYPE_FIELD:
String typeString = xcp.text();
try {
type = new IOCType(typeString);
} catch (Exception e) {
String error = String.format(
"Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ",
typeString,
id
);
logger.error(error, e);
throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e);
}
type = getString(xcp, type);
break;
case STIX2.VALUE_FIELD:
value = xcp.text();
value = getString(xcp, value);
break;
case STIX2.SEVERITY_FIELD:
severity = xcp.text();
severity = getString(xcp, severity);
break;
case STIX2.CREATED_FIELD:
if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) {
@@ -228,7 +212,7 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}
break;
case STIX2.DESCRIPTION_FIELD:
description = xcp.text();
description = getString(xcp, description);
break;
case STIX2.LABELS_FIELD:
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp);
@@ -240,13 +224,13 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
}
break;
case STIX2.SPEC_VERSION_FIELD:
specVersion = xcp.text();
specVersion = getString(xcp, specVersion);
break;
case STIX2IOC.FEED_ID_FIELD:
feedId = xcp.text();
feedId = getString(xcp, feedId);
break;
case STIX2IOC.FEED_NAME_FIELD:
feedName = xcp.text();
feedName = getString(xcp, feedName);
break;
default:
xcp.skipChildren();
@@ -270,6 +254,14 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr
);
}

private static String getString(XContentParser xcp, final String currVal) throws IOException {
if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) {
return xcp.text();
} else {
return currVal;
}
}

public String getId() {
return id;
}
@@ -286,11 +278,11 @@ public void setName(String name) {
this.name = name;
}

public IOCType getType() {
public String getType() {
return type;
}

public void setType(IOCType type) {
public void setType(String type) {
this.type = type;
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package org.opensearch.securityanalytics.services;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.securityanalytics.commons.connector.codec.InputCodec;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.threatIntel.model.JsonPathIocSchema;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.service.JsonPathIocSchemaThreatIntelHandler;

import java.io.InputStream;
import java.util.List;
import java.util.function.Consumer;

/**
* An implementation of InputCodec used to parse input stream using JsonPath notations from {@link JsonPathIocSchema} and build a list of {@link STIX2IOC} objects
*/
public class JsonPathAwareInputCodec implements InputCodec<STIX2IOC> {
private static final Logger logger = LogManager.getLogger(JsonPathAwareInputCodec.class);
private final SATIFSourceConfig satifSourceConfig;

public JsonPathAwareInputCodec(SATIFSourceConfig satifSourceConfig) {
this.satifSourceConfig = satifSourceConfig;
}

@Override
public void parse(final InputStream inputStream, final Consumer<STIX2IOC> consumer) {
try {
List<STIX2IOC> stix2IOCS = JsonPathIocSchemaThreatIntelHandler.parseCustomSchema(
(JsonPathIocSchema) satifSourceConfig.getIocSchema(), inputStream, satifSourceConfig.getName(), satifSourceConfig.getId());
stix2IOCS.forEach(ioc -> {
try {
consumer.accept(ioc);
} catch (Exception e) {
logger.error(String.format("Error while indexing STIX2Ioc - type [%s], value [%s]"), e);
}
});
} catch (Exception e) {
logger.error(String.format("Error while downloading and indexing STIX2Ioc"), e);
}
}
}
Loading
Loading