Skip to content

Commit 85720ed

Browse files
hauck-jvshpranikum
authored andcommitted
Ignore all malformed objects when ignore_malformed is true (opensearch-project#4494)
Fixes a bug to not fail the entire document when "ignore_malformed" is set to true. Allowing the valid fields to be indexed and ignore only the malformed fields. Signed-off-by: Hauck <joaoh14@gmail.com>
1 parent 70474fd commit 85720ed

File tree

2 files changed

+19
-11
lines changed

2 files changed

+19
-11
lines changed

CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
6464
- `opensearch.bat` fails to execute when install path includes spaces ([#4362](https://github.com/opensearch-project/OpenSearch/pull/4362))
6565
- Getting security exception due to access denied 'java.lang.RuntimePermission' 'accessDeclaredMembers' when trying to get snapshot with S3 IRSA ([#4469](https://github.com/opensearch-project/OpenSearch/pull/4469))
6666
- Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
67+
- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
6768

6869
### Security
6970
- CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

server/src/main/java/org/opensearch/index/mapper/FieldMapper.java

+18-11
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import com.carrotsearch.hppc.cursors.ObjectCursor;
3636
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
37+
3738
import org.apache.lucene.document.Field;
3839
import org.apache.lucene.document.FieldType;
3940
import org.apache.lucene.index.IndexOptions;
@@ -268,6 +269,8 @@ public void parse(ParseContext context) throws IOException {
268269
try {
269270
parseCreateField(context);
270271
} catch (Exception e) {
272+
boolean ignore_malformed = false;
273+
if (context.indexSettings() != null) ignore_malformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
271274
String valuePreview = "";
272275
try {
273276
XContentParser parser = context.parser();
@@ -278,23 +281,27 @@ public void parse(ParseContext context) throws IOException {
278281
valuePreview = complexValue.toString();
279282
}
280283
} catch (Exception innerException) {
284+
if (ignore_malformed == false) {
285+
throw new MapperParsingException(
286+
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
287+
e,
288+
fieldType().name(),
289+
fieldType().typeName(),
290+
context.sourceToParse().id()
291+
);
292+
}
293+
}
294+
295+
if (ignore_malformed == false) {
281296
throw new MapperParsingException(
282-
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Could not parse field value preview,",
297+
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
283298
e,
284299
fieldType().name(),
285300
fieldType().typeName(),
286-
context.sourceToParse().id()
301+
context.sourceToParse().id(),
302+
valuePreview
287303
);
288304
}
289-
290-
throw new MapperParsingException(
291-
"failed to parse field [{}] of type [{}] in document with id '{}'. " + "Preview of field's value: '{}'",
292-
e,
293-
fieldType().name(),
294-
fieldType().typeName(),
295-
context.sourceToParse().id(),
296-
valuePreview
297-
);
298305
}
299306
multiFields.parse(this, context);
300307
}

0 commit comments

Comments
 (0)