Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#282: Fix sourcetype IN not working #383

Merged
merged 5 commits into from
Oct 28, 2024
Merged

#282: Fix sourcetype IN not working #383

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
73ca8c3
fix sourcetype IN not working, cleanup searchQualifier in LogicalStat…
eemhu Oct 21, 2024
dc442aa
introduce OrColumn object with tests
eemhu Oct 23, 2024
719b8c4
run spotless
eemhu Oct 23, 2024
7cb5736
add archive(xml) and spark query tests for sourcetype IN
eemhu Oct 28, 2024
99b4a4f
applied spotless
eemhu Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 15 additions & 10 deletions src/main/java/com/teragrep/pth10/ast/commands/logicalstatement/LogicalStatementCatalyst.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,23 +424,17 @@ public Node visitSubindexStatement(DPLParser.SubindexStatementContext ctx) {
*/
@Override
public Node visitSearchQualifier(DPLParser.SearchQualifierContext ctx) {
// LOGGER.info("visitSearchQualifier: ");
Column sQualifier = null;
String value = null;
String value;

TerminalNode left = (TerminalNode) ctx.getChild(0);
TerminalNode operation = (TerminalNode) ctx.getChild(1);

List<String> listOfIndices = new ArrayList<>();

// Default clause used in WHERE-part
String columnName = null;
final String columnName;
// HOST and SOURCETYPE qualifier stored as additional list and used for Kafka content filtering
if (left.getSymbol().getType() == DPLLexer.INDEX_IN) {
value = "";
ctx.indexStringType().forEach(str -> {
listOfIndices.add(new UnquotedText(new TextString(str.getText().toLowerCase())).read());
});
columnName = "index";
}
else if (left.getSymbol().getType() == DPLLexer.HOST) {
Expand All @@ -467,8 +461,8 @@ else if (left.getSymbol().getType() == DPLLexer.SOURCETYPE) {

}
else if (left.getSymbol().getType() == DPLLexer.INDEX_IN) {
for (String index : listOfIndices) {
String rlikeStatement = glob2rlike(index);
for (DPLParser.IndexStringTypeContext indexCtx : ctx.indexStringType()) {
final String rlikeStatement = glob2rlike(new UnquotedText(new TextString(indexCtx.getText().toLowerCase())).read());
if (sQualifier == null) {
sQualifier = col.rlike(rlikeStatement);
}
Expand All @@ -478,6 +472,17 @@ else if (left.getSymbol().getType() == DPLLexer.INDEX_IN) {

}
}
else if (left.getSymbol().getType() == DPLLexer.SOURCETYPE && operation.getSymbol().getType() == DPLLexer.IN) {
for (DPLParser.StringTypeContext stringType : ctx.stringType()) {
final String sourceType = new UnquotedText(new TextString(stringType.getText())).read();
final String rlikeStatement = glob2rlike(sourceType);
if (sQualifier == null) {
sQualifier = col.rlike(rlikeStatement);
} else {
sQualifier = sQualifier.or(col.rlike(rlikeStatement));
}
}
}
else {
String rlikeStatement = glob2rlike(value);
sQualifier = functions.not(col.rlike(rlikeStatement));
Expand Down
56 changes: 28 additions & 28 deletions src/main/java/com/teragrep/pth10/ast/commands/logicalstatement/LogicalStatementXML.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,13 +381,10 @@ public Node visitSublogicalStatement(DPLParser.SublogicalStatementContext ctx) {
*/
@Override
public Node visitSearchQualifier(DPLParser.SearchQualifierContext ctx) {
// LOGGER.info("visitSearchQualifier: ");
Token comparisonToken;
Element el = null;
String value = null;
List<String> listOfIndices = new ArrayList<>();
String field;
final List<String> values = new ArrayList<>();

String qualifier;
TerminalNode left = (TerminalNode) ctx.getChild(0);
TerminalNode operation = (TerminalNode) ctx.getChild(1);

Expand All @@ -400,52 +397,56 @@ public Node visitSearchQualifier(DPLParser.SearchQualifierContext ctx) {
}

// Default clause used in WHERE-part
qualifier = left.getText() + " " + operation.getSymbol().getText() + " ";
final String qualifier = left.getText() + " " + operation.getSymbol().getText() + " ";

// HOST and SOURCETYPE qualifier stored as additional list and used for Kafka content filtering
if (left.getSymbol().getType() == DPLLexer.INDEX_IN) {
value = "";
ctx.indexStringType().forEach(str -> {
listOfIndices.add(new UnquotedText(new TextString(str.getText().toLowerCase())).read());
values.add(new UnquotedText(new TextString(str.getText().toLowerCase())).read());
});
comparisonToken = new Token(Type.EQUALS); // INDEX IN is equals
el = doc.createElement("index");
field = "index";
}
else if (left.getSymbol().getType() == DPLLexer.HOST) {
value = new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read();
el = doc.createElement("host");
values.add(new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read());
field = "host";
}
else if (left.getSymbol().getType() == DPLLexer.SOURCETYPE && operation.getSymbol().getType() == DPLLexer.IN) {
ctx.stringType().forEach(str -> {
values.add(new UnquotedText(new TextString(str.getText().toLowerCase())).read());
});
comparisonToken = new Token(Type.EQUALS); // SOURCETYPE IN is equals
field = "sourcetype";
}
else if (left.getSymbol().getType() == DPLLexer.SOURCETYPE) {
value = new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read();
el = doc.createElement("sourcetype");
LOGGER.debug("qualifier=<{}> sourcetype=<{}>", qualifier, value);
values.add(new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read());
field = "sourcetype";
LOGGER.debug("qualifier=<{}> sourcetype=<{}>", qualifier, values.get(0));
}
else {
// other column=value qualifier
value = new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read();
el = doc.createElement(ctx.getChild(0).getText().toLowerCase());
values.add(new UnquotedText(new TextString(ctx.getChild(2).getText().toLowerCase())).read());
field = ctx.getChild(0).getText().toLowerCase();
LOGGER
.debug("custom qualifier: field=<{}> = value=<{}>", ctx.getChild(0).getText(), ctx.getChild(2).getText());
}

if (listOfIndices.isEmpty()) {
Element el;
if (values.size() < 2) {
el = doc.createElement(field);
el.setAttribute("operation", comparisonToken.toString());
el.setAttribute("value", value);
el.setAttribute("value", values.get(0));
}
else {
// build xml string for index IN ( 1 2 )
el = doc.createElement("OR");

for (int i = 0; i < listOfIndices.size(); i++) {
Element indexElem = doc.createElement("index");
for (int i = 0; i < values.size(); i++) {
Element indexElem = doc.createElement(field);
indexElem.setAttribute("operation", comparisonToken.toString());
indexElem.setAttribute("value", listOfIndices.get(i));
indexElem.setAttribute("value", values.get(i));

if (listOfIndices.size() == 1) {
el = indexElem;
break;
}
else if (i < 2) {
if (i < 2) {
el.appendChild(indexElem);
}
else {
Expand All @@ -458,8 +459,7 @@ else if (i < 2) {
}
}

Node rv = new ElementNode(el);
return rv;
return new ElementNode(el);
}

/**
Expand Down
15 changes: 6 additions & 9 deletions src/test/java/com/teragrep/pth10/logicalOperationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,25 +422,22 @@ public void testWithMultipleHostsAndSourcetype() {
}

@Test
@Disabled(
value = "search does not allow having sourcetype in this format: sourcetype IN (sourcetyp1, sourcetype2), pth-10 issue #282"
)
public void testWithMultipleSourcetypes() {
String query = "\"index=index_* sourcetype IN (B:X:0, B:Y:0, C:X:0)\"";
String query = "index=index_* sourcetype IN (B:X:0, B:Y:0, C:X:0)";
this.streamingTestUtil.performDPLTest(query, this.testFile, res -> {
List<String> listOfSourcetype = res
.select("_raw")
.select("sourcetype")
.orderBy("offset")
.collectAsList()
.stream()
.map(r -> r.getAs(0).toString())
.collect(Collectors.toList());

Assertions.assertEquals(5, res.count());
Assertions.assertEquals("B:X:0", listOfSourcetype.get(0));
Assertions.assertEquals("B:X:0", listOfSourcetype.get(1));
Assertions.assertEquals("B:Y:0", listOfSourcetype.get(2));
Assertions.assertEquals("B:Y:0", listOfSourcetype.get(3));
Assertions.assertEquals("B:Y:0", listOfSourcetype.get(0));
Assertions.assertEquals("B:Y:0", listOfSourcetype.get(1));
Assertions.assertEquals("B:X:0", listOfSourcetype.get(2));
Assertions.assertEquals("B:X:0", listOfSourcetype.get(3));
Assertions.assertEquals("C:X:0", listOfSourcetype.get(4));
});

Expand Down