From 96b4f6ee162fba0546b3043bae73170579be9553 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 3 Dec 2024 16:51:41 -0800 Subject: [PATCH 1/3] Use default module instead of $ion_encoding --- ion-tests | 2 +- .../java/com/amazon/ion/SystemSymbols.java | 14 +- .../ion/impl/EncodingDirectiveReader.kt | 72 +++++++--- .../com/amazon/ion/impl/IonCursorBinary.java | 2 +- .../impl/IonReaderContinuableCoreBinary.java | 123 ++++++++++++------ .../amazon/ion/impl/IonReaderTextSystemX.java | 14 +- .../com/amazon/ion/impl/SystemSymbols_1_1.kt | 2 +- .../ion/impl/bin/IonManagedWriter_1_1.kt | 12 +- .../com/amazon/ion/impl/macro/SystemMacro.kt | 66 ++++++---- .../com/amazon/ion/Ion_1_1_RoundTripTest.kt | 2 +- .../EncodingDirectiveCompilationTest.java | 76 ++++++----- ...eaderContinuableApplicationBinaryTest.java | 16 ++- .../IonReaderContinuableCoreBinaryTest.java | 14 +- ...onReaderContinuableTopLevelBinaryTest.java | 6 +- .../ion/impl/bin/IonManagedWriter_1_1_Test.kt | 6 +- .../impl/bin/IonRawBinaryWriterTest_1_1.kt | 6 +- .../ion/impl/macro/MacroEvaluatorTest.kt | 34 ++--- 17 files changed, 286 insertions(+), 181 deletions(-) diff --git a/ion-tests b/ion-tests index c83270f08..edbd1089a 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit c83270f0842ce67e2eb4237998c5f52e9926cca2 +Subproject commit edbd1089a5626e6269493b12af9f2c1f6de4fdb2 diff --git a/src/main/java/com/amazon/ion/SystemSymbols.java b/src/main/java/com/amazon/ion/SystemSymbols.java index 9da1e41dd..73f8344f2 100644 --- a/src/main/java/com/amazon/ion/SystemSymbols.java +++ b/src/main/java/com/amazon/ion/SystemSymbols.java @@ -121,17 +121,7 @@ private SystemSymbols() { } // Ion 1.1 Symbols /** - * The annotation that denotes an Ion encoding directive in Ion 1.1+. + * The name of the default module in Ion 1.1 */ - public static final String ION_ENCODING = "$ion_encoding"; - - /** - * The name of the symbol table s-expression within an Ion encoding directive. - */ - public static final String SYMBOL_TABLE = "symbol_table"; - - /** - * The name of the macro table s-expression within an Ion encoding directive. - */ - public static final String MACRO_TABLE = "macro_table"; + public static final String DEFAULT_MODULE = "_"; } diff --git a/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt b/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt index c1ad4efa4..dfb6334e0 100644 --- a/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt +++ b/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt @@ -3,6 +3,7 @@ package com.amazon.ion.impl import com.amazon.ion.* +import com.amazon.ion.SystemSymbols.* import com.amazon.ion.impl.macro.* import com.amazon.ion.impl.macro.MacroRef.Companion.byId import com.amazon.ion.impl.macro.MacroRef.Companion.byName @@ -28,7 +29,9 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va var isMacroTableAlreadyClassified = false private enum class State { - IN_ION_ENCODING_SEXP, + IN_DIRECTIVE_SEXP, + IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME, + IN_MODULE_DIRECTIVE_SEXP, IN_SYMBOL_TABLE_SEXP, IN_SYMBOL_TABLE_LIST, IN_MACRO_TABLE_SEXP, @@ -36,14 +39,29 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va READING_VALUE } - private fun classifySexpWithinEncodingDirective() { + private fun classifyDirective() { + errorIf(reader.type != IonType.SYMBOL) { "Ion encoding directives must start with a directive keyword; found ${reader.type}" } + val name: String = reader.stringValue() + // TODO: Add support for `import` and `encoding` directives + if (SystemSymbols_1_1.MODULE.text == name) { + state = State.IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME + } else if (SystemSymbols_1_1.IMPORT.text == name) { + throw IonException("'import' directive not yet supported") + } else if (SystemSymbols_1_1.ENCODING.text == name) { + throw IonException("'encoding' directive not yet supported") + } else { + throw IonException(String.format("'%s' is not a valid directive keyword", name)) + } + } + + private fun classifySexpWithinModuleDirective() { val name: String = reader.stringValue() state = if (SystemSymbols_1_1.SYMBOL_TABLE.text == name) { State.IN_SYMBOL_TABLE_SEXP } else if (SystemSymbols_1_1.MACRO_TABLE.text == name) { State.IN_MACRO_TABLE_SEXP } else { - throw IonException(String.format("\$ion_encoding expressions '%s' not supported.", name)) + throw IonException("'$name' clause not supported in module definition") } } @@ -64,7 +82,7 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va } isSymbolTableAlreadyClassified = true if (IonType.isText(type)) { - if (SystemSymbols.ION_ENCODING == reader.stringValue() && !isSymbolTableAppend) { + if (DEFAULT_MODULE == reader.stringValue() && !isSymbolTableAppend) { isSymbolTableAppend = true if (reader.next() == null) { return true @@ -73,10 +91,10 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va throw IonException("symbol_table s-expression must begin with a list.") } } else { - throw IonException("symbol_table s-expression must begin with either \$ion_encoding or a list.") + throw IonException("symbol_table s-expression must begin with either '_' or a list.") } } else if (type != IonType.LIST) { - throw IonException("symbol_table s-expression must begin with either \$ion_encoding or a list.") + throw IonException("symbol_table s-expression must begin with either '_' or a list.") } reader.stepIn() state = State.IN_SYMBOL_TABLE_LIST @@ -98,7 +116,7 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va } isMacroTableAlreadyClassified = true if (IonType.isText(type)) { - if (SystemSymbols.ION_ENCODING == reader.stringValue() && !isMacroTableAppend) { + if (SystemSymbols.DEFAULT_MODULE == reader.stringValue() && !isMacroTableAppend) { isMacroTableAppend = true if (reader.next() == null) { return true @@ -107,16 +125,27 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va throw IonException("macro_table s-expression must begin with s-expression(s).") } } else { - throw IonException("macro_table s-expression must begin with either \$ion_encoding or s-expression(s).") + throw IonException("macro_table s-expression must begin with either '_' or s-expression(s).") } } else if (type == IonType.SEXP) { localMacroMaxOffset = -1 } else { - throw IonException("macro_table s-expression must begin with either \$ion_encoding or s-expression(s).") + throw IonException("macro_table s-expression must begin with either '_' or s-expression(s).") } return false } + /** + * Utility function to make error cases more concise. + * @param condition the condition under which an IonException should be thrown + * @param lazyErrorMessage the message to use in the exception + */ + private inline fun errorIf(condition: Boolean, lazyErrorMessage: () -> String) { + if (condition) { + throw IonException(lazyErrorMessage()) + } + } + /** * Reads an encoding directive. After this method returns, the caller should access this class's properties to * retrieve the symbols and macros declared within the directive. @@ -126,30 +155,41 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va val macroCompiler = MacroCompiler({ key -> resolveMacro(encodingContext, key) }, readerAdapter) reader.stepIn() - state = State.IN_ION_ENCODING_SEXP + state = State.IN_DIRECTIVE_SEXP while (true) { when (state) { - State.IN_ION_ENCODING_SEXP -> { + State.IN_DIRECTIVE_SEXP -> { + errorIf(reader.next() == null) { "invalid Ion directive; missing directive keyword" } + classifyDirective() + } + State.IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME -> { + errorIf(reader.next() == null) { "invalid module directive; missing module name" } + errorIf(reader.type != IonType.SYMBOL) { "invalid module directive; module name must be a symbol" } + // TODO: Support other module names + errorIf(DEFAULT_MODULE != reader.stringValue()) { "IonJava currently supports only the default module" } + state = State.IN_MODULE_DIRECTIVE_SEXP + } + State.IN_MODULE_DIRECTIVE_SEXP -> { if (reader.next() == null) { reader.stepOut() state = State.READING_VALUE return } if (reader.type != IonType.SEXP) { - throw IonException("Ion encoding directives must contain only s-expressions.") + throw IonException("module definition must contain only s-expressions.") } reader.stepIn() if (reader.next() == null || !IonType.isText(reader.type)) { - throw IonException("S-expressions within encoding directives must begin with a text token.") + throw IonException("S-expressions within module definitions must begin with a text token.") } - classifySexpWithinEncodingDirective() + classifySexpWithinModuleDirective() } State.IN_SYMBOL_TABLE_SEXP -> { if (reader.next() == null || classifySymbolTable()) { reader.stepOut() - state = State.IN_ION_ENCODING_SEXP + state = State.IN_MODULE_DIRECTIVE_SEXP continue } } @@ -169,7 +209,7 @@ internal class EncodingDirectiveReader(private val reader: IonReader, private va State.IN_MACRO_TABLE_SEXP -> { if (reader.next() == null || classifyMacroTable()) { reader.stepOut() - state = State.IN_ION_ENCODING_SEXP + state = State.IN_MODULE_DIRECTIVE_SEXP continue } state = State.COMPILING_MACRO diff --git a/src/main/java/com/amazon/ion/impl/IonCursorBinary.java b/src/main/java/com/amazon/ion/impl/IonCursorBinary.java index 96753cb2c..62fffdeb2 100644 --- a/src/main/java/com/amazon/ion/impl/IonCursorBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonCursorBinary.java @@ -2115,7 +2115,7 @@ private void setCheckpointBeforeUnannotatedTypeId() { */ private void setMarker(long endIndex, Marker markerToSet) { if (parent != null && endIndex > parent.endIndex && parent.endIndex > DELIMITED_MARKER) { - throw new IonException("Value exceeds the length of its parent container."); + throw new IonException(String.format("Value [%s:%s] exceeds the length of its parent container [%s:%s].", peekIndex, endIndex, parent.startIndex, parent.endIndex)); } markerToSet.startIndex = peekIndex; markerToSet.endIndex = endIndex; diff --git a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index f9172daad..69e59fa09 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -41,7 +41,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -51,9 +50,10 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Supplier; -import static com.amazon.ion.SystemSymbols.ION_ENCODING; import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.DEFAULT_MODULE; import static com.amazon.ion.impl.IonReaderContinuableApplicationBinary.SYMBOLS_LIST_INITIAL_CAPACITY; import static com.amazon.ion.impl.IonTypeID.SYSTEM_SYMBOL_VALUE; import static com.amazon.ion.impl.bin.Ion_1_1_Constants.*; @@ -63,9 +63,6 @@ */ class IonReaderContinuableCoreBinary extends IonCursorBinary implements IonReaderContinuableCore, MacroAwareIonReader { - // The UTF-8 bytes that represent the text "$ion_encoding" for quick byte-by-byte comparisons. - private static final byte[] ION_ENCODING_UTF8 = ION_ENCODING.getBytes(StandardCharsets.UTF_8); - // Isolates the highest bit in a byte. private static final int HIGHEST_BIT_BITMASK = 0x80; @@ -1100,12 +1097,12 @@ static boolean bytesMatch(byte[] target, byte[] buffer, int start, int end) { } /** - * @return true if current value has a sequence of annotations that begins with `$ion_encoding`; otherwise, false. + * @return true if current value has a sequence of annotations that begins with `$ion`; otherwise, false. */ - boolean startsWithIonEncoding() { + boolean startsWithIonAnnotation() { if (minorVersion > 0) { Marker marker = annotationTokenMarkers.get(0); - return matchesSystemSymbol_1_1(marker, SystemSymbols_1_1.ION_ENCODING); + return matchesSystemSymbol_1_1(marker, SystemSymbols_1_1.ION); } return false; } @@ -1147,7 +1144,7 @@ private boolean isPositionedOnEncodingDirective() { && hasAnnotations && valueTid.type == IonType.SEXP && parent == null - && startsWithIonEncoding(); + && startsWithIonAnnotation(); } /** @@ -1159,7 +1156,7 @@ private boolean isPositionedOnEvaluatedEncodingDirective() { } Iterator annotations = macroEvaluatorIonReader.iterateTypeAnnotations(); return annotations.hasNext() - && annotations.next().equals(SystemSymbols_1_1.ION_ENCODING.getText()); + && annotations.next().equals(SystemSymbols_1_1.ION.getText()); } /** @@ -1248,14 +1245,30 @@ private boolean valueUnavailable() { return event == Event.NEEDS_DATA || event == Event.NEEDS_INSTRUCTION; } - private void classifySexpWithinEncodingDirective() { + private void classifyDirective() { + errorIf(getEncodingType() != IonType.SYMBOL, () -> "Ion encoding directives must start with a directive keyword."); + String name = getSymbolText(); + // TODO: Add support for `import` and `encoding` directives + if (SystemSymbols_1_1.MODULE.getText().equals(name)) { + state = State.IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME; + } else if (SystemSymbols_1_1.IMPORT.getText().equals(name)) { + throw new IonException("'import' directive not yet supported"); + } else if (SystemSymbols_1_1.ENCODING.getText().equals(name)) { + throw new IonException("'encoding' directive not yet supported"); + } else { + throw new IonException(String.format("'%s' is not a valid directive keyword", name)); + } + } + + private void classifySexpWithinModuleDirective() { String name = getSymbolText(); if (SystemSymbols_1_1.SYMBOL_TABLE.getText().equals(name)) { state = State.IN_SYMBOL_TABLE_SEXP; } else if (SystemSymbols_1_1.MACRO_TABLE.getText().equals(name)) { state = State.IN_MACRO_TABLE_SEXP; } else { - throw new IonException(String.format("$ion_encoding expressions '%s' not supported.", name)); + // TODO: add support for 'module' and 'import' clauses + throw new IonException(String.format("'%s' clause not supported in module definition", name)); } } @@ -1275,15 +1288,15 @@ private void classifySymbolTable() { } isSymbolTableAlreadyClassified = true; if (IonType.isText(type)) { - if (ION_ENCODING.equals(stringValue()) && !isSymbolTableAppend) { + if (DEFAULT_MODULE.equals(stringValue()) && !isSymbolTableAppend) { state = State.IN_APPENDED_SYMBOL_TABLE; } else { - throw new IonException("symbol_table s-expression must begin with either $ion_encoding or a list."); + throw new IonException("symbol_table s-expression must begin with either '_' or a list."); } } else if (type == IonType.LIST) { state = State.ON_SYMBOL_TABLE_LIST; } else { - throw new IonException("symbol_table s-expression must begin with either $ion_encoding or a list."); + throw new IonException("symbol_table s-expression must begin with either '_' or a list."); } } @@ -1303,10 +1316,10 @@ private void classifyMacroTable() { } isMacroTableAlreadyClassified = true; if (IonType.isText(type)) { - if (ION_ENCODING.equals(stringValue()) && !isMacroTableAppend) { + if (DEFAULT_MODULE.equals(stringValue()) && !isMacroTableAppend) { state = State.IN_APPENDED_MACRO_TABLE; } else { - throw new IonException("macro_table s-expression must begin with either $ion_encoding or s-expression(s)."); + throw new IonException("macro_table s-expression must begin with either '_' or s-expression(s)."); } } else if (type == IonType.SEXP) { localMacroMaxOffset = -1; @@ -1318,7 +1331,7 @@ private void classifyMacroTable() { private void stepOutOfSexpWithinEncodingDirective() { stepOutOfContainer(); - state = State.IN_ION_ENCODING_SEXP; + state = State.IN_MODULE_DIRECTIVE_SEXP_BODY; } /** @@ -1362,6 +1375,17 @@ private Event coreNextValue() { } } + /** + * Utility function to make error cases more concise. + * @param condition the condition under which an IonException should be thrown + * @param lazyErrorMessage the message to use in the exception + */ + private void errorIf(boolean condition, Supplier lazyErrorMessage) { + if (condition) { + throw new IonException(lazyErrorMessage.get()); + } + } + /** * Read an encoding directive. If the stream ends before the encoding directive finishes, `event` will be * `NEEDS_DATA` and this method can be called again when more data is available. @@ -1370,13 +1394,32 @@ void readEncodingDirective() { Event event; while (true) { switch (state) { - case ON_ION_ENCODING_SEXP: + case ON_DIRECTIVE_SEXP: if (Event.NEEDS_DATA == stepIntoContainer()) { return; } - state = State.IN_ION_ENCODING_SEXP; + state = State.IN_DIRECTIVE_SEXP; + break; + case IN_DIRECTIVE_SEXP: + event = coreNextValue(); + if (event == Event.NEEDS_DATA) { + return; + } + errorIf(event == Event.END_CONTAINER, () -> "invalid Ion directive; missing directive keyword"); + classifyDirective(); + break; + case IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME: + event = coreNextValue(); + if (event == Event.NEEDS_DATA) { + return; + } + errorIf(event == Event.END_CONTAINER, () -> "invalid module definition; missing module name"); + errorIf(getEncodingType() != IonType.SYMBOL, () -> "invalid module definition; module name must be a symbol"); + // TODO: Support other module names + errorIf(!DEFAULT_MODULE.equals(getSymbolText()), () -> "IonJava currently supports only the default module"); + state = State.IN_MODULE_DIRECTIVE_SEXP_BODY; break; - case IN_ION_ENCODING_SEXP: + case IN_MODULE_DIRECTIVE_SEXP_BODY: event = coreNextValue(); if (event == Event.NEEDS_DATA) { return; @@ -1386,30 +1429,30 @@ void readEncodingDirective() { return; } if (getEncodingType() != IonType.SEXP) { - throw new IonException("Ion encoding directives must contain only s-expressions."); + throw new IonException("module definitions must contain only s-expressions."); } - state = State.ON_SEXP_IN_ION_ENCODING; + state = State.ON_SEXP_IN_MODULE_DIRECTIVE; break; - case ON_SEXP_IN_ION_ENCODING: + case ON_SEXP_IN_MODULE_DIRECTIVE: if (Event.NEEDS_DATA == stepIntoContainer()) { return; } - state = State.IN_SEXP_IN_ION_ENCODING; + state = State.IN_SEXP_IN_MODULE_DIRECTIVE; break; - case IN_SEXP_IN_ION_ENCODING: + case IN_SEXP_IN_MODULE_DIRECTIVE: if (Event.NEEDS_DATA == coreNextValue()) { return; } if (!IonType.isText(getEncodingType())) { - throw new IonException("S-expressions within encoding directives must begin with a text token."); + throw new IonException("S-expressions within module definitions must begin with a text token."); } - state = State.CLASSIFYING_SEXP_IN_ION_ENCODING; + state = State.CLASSIFYING_SEXP_IN_MODULE_DIRECTIVE; break; - case CLASSIFYING_SEXP_IN_ION_ENCODING: + case CLASSIFYING_SEXP_IN_MODULE_DIRECTIVE: if (valueUnavailable()) { return; } - classifySexpWithinEncodingDirective(); + classifySexpWithinModuleDirective(); break; case IN_SYMBOL_TABLE_SEXP: event = coreNextValue(); @@ -1526,11 +1569,13 @@ void resetState() { * indicate that the reader is in the middle of reading an encoding directive. */ private enum State { - ON_ION_ENCODING_SEXP, - IN_ION_ENCODING_SEXP, - ON_SEXP_IN_ION_ENCODING, - IN_SEXP_IN_ION_ENCODING, - CLASSIFYING_SEXP_IN_ION_ENCODING, + ON_DIRECTIVE_SEXP, + IN_DIRECTIVE_SEXP, + IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME, + IN_MODULE_DIRECTIVE_SEXP_BODY, + ON_SEXP_IN_MODULE_DIRECTIVE, + IN_SEXP_IN_MODULE_DIRECTIVE, + CLASSIFYING_SEXP_IN_MODULE_DIRECTIVE, IN_SYMBOL_TABLE_SEXP, IN_APPENDED_SYMBOL_TABLE, ON_SYMBOL_TABLE_LIST, @@ -1848,7 +1893,7 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { } if (minorVersion == 1 && parent == null && isPositionedOnEncodingDirective()) { encodingDirectiveReader.resetState(); - state = State.ON_ION_ENCODING_SEXP; + state = State.ON_DIRECTIVE_SEXP; continue; } } else if (isEvaluatingEExpression) { @@ -1867,7 +1912,7 @@ Event transcodeNextTo(MacroAwareIonWriter writer) throws IOException { } if (parent == null && isPositionedOnEvaluatedEncodingDirective()) { encodingDirectiveReader.resetState(); - state = State.ON_ION_ENCODING_SEXP; + state = State.ON_DIRECTIVE_SEXP; continue; } } @@ -1909,7 +1954,7 @@ public Event nextValue() { } if (minorVersion == 1 && parent == null && isPositionedOnEncodingDirective()) { encodingDirectiveReader.resetState(); - state = State.ON_ION_ENCODING_SEXP; + state = State.ON_DIRECTIVE_SEXP; continue; } } else if (isEvaluatingEExpression) { @@ -1936,7 +1981,7 @@ public Event nextValue() { } if (parent == null && isPositionedOnEvaluatedEncodingDirective()) { encodingDirectiveReader.resetState(); - state = State.ON_ION_ENCODING_SEXP; + state = State.ON_DIRECTIVE_SEXP; continue; } } diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java index d5115cfc6..14a95279d 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -1146,14 +1146,14 @@ private boolean macroCompilationNotInProgress() { } /** - * @return true if current value has a sequence of annotations that begins with `$ion_encoding`; otherwise, false. + * @return true if current value has a sequence of annotations that begins with `$ion`; otherwise, false. */ - boolean startsWithIonEncoding() { + boolean startsWithIonAnnotation() { if (isEvaluatingEExpression) { - return SystemSymbols_1_1.ION_ENCODING.getText().equals(macroEvaluatorIonReader.iterateTypeAnnotations().next()); + return SystemSymbols_1_1.ION.getText().equals(macroEvaluatorIonReader.iterateTypeAnnotations().next()); } - // TODO also resolve symbol identifiers and compare against text that looks like $ion_encoding - return SystemSymbols_1_1.ION_ENCODING.getText().equals(_annotations[0].getText()); + // TODO also resolve symbol identifiers and compare against text that looks like $ion + return SystemSymbols_1_1.ION.getText().equals(_annotations[0].getText()); } /** @@ -1171,12 +1171,12 @@ private boolean isPositionedOnEncodingDirective() { && _value_type == IonType.SEXP && !isNullValue() && macroCompilationNotInProgress() - && startsWithIonEncoding(); + && startsWithIonAnnotation(); } /** * Reads an encoding directive and installs any symbols and/or macros found within. Upon calling this method, - * the reader must be positioned on an s-expression annotated with `$ion_encoding`. + * the reader must be positioned on a top-level s-expression annotated with `$ion`. */ private void readEncodingDirective() { if (encodingDirectiveReader == null) { diff --git a/src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt b/src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt index 9a0f0e988..8a63e6585 100644 --- a/src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/SystemSymbols_1_1.kt @@ -17,7 +17,7 @@ enum class SystemSymbols_1_1(val id: Int, val text: String) { SYMBOLS( /* */ 7, "symbols"), MAX_ID( /* */ 8, "max_id"), ION_SHARED_SYMBOL_TABLE( /* */ 9, "\$ion_shared_symbol_table"), - ION_ENCODING( /* */ 10, "\$ion_encoding"), + ENCODING( /* */ 10, "encoding"), ION_LITERAL( /* */ 11, "\$ion_literal"), ION_SHARED_MODULE( /* */ 12, "\$ion_shared_module"), MACRO( /* */ 13, "macro"), diff --git a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt index c3caf1d2e..7848944fc 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt @@ -195,7 +195,7 @@ internal class IonManagedWriter_1_1( * - It could mangle the name * - It could remove the name from a macro in macroTable, but then it would have to immediately flush to * make sure that any prior e-expressions are still valid. In addition, we would need to re-export all - * the other macros from `$ion_encoding`. + * the other macros from `_` (the default module). * - For now, we're just throwing an Exception. * * Visible for testing. @@ -361,15 +361,17 @@ internal class IonManagedWriter_1_1( } /** - * Writes an encoding directive for the current encoding context using the verbose `$ion_encoding::(...)` syntax, + * Writes an encoding directive for the current encoding context using the verbose `$ion::(module _ ...)` syntax, * and updates internal state accordingly. This always appends to the current encoding context. If there is nothing * to append, calling this function is a no-op. */ private fun writeVerboseEncodingDirective() { if (newSymbols.isEmpty() && newMacros.isEmpty()) return - systemData.writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + systemData.writeAnnotations(SystemSymbols_1_1.ION) writeSystemSexp { + writeSymbol(SystemSymbols_1_1.MODULE) + writeSymbol(SystemSymbols.DEFAULT_MODULE) writeVerboseSymbolTableClause() writeVerboseMacroTableClause() } @@ -420,7 +422,7 @@ internal class IonManagedWriter_1_1( // Add previous symbol table if (hasSymbolsToRetain) { if (newSymbols.size > 0) forceNoNewlines(false) - writeSymbol(SystemSymbols_1_1.ION_ENCODING) + writeSymbol(SystemSymbols.DEFAULT_MODULE) } // Add new symbols @@ -484,7 +486,7 @@ internal class IonManagedWriter_1_1( writeSymbol(SystemSymbols_1_1.MACRO_TABLE) if (newMacros.size > 0) forceNoNewlines(false) if (hasMacrosToRetain) { - writeSymbol(SystemSymbols_1_1.ION_ENCODING) + writeSymbol(SystemSymbols.DEFAULT_MODULE) } forceNoNewlines(false) newMacros.forEach { (macro, address) -> diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index 44c7b64e1..ae8d60acd 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro +import com.amazon.ion.SystemSymbols import com.amazon.ion.impl.* import com.amazon.ion.impl.SystemSymbols_1_1.* import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody @@ -52,23 +53,25 @@ enum class SystemMacro( /** * ```ion * (macro set_symbols (symbols*) - * $ion_encoding::( + * $ion::( * (symbol_table [(%symbols)]) - * (macro_table $ion_encoding) + * (macro_table _) * )) * ``` */ SetSymbols( 11, SET_SYMBOLS, listOf(zeroToManyTagged("symbols")), templateBody { - annotated(ION_ENCODING, ::sexp) { + annotated(ION, ::sexp) { + symbol(MODULE) + symbol(SystemSymbols.DEFAULT_MODULE) sexp { symbol(SYMBOL_TABLE) list { variable(0) } } sexp { symbol(MACRO_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) } } } @@ -77,24 +80,26 @@ enum class SystemMacro( /** * ```ion * (macro add_symbols (symbols*) - * $ion_encoding::( - * (symbol_table $ion_encoding [(%symbols)]) - * (macro_table $ion_encoding) + * $ion::( + * (symbol_table _ [(%symbols)]) + * (macro_table _) * )) * ``` */ AddSymbols( 12, ADD_SYMBOLS, listOf(zeroToManyTagged("symbols")), templateBody { - annotated(ION_ENCODING, ::sexp) { + annotated(ION, ::sexp) { + symbol(MODULE) + symbol(com.amazon.ion.SystemSymbols.DEFAULT_MODULE) sexp { symbol(SYMBOL_TABLE) - symbol(ION_ENCODING) + symbol(com.amazon.ion.SystemSymbols.DEFAULT_MODULE) list { variable(0) } } sexp { symbol(MACRO_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) } } } @@ -103,8 +108,8 @@ enum class SystemMacro( /** * ```ion * (macro set_macros (macros*) - * $ion_encoding::( - * (symbol_table $ion_encoding) + * $ion::(module _ + * (symbol_table _) * (macro_table (%macros)) * )) * ``` @@ -112,10 +117,12 @@ enum class SystemMacro( SetMacros( 13, SET_MACROS, listOf(zeroToManyTagged("macros")), templateBody { - annotated(ION_ENCODING, ::sexp) { + annotated(ION, ::sexp) { + symbol(MODULE) + symbol(SystemSymbols.DEFAULT_MODULE) sexp { symbol(SYMBOL_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) } sexp { symbol(MACRO_TABLE) @@ -128,23 +135,25 @@ enum class SystemMacro( /** * ```ion * (macro add_macros (macros*) - * $ion_encoding::( - * (symbol_table $ion_encoding) - * (macro_table $ion_encoding (%macros)) + * $ion::(module _ + * (symbol_table _) + * (macro_table _ (%macros)) * )) * ``` */ AddMacros( 14, ADD_MACROS, listOf(zeroToManyTagged("macros")), templateBody { - annotated(ION_ENCODING, ::sexp) { + annotated(ION, ::sexp) { + symbol(MODULE) + symbol(SystemSymbols.DEFAULT_MODULE) sexp { symbol(SYMBOL_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) } sexp { symbol(MACRO_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) variable(0) } } @@ -154,10 +163,10 @@ enum class SystemMacro( /** * ```ion * (macro use (catalog_key version?) - * $ion_encoding::( - * (import the_module (%catalog_key) (.if_none (%version) 1 (%version))) - * (symbol_table $ion_encoding the_module) - * (macro_table $ion_encoding the_module) + * $ion::(module _ + * (import the_module catalog_key (.default (%version) 1)) + * (symbol_table _ the_module) + * (macro_table _ the_module) * )) * ``` */ @@ -165,11 +174,14 @@ enum class SystemMacro( 15, USE, listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")), templateBody { val theModule = _Private_Utils.newSymbolToken("the_module") - annotated(ION_ENCODING, ::sexp) { + annotated(ION, ::sexp) { + symbol(MODULE) + symbol(SystemSymbols.DEFAULT_MODULE) sexp { symbol(IMPORT) symbol(theModule) variable(0) + // This is equivalent to `(.default (%version) 1)`, but eliminates a layer of indirection. macro(IfNone) { variable(1) int(1) @@ -178,12 +190,12 @@ enum class SystemMacro( } sexp { symbol(SYMBOL_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) symbol(theModule) } sexp { symbol(MACRO_TABLE) - symbol(ION_ENCODING) + symbol(SystemSymbols.DEFAULT_MODULE) symbol(theModule) } } diff --git a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt index 52f7b5e3a..40578c38b 100644 --- a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt +++ b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt @@ -417,7 +417,7 @@ class Ion_1_1_RoundTripTest { companion object { @JvmStatic - protected val DEBUG_MODE = false + protected val DEBUG_MODE = true @JvmStatic protected val ION = IonSystemBuilder.standard().build() as _Private_IonSystem diff --git a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java index 3178f4514..b72fb8cbb 100644 --- a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java +++ b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java @@ -65,6 +65,8 @@ public class EncodingDirectiveCompilationTest { private static final int FIRST_LOCAL_SYMBOL_ID = 1; + private static final String DEFAULT_MODULE_DIRECTIVE_PREFIX = "$ion::(module _"; + private static void assertMacroTablesContainsExpectedMappings(IonReader reader, StreamType streamType, SortedMap expected) { Map expectedByRef = streamType.newMacroTableByMacroRef(expected); @@ -73,9 +75,11 @@ private static void assertMacroTablesContainsExpectedMappings(IonReader reader, expectedByRef.forEach((k,v) -> assertEquals(v, actual.get(k))); } - private static void startEncodingDirective(IonRawWriter_1_1 writer) { - writer.writeAnnotations(SystemSymbols_1_1.ION_ENCODING); + private static void startModuleDirectiveForDefaultModule(IonRawWriter_1_1 writer) { + writer.writeAnnotations(SystemSymbols_1_1.ION); writer.stepInSExp(false); + writer.writeSymbol(SystemSymbols_1_1.MODULE); + writer.writeSymbol(SystemSymbols.DEFAULT_MODULE); } private static void endEncodingDirective(IonRawWriter_1_1 writer) { @@ -84,9 +88,9 @@ private static void endEncodingDirective(IonRawWriter_1_1 writer) { private static void writeEncodingDirectiveSymbolTable(IonRawWriter_1_1 writer, boolean append, String... userSymbols) { writer.stepInSExp(false); - writer.writeSymbol(SystemSymbols.SYMBOL_TABLE); + writer.writeSymbol(SystemSymbols_1_1.SYMBOL_TABLE); if (append) { - writer.writeSymbol(SystemSymbols.ION_ENCODING); + writer.writeSymbol(SystemSymbols.DEFAULT_MODULE); } writer.stepInList(false); for (String userSymbol : userSymbols) { @@ -110,7 +114,7 @@ private static Map makeSymbolsMap(int startId, String... userSy } private static Map initializeSymbolTable(IonRawWriter_1_1 writer, String... userSymbols) { - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, userSymbols); endEncodingDirective(writer); return makeSymbolsMap(FIRST_LOCAL_SYMBOL_ID, userSymbols); @@ -343,7 +347,7 @@ public void symbolsOnly(InputType inputType, StreamType streamType) throws Excep ByteArrayOutputStream out = new ByteArrayOutputStream(); IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo", "bar"); endEncodingDirective(writer); writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID); @@ -364,10 +368,10 @@ public void symbolAppendWithoutMacros(InputType inputType, StreamType streamType ByteArrayOutputStream out = new ByteArrayOutputStream(); IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo", "bar"); endEncodingDirective(writer); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, true, "baz"); endEncodingDirective(writer); writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID); @@ -397,7 +401,7 @@ public void structMacroWithOneOptional(InputType inputType, StreamType streamTyp } else { symbols = Collections.emptyMap(); } - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); startMacro(writer, symbols, "People"); writeMacroSignature(writer, symbols, "$ID", "$Name", "$Bald", "?"); @@ -448,7 +452,7 @@ public void constantMacroWithUserSymbol(InputType inputType, StreamType streamTy IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "Pi"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "Pi"); @@ -480,7 +484,7 @@ public void structMacroWithOneOptionalInvoked(InputType inputType, StreamType st IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "People", "ID", "Name", "Bald", "$ID", "$Name", "$Bald", "?"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); startMacro(writer, symbols, "People"); writeMacroSignature(writer, symbols, "$ID", "$Name", "$Bald", "?"); @@ -571,7 +575,7 @@ public void macroInvocationWithinStruct(InputType inputType, StreamType streamTy IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "People", "ID", "Name", "Bald", "$ID", "$Name", "$Bald", "?"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "People"); @@ -645,7 +649,7 @@ public void macroInvocationWithOptionalSuppressedBeforeEndWithinStruct(InputType IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "People", "ID", "Name", "Bald", "$ID", "$Name", "$Bald", "?"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "People"); @@ -723,7 +727,7 @@ public void constantMacroInvoked(InputType inputType, StreamType streamType) thr IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "Pi"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "Pi"); @@ -758,7 +762,7 @@ public void constantMacroInvoked(InputType inputType, StreamType streamType) thr private Macro writeSimonSaysMacro(IonRawWriter_1_1 writer) { writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "SimonSays", "anything"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "SimonSays"); @@ -993,7 +997,7 @@ public void macroInvocationsNestedWithinParameterMacroAwareTranscode() throws Ex substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 2) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 2) ); } @@ -1029,7 +1033,7 @@ public void twoArgumentGroups(InputType inputType, StreamType streamType) throws writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "Groups", "these", "those", "*", "+"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "Groups"); @@ -1100,7 +1104,7 @@ private byte[] macroInvocationInMacroDefinition(StreamType streamType) { writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "SimonSays", "anything", "Echo"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writeEncodingDirectiveSymbolTable(writer, "foo"); startMacroTable(writer); startMacro(writer, symbols, "SimonSays"); @@ -1160,7 +1164,7 @@ public void macroInvocationInMacroDefinitionMacroAwareTranscode() throws Excepti substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 2) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 2) ); } @@ -1174,7 +1178,7 @@ public void blobsAndClobs(InputType inputType, StreamType streamType) throws Exc byte[] clobContents = new byte[] {3}; writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "lobs", "a"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); startMacro(writer, symbols, "lobs"); writeMacroSignature(writer, symbols, "a"); @@ -1221,7 +1225,7 @@ public void macroInvocationInTaggedExpressionGroup(InputType inputType, StreamTy writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "foo", "value"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); startMacro(writer, symbols, "foo"); writeMacroSignature(writer, symbols, "value", "*"); @@ -1262,7 +1266,7 @@ public void taglessExpressionGroup(InputType inputType, StreamType streamType) t writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "foo", "value"); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); startMacro(writer, symbols, "foo"); writeMacroSignatureFromDatagram(writer, symbols, LOADER.load("uint8::value '*'")); @@ -1368,7 +1372,7 @@ public void macroInvocationsProduceEncodingDirectivesThatModifySymbolTableMacroA substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 2), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 0) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 0) ); } @@ -1467,7 +1471,7 @@ public void macroInvocationsProduceEncodingDirectivesThatModifyMacroTableMacroAw substringCount(SystemSymbols_1_1.ADD_MACROS, 2), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 1), substringCount(SystemSymbols_1_1.SET_MACROS, 1), - substringCount(SystemSymbols_1_1.ION_ENCODING, 0) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 0) ); } @@ -1478,9 +1482,9 @@ public void multipleListsWithinSymbolTableDeclaration(InputType inputType, Strea IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); writer.stepInSExp(false); - writer.writeSymbol(SystemSymbols.SYMBOL_TABLE); + writer.writeSymbol(SystemSymbols_1_1.SYMBOL_TABLE); writer.stepInList(false); writer.writeString("foo"); writer.stepOut(); @@ -1509,9 +1513,9 @@ private byte[] emptyMacroAppendToEmptyTable(StreamType streamType) { IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); - startEncodingDirective(writer); + startModuleDirectiveForDefaultModule(writer); startMacroTable(writer); - writer.writeSymbol(SystemSymbols_1_1.ION_ENCODING); + writer.writeSymbol(SystemSymbols.DEFAULT_MODULE); endMacroTable(writer); endEncodingDirective(writer); @@ -1536,7 +1540,7 @@ public void emptyMacroAppendToEmptyTableMacroAwareTranscode() throws Exception { substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 0) // The empty append to an empty table has no effect, and it is not transcoded. This is a known limitation. + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 0) // The empty append to an empty table has no effect, and it is not transcoded. This is a known limitation. ); } @@ -1552,7 +1556,7 @@ private byte[] emptyMacroAppendToNonEmptyTable(StreamType streamType) { )); Map symbols = Collections.emptyMap(); - startEncodingDirective(writer); { + startModuleDirectiveForDefaultModule(writer); { startMacroTable(writer); { startMacro(writer, symbols, "foo"); { writeMacroSignature(writer, symbols, "x"); @@ -1562,9 +1566,9 @@ private byte[] emptyMacroAppendToNonEmptyTable(StreamType streamType) { } endEncodingDirective(writer); - startEncodingDirective(writer); { + startModuleDirectiveForDefaultModule(writer); { startMacroTable(writer); { - writer.writeSymbol(SystemSymbols_1_1.ION_ENCODING); + writer.writeSymbol(SystemSymbols.DEFAULT_MODULE); } endMacroTable(writer); writeEncodingDirectiveSymbolTable(writer, true, "bar"); } endEncodingDirective(writer); @@ -1595,7 +1599,7 @@ public void emptyMacroAppendToNonEmptyTableMacroAwareTranscode() throws Exceptio substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 3) // Two encoding directives, plus one $ion_encoding symbol to denote the macro table append. + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 2) // Two encoding directives ); } @@ -1616,7 +1620,7 @@ private byte[] invokeUnqualifiedSystemMacroInTDL(StreamType streamType) { Map symbols = Collections.emptyMap(); - startEncodingDirective(writer); { + startModuleDirectiveForDefaultModule(writer); { startMacroTable(writer); { // Define our macro (macro foo (x) (.default (%x) "hello world")) startMacro(writer, symbols, "foo"); { @@ -1655,7 +1659,7 @@ public void invokeUnqualifiedSystemMacroInTDLMacroAwareTranscode() throws Except substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 1) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 1) ); } @@ -1677,7 +1681,7 @@ public void multipleIonVersionMarkersMacroAwareTranscode() throws Exception { substringCount(SystemSymbols_1_1.ADD_MACROS, 0), substringCount(SystemSymbols_1_1.SET_SYMBOLS, 0), substringCount(SystemSymbols_1_1.SET_MACROS, 0), - substringCount(SystemSymbols_1_1.ION_ENCODING, 0) + substringCount(DEFAULT_MODULE_DIRECTIVE_PREFIX, 0) ); } diff --git a/src/test/java/com/amazon/ion/impl/IonReaderContinuableApplicationBinaryTest.java b/src/test/java/com/amazon/ion/impl/IonReaderContinuableApplicationBinaryTest.java index e7ecd4b29..7343768cf 100644 --- a/src/test/java/com/amazon/ion/impl/IonReaderContinuableApplicationBinaryTest.java +++ b/src/test/java/com/amazon/ion/impl/IonReaderContinuableApplicationBinaryTest.java @@ -201,13 +201,17 @@ public void systemSymbolsEncodedUsingUserIdsAndInlineText_1_1(boolean constructF IonReaderContinuableApplicationBinary reader = initializeReader( constructFromBytes, 0xE0, 0x01, 0x01, 0xEA, - 0xE7, 0xE7, '$', 'i', 'o', 'n', '_', 'e', 'n', 'c', 'o', 'd', 'i', 'n', 'g', // $ion_encoding:: - 0xFC, 0x27, // s-expression, length 19 - 0xFC, 0x23, // s-expression, length 17 + 0xE7, 0xF9, '$', 'i', 'o', 'n', // $ion:: + 0xCD, // s-expression, length 13 + 0xEE, 0x10, // 'module' (encoded as system symbol ID 16) + 0xA1, '_', // Inline symbol '_' + 0xC8, // s-expression, length 8 0xEE, 0x0F, // 'symbol_table' (encoded as system symbol ID 15) - 0xBE, 0x9D, '$', 'i', 'o', 'n', '_', 'e', 'n', 'c', 'o', 'd', 'i', 'n', 'g', // ["$ion_encoding"] - 0xE4, 0x03, // $1::, where $1 is a local SID that points to the text "$ion_encoding" - 0xC6, // s-expression, length 6 + 0xB5, 0x94, '$', 'i', 'o', 'n', // ["$ion"] + 0xE4, 0x03, // $1::, where $1 is a local SID that points to the text "$ion" + 0xCA, // s-expression, length 10 + 0xEE, 0x10, // 'module' (encoded as system symbol ID 16) + 0xA1, '_', // Inline symbol '_' 0xC5, // s-expression, length 5 0xEE, 0x0F, // 'symbol_table' (encoded as system symbol ID 15) 0xB2, 0x91, 'a', // ["a"] diff --git a/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java b/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java index 37487506a..d2e29e2eb 100644 --- a/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java +++ b/src/test/java/com/amazon/ion/impl/IonReaderContinuableCoreBinaryTest.java @@ -1108,8 +1108,10 @@ public void addSymbolsSystemMacro(boolean constructFromBytes) throws Exception { @ValueSource(booleans = {true, false}) public void systemReaderWrapperReadsEncodingDirective(boolean constructFromBytes) throws Exception { byte[] data = withIvm(1, bytes( - 0xE7, 0x01, 0x6A, // One FlexSym annotation, with opcode, opcode 6A = system symbol A = $ion_encoding - 0xC6, // ( + 0xE7, 0x01, 0x61, // One FlexSym annotation, with opcode, opcode 61 = system symbol 1 = $ion + 0xCA, // ( + 0xEE, 0x10, // module + 0xA1, '_', // _ 0xC5, 0xEE, 0x0F, // S-exp, system symbol 0xF = symbol_table 0xB2, 0x91, 'a', // ["a"] 0xE1, 0x01 // $1 = a @@ -1134,10 +1136,12 @@ public void systemReaderWrapperReadsEncodingDirective(boolean constructFromBytes @ValueSource(booleans = {true, false}) public void systemReaderWrapperReadsEncodingDirectiveWithAppend(boolean constructFromBytes) throws Exception { byte[] data = withIvm(1, bytes( - 0xE7, 0x01, 0x6A, // One FlexSym annotation, with opcode, opcode 6A = system symbol A = $ion_encoding - 0xC8, // ( + 0xE7, 0x01, 0x61, // One FlexSym annotation, with opcode, opcode 61 = system symbol 1 = $ion + 0xCC, // ( + 0xEE, 0x10, // module + 0xA1, '_', // Inline symbol '_' 0xC7, 0xEE, 0x0F, // S-exp, system symbol 0xF = symbol_table - 0xEE, 0x0A, // System symbol value, ID 10 = $ion_encoding, denoting append + 0xA1, '_', // Inline symbol '_' 0xB2, 0x91, 'a', // ["a"] 0xE1, SystemSymbols_1_1.size() + 1 // first local symbol = a )); diff --git a/src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java b/src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java index 330910915..33041c6ea 100644 --- a/src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java +++ b/src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java @@ -6046,8 +6046,10 @@ public void readIon11SystemSymbolFromUserId(boolean constructFromBytes) throws E public void readIon11EncodingDirectiveUsingSystemSymbolAnnotation(boolean constructFromBytes) throws Exception { reader = readerForIon11( bytes( - 0xE7, 0x01, 0x6A, // One FlexSym annotation, with opcode, opcode 6A = system symbol A = $ion_encoding - 0xC6, // ( + 0xE7, 0x01, 0x61, // One FlexSym annotation, with opcode, opcode 61 = system symbol 1 = $ion + 0xCA, // ( + 0xEE, 0x10, // module + 0xA1, '_', // _ 0xC5, 0xEE, 0x0F, // S-exp, system symbol 0xF = symbol_table 0xB2, 0x91, 'a', // ["a"] 0xE1, 0x01 // $1 = a diff --git a/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt b/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt index fcb8f532a..8df7407f4 100644 --- a/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt +++ b/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt @@ -227,8 +227,10 @@ internal class IonManagedWriter_1_1_Test { TestUtils.cleanCommentedHexBytes( """ E0 01 01 EA | IVM - E7 01 6A | $ion_encoding:: - C6 | ( + E7 01 61 | $ion:: + CA | ( + EE 10 | module + A1 5F | _ C5 | ( EE 0F | symbol_table B2 91 61 | ["a"] diff --git a/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt b/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt index bfc27cc52..fd81ffad5 100644 --- a/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt +++ b/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt @@ -877,7 +877,7 @@ class IonRawBinaryWriterTest_1_1 { val expectedBytes = "E8 15 01 6A 6F" assertWriterOutputEquals(expectedBytes) { writeAnnotations(10) - writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeAnnotations(SystemSymbols_1_1.ENCODING) writeBool(false) } } @@ -887,7 +887,7 @@ class IonRawBinaryWriterTest_1_1 { val expectedBytes = "E8 FB 66 6F 6F 01 6A 6F" assertWriterOutputEquals(expectedBytes) { writeAnnotations("foo") - writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeAnnotations(SystemSymbols_1_1.ENCODING) writeBool(false) } } @@ -898,7 +898,7 @@ class IonRawBinaryWriterTest_1_1 { assertWriterOutputEquals(expectedBytes) { writeAnnotations(10) writeAnnotations("foo") - writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeAnnotations(SystemSymbols_1_1.ENCODING) writeBool(false) } } diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 906991013..358a08d99 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -6,7 +6,7 @@ import com.amazon.ion.* import com.amazon.ion.impl.* import com.amazon.ion.impl.SystemSymbols_1_1.* import com.amazon.ion.impl._Private_Utils.newSymbolToken -import com.amazon.ion.impl.bin.IonManagedWriter_1_1_Test.Companion.ion_encoding +import com.amazon.ion.impl.bin.IonManagedWriter_1_1_Test.Companion.ion import com.amazon.ion.impl.macro.Expression.* import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.eExpBody import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody @@ -1140,9 +1140,9 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( + $ion::(module _ (symbol_table [a, b, c]) - (macro_table $ion_encoding) + (macro_table _) ) """ ) @@ -1162,9 +1162,9 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( - (symbol_table $ion_encoding [a, b, c]) - (macro_table $ion_encoding) + $ion::(module _ + (symbol_table _ [a, b, c]) + (macro_table _) ) """ ) @@ -1187,8 +1187,8 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( - (symbol_table $ion_encoding) + $ion::(module _ + (symbol_table _) (macro_table (macro answer () 42)) ) @@ -1213,10 +1213,10 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( - (symbol_table $ion_encoding) + $ion::(module _ + (symbol_table _) (macro_table - $ion_encoding + _ (macro answer () 42)) ) """ @@ -1234,10 +1234,10 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( + $ion::(module _ (import the_module "com.amazon.Foo" 2) - (symbol_table $ion_encoding the_module) - (macro_table $ion_encoding the_module) + (symbol_table _ the_module) + (macro_table _ the_module) ) """ ) @@ -1254,10 +1254,10 @@ class MacroEvaluatorTest { } evaluator.assertExpansion( """ - $ion_encoding::( + $ion::(module _ (import the_module "com.amazon.Foo" 1) - (symbol_table $ion_encoding the_module) - (macro_table $ion_encoding the_module) + (symbol_table _ the_module) + (macro_table _ the_module) ) """ ) From fece5cec275cb4ac75fbf2ab26abf483c8edc8e3 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 4 Dec 2024 15:38:40 -0800 Subject: [PATCH 2/3] Adds suggested changes --- .../java/com/amazon/ion/impl/IonCursorBinary.java | 2 +- .../ion/impl/IonReaderContinuableCoreBinary.java | 15 +++++++-------- .../java/com/amazon/ion/impl/macro/SystemMacro.kt | 4 ++-- .../java/com/amazon/ion/Ion_1_1_RoundTripTest.kt | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/IonCursorBinary.java b/src/main/java/com/amazon/ion/impl/IonCursorBinary.java index 62fffdeb2..026299795 100644 --- a/src/main/java/com/amazon/ion/impl/IonCursorBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonCursorBinary.java @@ -2115,7 +2115,7 @@ private void setCheckpointBeforeUnannotatedTypeId() { */ private void setMarker(long endIndex, Marker markerToSet) { if (parent != null && endIndex > parent.endIndex && parent.endIndex > DELIMITED_MARKER) { - throw new IonException(String.format("Value [%s:%s] exceeds the length of its parent container [%s:%s].", peekIndex, endIndex, parent.startIndex, parent.endIndex)); + throw new IonException(String.format("Value [%d:%d] exceeds the length of its parent container [%d:%d].", peekIndex, endIndex, parent.startIndex, parent.endIndex)); } markerToSet.startIndex = peekIndex; markerToSet.endIndex = endIndex; diff --git a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 69e59fa09..9084096c8 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -50,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; -import java.util.function.Supplier; import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; import static com.amazon.ion.SystemSymbols.DEFAULT_MODULE; @@ -1378,11 +1377,11 @@ private Event coreNextValue() { /** * Utility function to make error cases more concise. * @param condition the condition under which an IonException should be thrown - * @param lazyErrorMessage the message to use in the exception + * @param errorMessage the message to use in the exception */ - private void errorIf(boolean condition, Supplier lazyErrorMessage) { + private void errorIf(boolean condition, String errorMessage) { if (condition) { - throw new IonException(lazyErrorMessage.get()); + throw new IonException(errorMessage); } } @@ -1405,7 +1404,7 @@ void readEncodingDirective() { if (event == Event.NEEDS_DATA) { return; } - errorIf(event == Event.END_CONTAINER, () -> "invalid Ion directive; missing directive keyword"); + errorIf(event == Event.END_CONTAINER, "invalid Ion directive; missing directive keyword"); classifyDirective(); break; case IN_MODULE_DIRECTIVE_SEXP_AWAITING_MODULE_NAME: @@ -1413,10 +1412,10 @@ void readEncodingDirective() { if (event == Event.NEEDS_DATA) { return; } - errorIf(event == Event.END_CONTAINER, () -> "invalid module definition; missing module name"); - errorIf(getEncodingType() != IonType.SYMBOL, () -> "invalid module definition; module name must be a symbol"); + errorIf(event == Event.END_CONTAINER, "invalid module definition; missing module name"); + errorIf(getEncodingType() != IonType.SYMBOL, "invalid module definition; module name must be a symbol"); // TODO: Support other module names - errorIf(!DEFAULT_MODULE.equals(getSymbolText()), () -> "IonJava currently supports only the default module"); + errorIf(!DEFAULT_MODULE.equals(getSymbolText()), "IonJava currently supports only the default module"); state = State.IN_MODULE_DIRECTIVE_SEXP_BODY; break; case IN_MODULE_DIRECTIVE_SEXP_BODY: diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index ae8d60acd..6a9df0917 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -53,7 +53,7 @@ enum class SystemMacro( /** * ```ion * (macro set_symbols (symbols*) - * $ion::( + * $ion::(module _ * (symbol_table [(%symbols)]) * (macro_table _) * )) @@ -80,7 +80,7 @@ enum class SystemMacro( /** * ```ion * (macro add_symbols (symbols*) - * $ion::( + * $ion::(module _ * (symbol_table _ [(%symbols)]) * (macro_table _) * )) diff --git a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt index 40578c38b..52f7b5e3a 100644 --- a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt +++ b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt @@ -417,7 +417,7 @@ class Ion_1_1_RoundTripTest { companion object { @JvmStatic - protected val DEBUG_MODE = true + protected val DEBUG_MODE = false @JvmStatic protected val ION = IonSystemBuilder.standard().build() as _Private_IonSystem From 0b072ee7e70e93bf115b712787e3fea96e9c194d Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 4 Dec 2024 15:42:33 -0800 Subject: [PATCH 3/3] Adds suggested change that was missed in the last commit --- .../com/amazon/ion/impl/IonReaderContinuableCoreBinary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 9084096c8..947bf02f4 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -1245,7 +1245,7 @@ private boolean valueUnavailable() { } private void classifyDirective() { - errorIf(getEncodingType() != IonType.SYMBOL, () -> "Ion encoding directives must start with a directive keyword."); + errorIf(getEncodingType() != IonType.SYMBOL, "Ion encoding directives must start with a directive keyword."); String name = getSymbolText(); // TODO: Add support for `import` and `encoding` directives if (SystemSymbols_1_1.MODULE.getText().equals(name)) {