Ensure parser error snippets are valid UTF-8

Fix: #755

Error messages now include a snippet of the document
that doesn't parse to help locate the issue, however
the way it was done wasn't UTF-8 aware, and it could
result in exception messages with truncated characters.

It would be nice to go a bit farther and actually support
codepoints, but it's a lot of complexity to do it in C,
perhaps if we move that logic to Ruby given it's not a
performance sensitive codepath.

Author: byroot

CHANGES.md

@@ -1,5 +1,7 @@
 # Changes
 
+* Ensure document snippets that are included in parser errors don't include truncated multibyte characters.
+
 ### 2025-02-10 (2.10.1)
 
 * Fix a compatibility issue with `MultiJson.dump(obj, pretty: true)`: `no implicit conversion of false into Proc (TypeError)`.

ext/json/ext/parser/parser.c

@@ -454,15 +454,24 @@ RBIMPL_ATTR_NORETURN()
 #endif
 static void raise_parse_error(const char *format, const char *start)
 {
-    char buffer[PARSE_ERROR_FRAGMENT_LEN + 1];
+    unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1];
 
     size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0;
     const char *ptr = start;
 
     if (len == PARSE_ERROR_FRAGMENT_LEN) {
         MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN);
-        buffer[PARSE_ERROR_FRAGMENT_LEN] = '\0';
-        ptr = buffer;
+
+        while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte
+            len--;
+        }
+
+        if (buffer[len - 1] >= 0xC0) { // multibyte character start
+            len--;
+        }
+
+        buffer[len] = '\0';
+        ptr = (const char *)buffer;
     }
 
     rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr);

test/json/json_parser_test.rb

@@ -645,6 +645,22 @@ def test_parse_error_incomplete_hash
     end
   end
 
+  def test_parse_error_snippet
+    omit "C ext only test" unless RUBY_ENGINE == "ruby"
+
+    error = assert_raise(JSON::ParserError) { JSON.parse("あああああああああああああああああああああああ") }
+    assert_equal "unexpected character: 'ああああああああああ'", error.message
+
+    error = assert_raise(JSON::ParserError) { JSON.parse("aあああああああああああああああああああああああ") }
+    assert_equal "unexpected character: 'aああああああああああ'", error.message
+
+    error = assert_raise(JSON::ParserError) { JSON.parse("abあああああああああああああああああああああああ") }
+    assert_equal "unexpected character: 'abあああああああああ'", error.message
+
+    error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") }
+    assert_equal "unexpected character: 'abcあああああああああ'", error.message
+  end
+
   def test_parse_leading_slash
     # ref: https://github.com/ruby/ruby/pull/12598
     assert_raise(JSON::ParserError) do