From 2e015ff839ed2044ead0fd27b63a912766270a1b Mon Sep 17 00:00:00 2001 From: Rahim Packir Saibo Date: Wed, 5 Mar 2025 18:03:03 +0000 Subject: [PATCH 001/506] Fix JSON::GeneratorError#detailed_message with Ruby < 3.2 --- lib/json/common.rb | 7 +++++-- test/json/json_generator_test.rb | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 3f1e5ce96..9094df00a 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -152,10 +152,13 @@ def initialize(message, invalid_object = nil) end def detailed_message(...) + # Exception#detailed_message doesn't exist until Ruby 3.2 + super_message = defined?(super) ? super : message + if @invalid_object.nil? - super + super_message else - "#{super}\nInvalid object: #{@invalid_object.inspect}" + "#{super_message}\nInvalid object: #{@invalid_object.inspect}" end end end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index c67cd3349..4a92801fe 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -410,6 +410,14 @@ def test_json_generate end end + def test_json_generate_error_detailed_message + error = assert_raise JSON::GeneratorError do + generate(["\xea"]) + end + + assert_not_nil(error.detailed_message) + end + def test_json_generate_unsupported_types assert_raise JSON::GeneratorError do generate(Object.new, strict: true) From c079793b7655b749a4d85f5c8e6bd2649fd31c0c Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Mon, 10 Mar 2025 04:34:57 -0500 Subject: [PATCH 002/506] Avoid fast-path IO writes when IO has ext enc This works around a bug in JRuby's IOOutputStream logic whereby an IO with an external encoding will always fail to write incoming bytes. We use base logic to detect if the target object is a "real IO" and if so and it has an external encoding, we drop the realIO and. This causes the rest of IOOutputStream to avoid the fast path and always use dyncall logic with RubyString wrappers. This works around the issue in jruby/jruby#8682. This should be temporary, since it will definitely degrade direct writes to such IO objects, but a longer-term fix for the encoding issues spelled out in jruby/jruby#8682 will need to come first, or else json will have to be modified to not use IOOutputStream at all. --- java/src/json/ext/Generator.java | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index c46a1e475..852509205 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -5,6 +5,7 @@ */ package json.ext; +import org.jcodings.Encoding; import org.jcodings.specific.UTF8Encoding; import org.jruby.Ruby; import org.jruby.RubyArray; @@ -15,6 +16,7 @@ import org.jruby.RubyFixnum; import org.jruby.RubyFloat; import org.jruby.RubyHash; +import org.jruby.RubyIO; import org.jruby.RubyString; import org.jruby.RubySymbol; import org.jruby.runtime.Helpers; @@ -81,11 +83,41 @@ static RubyString generateJson(ThreadContext context, T return handler.generateNew(context, session, object); } - BufferedOutputStream buffer = new BufferedOutputStream(new IOOutputStream(io), IO_BUFFER_SIZE); + BufferedOutputStream buffer = + new BufferedOutputStream( + new PatchedIOOutputStream(io, UTF8Encoding.INSTANCE), + IO_BUFFER_SIZE); handler.generateToBuffer(context, session, object, buffer); return io; } + /** + * A version of IOOutputStream hacked to avoid fast-path RubyIO calls when the target IO has an external encoding. + * + * All calls to the underlying IO will be done dynamically and all incoming bytes wrapped in RubyString instances. + * This avoids bugs in the fast-path logic in JRuby 9.4.12.0 and earlier that fails to properly handle writing bytes + * when the source and target destination are the same. + * + * See https://github.com/jruby/jruby/issues/8682 + */ + private static class PatchedIOOutputStream extends IOOutputStream { + public PatchedIOOutputStream(IRubyObject io, Encoding encoding) { + super(io, encoding); + } + + @Override + public RubyIO getRealIO(IRubyObject io) { + RubyIO realIO = super.getRealIO(io); + + // if the real IO has an external encoding, don't use fast path + if (realIO == null || realIO.getEnc() != null) { + return null; + } + + return realIO; + } + } + /** * Returns the best serialization handler for the given object. */ From 7d0637b9e6e0269c88418b142cb9a1ef2799587d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 11 Mar 2025 20:50:26 +0100 Subject: [PATCH 003/506] Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. `"\u1"` would raise, but `"\u1zzz"` wouldn't. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 85 +++++++++++++++++------------------ test/json/json_parser_test.rb | 5 +++ 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d2ed40525..f57dd36af 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Changes +* Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. * Ensure document snippets that are included in parser errors don't include truncated multibyte characters. ### 2025-02-10 (2.10.1) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 776eb916f..0a1d93757 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -341,6 +341,44 @@ static void rvalue_stack_eagerly_release(VALUE handle) } } + +#ifndef HAVE_STRNLEN +static size_t strnlen(const char *s, size_t maxlen) +{ + char *p; + return ((p = memchr(s, '\0', maxlen)) ? p - s : maxlen); +} +#endif + +#define PARSE_ERROR_FRAGMENT_LEN 32 +#ifdef RBIMPL_ATTR_NORETURN +RBIMPL_ATTR_NORETURN() +#endif +static void raise_parse_error(const char *format, const char *start) +{ + unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; + + size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0; + const char *ptr = start; + + if (len == PARSE_ERROR_FRAGMENT_LEN) { + MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN); + + while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte + len--; + } + + if (buffer[len - 1] >= 0xC0) { // multibyte character start + len--; + } + + buffer[len] = '\0'; + ptr = (const char *)buffer; + } + + rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr); +} + /* unicode */ static const signed char digit_values[256] = { @@ -362,21 +400,19 @@ static const signed char digit_values[256] = { static uint32_t unescape_unicode(const unsigned char *p) { - const uint32_t replacement_char = 0xFFFD; - signed char b; uint32_t result = 0; b = digit_values[p[0]]; - if (b < 0) return replacement_char; + if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[1]]; - if (b < 0) return replacement_char; + if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[2]]; - if (b < 0) return replacement_char; + if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[3]]; - if (b < 0) return replacement_char; + if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); result = (result << 4) | (unsigned char)b; return result; } @@ -440,43 +476,6 @@ typedef struct JSON_ParserStateStruct { static const rb_data_type_t JSON_ParserConfig_type; -#ifndef HAVE_STRNLEN -static size_t strnlen(const char *s, size_t maxlen) -{ - char *p; - return ((p = memchr(s, '\0', maxlen)) ? p - s : maxlen); -} -#endif - -#define PARSE_ERROR_FRAGMENT_LEN 32 -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_parse_error(const char *format, const char *start) -{ - unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; - - size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0; - const char *ptr = start; - - if (len == PARSE_ERROR_FRAGMENT_LEN) { - MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN); - - while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte - len--; - } - - if (buffer[len - 1] >= 0xC0) { // multibyte character start - len--; - } - - buffer[len] = '\0'; - ptr = (const char *)buffer; - } - - rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr); -} - static const bool whitespace[256] = { [' '] = 1, ['\t'] = 1, diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index ae0f285d3..87b78fb0c 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -311,6 +311,11 @@ def test_invalid_unicode_escape assert_raise(JSON::ParserError) { parse('"\uaa"') } assert_raise(JSON::ParserError) { parse('"\uaaa"') } assert_equal "\uaaaa", parse('"\uaaaa"') + + assert_raise(JSON::ParserError) { parse('"\u______"') } + assert_raise(JSON::ParserError) { parse('"\u1_____"') } + assert_raise(JSON::ParserError) { parse('"\u11____"') } + assert_raise(JSON::ParserError) { parse('"\u111___"') } end def test_parse_big_integers From cf242d89a0523bacd5238a59c77b33411b8c3208 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 11 Mar 2025 18:49:14 +0100 Subject: [PATCH 004/506] Fix potential out of bound read in `json_string_unescape`. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 2 +- test/json/json_parser_test.rb | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f57dd36af..bae88d71f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Changes +* Fix a potential crash in the C extension parser. * Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. * Ensure document snippets that are included in parser errors don't include truncated multibyte characters. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 0a1d93757..d990612a2 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -608,7 +608,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c buffer = RSTRING_PTR(result); bufferStart = buffer; - while ((pe = memchr(pe, '\\', stringEnd - pe))) { + while (pe < stringEnd && (pe = memchr(pe, '\\', stringEnd - pe))) { unescape = (char *) "?"; unescape_len = 1; if (pe > p) { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 87b78fb0c..5519d4c59 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -673,6 +673,13 @@ def test_parse_leading_slash end end + def test_parse_malformated_unicode_escapes + assert_equal "�|", JSON.parse('"\\u1gef|"') + assert_equal "�|", JSON.parse('"\\u12ge|"') + assert_equal "�|", JSON.parse('"\\u123g|"') + assert_equal '�\\\\"', JSON.parse('"\\u1\\\\\\\\\\\\\\\\"') + end + private def string_deduplication_available? From 350c1fd154eaf7840f696c623362478a9148166c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 12 Mar 2025 14:06:14 +0100 Subject: [PATCH 005/506] Release 2.10.2 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index bae88d71f..697624238 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +### 2025-03-12 (2.10.2) + * Fix a potential crash in the C extension parser. * Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. * Ensure document snippets that are included in parser errors don't include truncated multibyte characters. diff --git a/lib/json/version.rb b/lib/json/version.rb index 65bcb976f..9c21f18b3 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.10.1' + VERSION = '2.10.2' end From 2074f0c2c57c383eca007b7cec1d35defe58dc60 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 12 Mar 2025 14:12:38 +0100 Subject: [PATCH 006/506] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 697624238..eca83cdaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,8 @@ * Fix a potential crash in the C extension parser. * Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. * Ensure document snippets that are included in parser errors don't include truncated multibyte characters. +* Ensure parser error snippets are valid UTF-8. +* Fix `JSON::GeneratorError#detailed_message` on Ruby < 3.2 ### 2025-02-10 (2.10.1) From 7d77415108c1fd0f55543cf0705abcb7c679d17f Mon Sep 17 00:00:00 2001 From: eno Date: Sun, 16 Mar 2025 20:28:46 +0100 Subject: [PATCH 007/506] Faster float formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit provides an alternative implementation for a float → decimal conversion. It integrates a C implementation of Fabian Loitsch's Grisu-algorithm [[pdf]](http://florian.loitsch.com/publications/dtoa-pldi2010.pdf), extracted from https://github.com/night-shift/fpconv. The relevant files are added in this PR, they are, as is all of https://github.com/night-shift/fpconv, available under a MIT License. As a result, I see a speedup of 900% on Apple Silicon M1 for a float set of benchmarks. floats don't have a single correct string representation: a float like `1000.0` can be represented as "1000", "1e3", "1000.0" (and more). The Grisu algorithm converts floating point numbers to an optimal decimal string representation without loss of precision. As a result, a float that is exactly an integer (like `Float(10)`) will be converted by that algorithm into `"10"`. While technically correct – the JSON format treats floats and integers identically –, this differs from the current behaviour of the `"json"` gem. To address this, the integration checks for that case, and explicitely adds a ".0" suffix in those cases. This is sufficient to meet all existing tests; there is, however, a chance that the current implementation and this implementation occasionally encode floats differently. ``` == Encoding floats (4179311 bytes) ruby 3.4.1 (2024-12-25 revision 48d4efcb85) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- json (local) 4.000 i/100ms Calculating ------------------------------------- json (local) 46.046 (± 2.2%) i/s (21.72 ms/i) - 232.000 in 5.039611s Normalize to 2090234 byte == Encoding floats (4179242 bytes) ruby 3.4.1 (2024-12-25 revision 48d4efcb85) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- json (2.10.2) 1.000 i/100ms Calculating ------------------------------------- json (2.10.2) 4.614 (± 0.0%) i/s (216.74 ms/i) - 24.000 in 5.201871s ``` These benchmarks are run via a script ([link](https://gist.github.com/radiospiel/04019402726a28b31616df3d0c17bd1c)) which is based on the gem's `benchmark/encoder.rb` file. There are probably better ways to run benchmarks :) My version allows to combine multiple test cases into a single one. The `dumps` benchmark, which covers the JSON files in `benchmark/data/*.json` – with the exception of `canada.json` – , reported a minor speedup within statistical uncertainty. --- ext/json/ext/generator/generator.c | 34 ++- ext/json/ext/vendor/fpconv/README.md | 3 + ext/json/ext/vendor/fpconv/license | 23 ++ ext/json/ext/vendor/fpconv/src/fpconv.c | 339 ++++++++++++++++++++++++ ext/json/ext/vendor/fpconv/src/fpconv.h | 33 +++ ext/json/ext/vendor/fpconv/src/powers.h | 87 ++++++ 6 files changed, 516 insertions(+), 3 deletions(-) create mode 100644 ext/json/ext/vendor/fpconv/README.md create mode 100644 ext/json/ext/vendor/fpconv/license create mode 100644 ext/json/ext/vendor/fpconv/src/fpconv.c create mode 100644 ext/json/ext/vendor/fpconv/src/fpconv.h create mode 100644 ext/json/ext/vendor/fpconv/src/powers.h diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 083388537..0f028fe07 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1050,12 +1050,15 @@ static void generate_json_integer(FBuffer *buffer, struct generate_json_data *da } #endif +#include "../vendor/fpconv/src/fpconv.c" + static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) { double value = RFLOAT_VALUE(obj); char allow_nan = state->allow_nan; - if (!allow_nan) { - if (isinf(value) || isnan(value)) { + if (isinf(value) || isnan(value)) { + /* for NaN and Infinity values we either raise an error or rely on Float#to_s. */ + if (!allow_nan) { if (state->strict && state->as_json) { VALUE casted_obj = rb_proc_call_with_block(state->as_json, 1, &obj, Qnil); if (casted_obj != obj) { @@ -1067,8 +1070,33 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data } raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", rb_funcall(obj, i_to_s, 0)); } + + VALUE tmp = rb_funcall(obj, i_to_s, 0); + fbuffer_append_str(buffer, tmp); + return; + } + + /* This implementation writes directly into the buffer. We reserve + * the 24 characters that fpconv_dtoa states as its maximum, plus + * 2 more characters for the potential ".0" suffix. + */ + fbuffer_inc_capa(buffer, 26); + char* d = buffer->ptr + buffer->len; + int len = fpconv_dtoa(value, d); + + /* fpconv_dtoa converts a float to its shorted string representation. When + * converting a float that is exactly an integer (e.g. `Float(2)`) this + * returns in a string that looks like an integer. This is correct, since + * JSON treats ints and floats the same. However, to not break integrations + * that expect a string representation looking like a float, we append a + * "." in that case. + */ + if(!memchr(d, '.', len) && !memchr(d, 'e', len)) { + d[len] = '.'; + d[len+1] = '0'; + len += 2; } - fbuffer_append_str(buffer, rb_funcall(obj, i_to_s, 0)); + buffer->len += len; } static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) diff --git a/ext/json/ext/vendor/fpconv/README.md b/ext/json/ext/vendor/fpconv/README.md new file mode 100644 index 000000000..0d40424d2 --- /dev/null +++ b/ext/json/ext/vendor/fpconv/README.md @@ -0,0 +1,3 @@ +The contents of this directory is extracted from https://github.com/night-shift/fpconv + +It is licensed under the provisions of the Boost Software License - Version 1.0 - August 17th, 2003. See the ./license file for details. \ No newline at end of file diff --git a/ext/json/ext/vendor/fpconv/license b/ext/json/ext/vendor/fpconv/license new file mode 100644 index 000000000..36b7cd93c --- /dev/null +++ b/ext/json/ext/vendor/fpconv/license @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/ext/json/ext/vendor/fpconv/src/fpconv.c b/ext/json/ext/vendor/fpconv/src/fpconv.c new file mode 100644 index 000000000..56ebb8944 --- /dev/null +++ b/ext/json/ext/vendor/fpconv/src/fpconv.c @@ -0,0 +1,339 @@ +#include +#include + +#include "fpconv.h" +#include "powers.h" + +#define fracmask 0x000FFFFFFFFFFFFFU +#define expmask 0x7FF0000000000000U +#define hiddenbit 0x0010000000000000U +#define signmask 0x8000000000000000U +#define expbias (1023 + 52) + +#define absv(n) ((n) < 0 ? -(n) : (n)) +#define minv(a, b) ((a) < (b) ? (a) : (b)) + +static uint64_t tens[] = { + 10000000000000000000U, 1000000000000000000U, 100000000000000000U, + 10000000000000000U, 1000000000000000U, 100000000000000U, + 10000000000000U, 1000000000000U, 100000000000U, + 10000000000U, 1000000000U, 100000000U, + 10000000U, 1000000U, 100000U, + 10000U, 1000U, 100U, + 10U, 1U +}; + +static inline uint64_t get_dbits(double d) +{ + union { + double dbl; + uint64_t i; + } dbl_bits = { d }; + + return dbl_bits.i; +} + +static Fp build_fp(double d) +{ + uint64_t bits = get_dbits(d); + + Fp fp; + fp.frac = bits & fracmask; + fp.exp = (bits & expmask) >> 52; + + if(fp.exp) { + fp.frac += hiddenbit; + fp.exp -= expbias; + + } else { + fp.exp = -expbias + 1; + } + + return fp; +} + +static void normalize(Fp* fp) +{ + while ((fp->frac & hiddenbit) == 0) { + fp->frac <<= 1; + fp->exp--; + } + + int shift = 64 - 52 - 1; + fp->frac <<= shift; + fp->exp -= shift; +} + +static void get_normalized_boundaries(Fp* fp, Fp* lower, Fp* upper) +{ + upper->frac = (fp->frac << 1) + 1; + upper->exp = fp->exp - 1; + + while ((upper->frac & (hiddenbit << 1)) == 0) { + upper->frac <<= 1; + upper->exp--; + } + + int u_shift = 64 - 52 - 2; + + upper->frac <<= u_shift; + upper->exp = upper->exp - u_shift; + + + int l_shift = fp->frac == hiddenbit ? 2 : 1; + + lower->frac = (fp->frac << l_shift) - 1; + lower->exp = fp->exp - l_shift; + + + lower->frac <<= lower->exp - upper->exp; + lower->exp = upper->exp; +} + +static Fp multiply(Fp* a, Fp* b) +{ + const uint64_t lomask = 0x00000000FFFFFFFF; + + uint64_t ah_bl = (a->frac >> 32) * (b->frac & lomask); + uint64_t al_bh = (a->frac & lomask) * (b->frac >> 32); + uint64_t al_bl = (a->frac & lomask) * (b->frac & lomask); + uint64_t ah_bh = (a->frac >> 32) * (b->frac >> 32); + + uint64_t tmp = (ah_bl & lomask) + (al_bh & lomask) + (al_bl >> 32); + /* round up */ + tmp += 1U << 31; + + Fp fp = { + ah_bh + (ah_bl >> 32) + (al_bh >> 32) + (tmp >> 32), + a->exp + b->exp + 64 + }; + + return fp; +} + +static void round_digit(char* digits, int ndigits, uint64_t delta, uint64_t rem, uint64_t kappa, uint64_t frac) +{ + while (rem < frac && delta - rem >= kappa && + (rem + kappa < frac || frac - rem > rem + kappa - frac)) { + + digits[ndigits - 1]--; + rem += kappa; + } +} + +static int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) +{ + uint64_t wfrac = upper->frac - fp->frac; + uint64_t delta = upper->frac - lower->frac; + + Fp one; + one.frac = 1ULL << -upper->exp; + one.exp = upper->exp; + + uint64_t part1 = upper->frac >> -one.exp; + uint64_t part2 = upper->frac & (one.frac - 1); + + int idx = 0, kappa = 10; + uint64_t* divp; + /* 1000000000 */ + for(divp = tens + 10; kappa > 0; divp++) { + + uint64_t div = *divp; + unsigned digit = (unsigned) (part1 / div); + + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part1 -= digit * div; + kappa--; + + uint64_t tmp = (part1 <<-one.exp) + part2; + if (tmp <= delta) { + *K += kappa; + round_digit(digits, idx, delta, tmp, div << -one.exp, wfrac); + + return idx; + } + } + + /* 10 */ + uint64_t* unit = tens + 18; + + while(true) { + part2 *= 10; + delta *= 10; + kappa--; + + unsigned digit = (unsigned) (part2 >> -one.exp); + if (digit || idx) { + digits[idx++] = digit + '0'; + } + + part2 &= one.frac - 1; + if (part2 < delta) { + *K += kappa; + round_digit(digits, idx, delta, part2, one.frac, wfrac * *unit); + + return idx; + } + + unit--; + } +} + +static int grisu2(double d, char* digits, int* K) +{ + Fp w = build_fp(d); + + Fp lower, upper; + get_normalized_boundaries(&w, &lower, &upper); + + normalize(&w); + + int k; + Fp cp = find_cachedpow10(upper.exp, &k); + + w = multiply(&w, &cp); + upper = multiply(&upper, &cp); + lower = multiply(&lower, &cp); + + lower.frac++; + upper.frac--; + + *K = -k; + + return generate_digits(&w, &upper, &lower, digits, K); +} + +static int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) +{ + int exp = absv(K + ndigits - 1); + + int max_trailing_zeros = 7; + + if(neg) { + max_trailing_zeros -= 1; + } + + /* write plain integer */ + if(K >= 0 && (exp < (ndigits + max_trailing_zeros))) { + + memcpy(dest, digits, ndigits); + memset(dest + ndigits, '0', K); + + return ndigits + K; + } + + /* write decimal w/o scientific notation */ + if(K < 0 && (K > -7 || exp < 4)) { + int offset = ndigits - absv(K); + /* fp < 1.0 -> write leading zero */ + if(offset <= 0) { + offset = -offset; + dest[0] = '0'; + dest[1] = '.'; + memset(dest + 2, '0', offset); + memcpy(dest + offset + 2, digits, ndigits); + + return ndigits + 2 + offset; + + /* fp > 1.0 */ + } else { + memcpy(dest, digits, offset); + dest[offset] = '.'; + memcpy(dest + offset + 1, digits + offset, ndigits - offset); + + return ndigits + 1; + } + } + + /* write decimal w/ scientific notation */ + ndigits = minv(ndigits, 18 - neg); + + int idx = 0; + dest[idx++] = digits[0]; + + if(ndigits > 1) { + dest[idx++] = '.'; + memcpy(dest + idx, digits + 1, ndigits - 1); + idx += ndigits - 1; + } + + dest[idx++] = 'e'; + + char sign = K + ndigits - 1 < 0 ? '-' : '+'; + dest[idx++] = sign; + + int cent = 0; + + if(exp > 99) { + cent = exp / 100; + dest[idx++] = cent + '0'; + exp -= cent * 100; + } + if(exp > 9) { + int dec = exp / 10; + dest[idx++] = dec + '0'; + exp -= dec * 10; + + } else if(cent) { + dest[idx++] = '0'; + } + + dest[idx++] = exp % 10 + '0'; + + return idx; +} + +static int filter_special(double fp, char* dest) +{ + if(fp == 0.0) { + dest[0] = '0'; + return 1; + } + + uint64_t bits = get_dbits(fp); + + bool nan = (bits & expmask) == expmask; + + if(!nan) { + return 0; + } + + if(bits & fracmask) { + dest[0] = 'n'; dest[1] = 'a'; dest[2] = 'n'; + + } else { + dest[0] = 'i'; dest[1] = 'n'; dest[2] = 'f'; + } + + return 3; +} + +int fpconv_dtoa(double d, char dest[24]) +{ + char digits[18]; + + int str_len = 0; + bool neg = false; + + if(get_dbits(d) & signmask) { + dest[0] = '-'; + str_len++; + neg = true; + } + + int spec = filter_special(d, dest + str_len); + + if(spec) { + return str_len + spec; + } + + int K = 0; + int ndigits = grisu2(d, digits, &K); + + str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); + + return str_len; +} diff --git a/ext/json/ext/vendor/fpconv/src/fpconv.h b/ext/json/ext/vendor/fpconv/src/fpconv.h new file mode 100644 index 000000000..58bbcccbb --- /dev/null +++ b/ext/json/ext/vendor/fpconv/src/fpconv.h @@ -0,0 +1,33 @@ +#ifndef FPCONV_H +#define FPCONV_H + +/* Fast and accurate double to string conversion based on Florian Loitsch's + * Grisu-algorithm[1]. + * + * Input: + * fp -> the double to convert, dest -> destination buffer. + * The generated string will never be longer than 24 characters. + * Make sure to pass a pointer to at least 24 bytes of memory. + * The emitted string will not be null terminated. + * + * Output: + * The number of written characters. + * + * Exemplary usage: + * + * void print(double d) + * { + * char buf[24 + 1] // plus null terminator + * int str_len = fpconv_dtoa(d, buf); + * + * buf[str_len] = '\0'; + * printf("%s", buf); + * } + * + */ + +int fpconv_dtoa(double fp, char dest[24]); + +#endif + +/* [1] http://florian.loitsch.com/publications/dtoa-pldi2010.pdf */ diff --git a/ext/json/ext/vendor/fpconv/src/powers.h b/ext/json/ext/vendor/fpconv/src/powers.h new file mode 100644 index 000000000..c707eedb0 --- /dev/null +++ b/ext/json/ext/vendor/fpconv/src/powers.h @@ -0,0 +1,87 @@ +#include + +#define npowers 87 +#define steppowers 8 +#define firstpower -348 /* 10 ^ -348 */ + +#define expmax -32 +#define expmin -60 + + +typedef struct Fp { + uint64_t frac; + int exp; +} Fp; + +static Fp powers_ten[] = { + { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, + { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 }, + { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, + { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, + { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 }, + { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, + { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, + { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 }, + { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, + { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, + { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 }, + { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, + { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, + { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 }, + { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, + { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, + { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 }, + { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, + { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, + { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 }, + { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, + { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, + { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 }, + { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, + { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, + { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 }, + { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, + { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, + { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 }, + { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, + { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, + { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 }, + { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, + { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, + { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 }, + { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, + { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, + { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 }, + { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, + { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, + { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 }, + { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, + { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, + { 12648080533535911531U, 1066 } +}; + +static Fp find_cachedpow10(int exp, int* k) +{ + const double one_log_ten = 0.30102999566398114; + + int approx = -(exp + npowers) * one_log_ten; + int idx = (approx - firstpower) / steppowers; + + while(1) { + int current = exp + powers_ten[idx].exp + 64; + + if(current < expmin) { + idx++; + continue; + } + + if(current > expmax) { + idx--; + continue; + } + + *k = (firstpower + idx * steppowers); + + return powers_ten[idx]; + } +} From fa5bdf87cb52d1cf06379911b9f374d8aa0cf1c7 Mon Sep 17 00:00:00 2001 From: eno Date: Wed, 19 Mar 2025 22:48:08 +0100 Subject: [PATCH 008/506] Adjust fpconv to add ".0" to integers Adds a test case fix --- ext/json/ext/generator/generator.c | 13 ++----------- ext/json/ext/vendor/fpconv/src/fpconv.c | 10 ++++++++-- test/json/json_generator_test.rb | 9 +++++++++ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 0f028fe07..6ef16ced8 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1084,18 +1084,9 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data char* d = buffer->ptr + buffer->len; int len = fpconv_dtoa(value, d); - /* fpconv_dtoa converts a float to its shorted string representation. When - * converting a float that is exactly an integer (e.g. `Float(2)`) this - * returns in a string that looks like an integer. This is correct, since - * JSON treats ints and floats the same. However, to not break integrations - * that expect a string representation looking like a float, we append a - * "." in that case. + /* fpconv_dtoa converts a float to its shortest string representation, + * but it adds a ".0" if this is a plain integer. */ - if(!memchr(d, '.', len) && !memchr(d, 'e', len)) { - d[len] = '.'; - d[len+1] = '0'; - len += 2; - } buffer->len += len; } diff --git a/ext/json/ext/vendor/fpconv/src/fpconv.c b/ext/json/ext/vendor/fpconv/src/fpconv.c index 56ebb8944..8ad80d9c6 100644 --- a/ext/json/ext/vendor/fpconv/src/fpconv.c +++ b/ext/json/ext/vendor/fpconv/src/fpconv.c @@ -222,7 +222,11 @@ static int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) memcpy(dest, digits, ndigits); memset(dest + ndigits, '0', K); - return ndigits + K; + /* add a .0 to mark this as a float. */ + dest[ndigits + K] = '.'; + dest[ndigits + K + 1] = '0'; + + return ndigits + K + 2; } /* write decimal w/o scientific notation */ @@ -290,7 +294,9 @@ static int filter_special(double fp, char* dest) { if(fp == 0.0) { dest[0] = '0'; - return 1; + dest[1] = '.'; + dest[2] = '0'; + return 3; } uint64_t bits = get_dbits(fp); diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 4a92801fe..bf4796a6b 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -698,4 +698,13 @@ def test_json_generate_as_json_convert_to_proc object = Object.new assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: :object_id) end + + def test_json_generate_float + values = [-1.0, 1.0, 0.0, 12.2, 7.5 / 3.2, 12.0, 100.0, 1000.0] + expecteds = ["-1.0", "1.0", "0.0", "12.2", "2.34375", "12.0", "100.0", "1000.0"] + + values.zip(expecteds).each do |value, expected| + assert_equal expected, value.to_json + end + end end From cce3d1f6c1d9c8b742129bc4ce449e0e8de0259a Mon Sep 17 00:00:00 2001 From: eno Date: Thu, 20 Mar 2025 10:05:13 +0100 Subject: [PATCH 009/506] Extends license description --- ext/json/ext/vendor/fpconv/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fpconv/README.md b/ext/json/ext/vendor/fpconv/README.md index 0d40424d2..3c555f178 100644 --- a/ext/json/ext/vendor/fpconv/README.md +++ b/ext/json/ext/vendor/fpconv/README.md @@ -1,3 +1,5 @@ The contents of this directory is extracted from https://github.com/night-shift/fpconv -It is licensed under the provisions of the Boost Software License - Version 1.0 - August 17th, 2003. See the ./license file for details. \ No newline at end of file +It is licensed under the provisions of the Boost Software License - Version 1.0 - August 17th, 2003. See the ./license file for details. + +It was slightly modified to append ".0" to plain floats, for use with the https://github.com/ruby/json package. \ No newline at end of file From dc4773de88c77a0b148bdeb4d56b33d649e08744 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 20 Mar 2025 11:05:16 +0100 Subject: [PATCH 010/506] Reorganize `fpconv` vendoring Make it a single file and declare the dependency. --- ext/json/ext/generator/depend | 1 + ext/json/ext/generator/generator.c | 3 +- ext/json/ext/vendor/{fpconv/src => }/fpconv.c | 140 +++++++++++++++++- ext/json/ext/vendor/fpconv/README.md | 5 - ext/json/ext/vendor/fpconv/license | 23 --- ext/json/ext/vendor/fpconv/src/fpconv.h | 33 ----- ext/json/ext/vendor/fpconv/src/powers.h | 87 ----------- 7 files changed, 139 insertions(+), 153 deletions(-) rename ext/json/ext/vendor/{fpconv/src => }/fpconv.c (54%) delete mode 100644 ext/json/ext/vendor/fpconv/README.md delete mode 100644 ext/json/ext/vendor/fpconv/license delete mode 100644 ext/json/ext/vendor/fpconv/src/fpconv.h delete mode 100644 ext/json/ext/vendor/fpconv/src/powers.h diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 967aa7659..2598308cc 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1 +1,2 @@ generator.o: generator.c $(srcdir)/../fbuffer/fbuffer.h +generator.o: generator.c $(srcdir)/../vendor/fpconv.c diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 6ef16ced8..2207dc8ad 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1,5 +1,6 @@ #include "ruby.h" #include "../fbuffer/fbuffer.h" +#include "../vendor/fpconv.c" #include #include @@ -1050,8 +1051,6 @@ static void generate_json_integer(FBuffer *buffer, struct generate_json_data *da } #endif -#include "../vendor/fpconv/src/fpconv.c" - static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) { double value = RFLOAT_VALUE(obj); diff --git a/ext/json/ext/vendor/fpconv/src/fpconv.c b/ext/json/ext/vendor/fpconv.c similarity index 54% rename from ext/json/ext/vendor/fpconv/src/fpconv.c rename to ext/json/ext/vendor/fpconv.c index 8ad80d9c6..2887c6488 100644 --- a/ext/json/ext/vendor/fpconv/src/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -1,8 +1,118 @@ +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +// The contents of this file is extracted from https://github.com/night-shift/fpconv +// It was slightly modified to append ".0" to plain floats, for use with the https://github.com/ruby/json package. + #include #include +#include + +#define npowers 87 +#define steppowers 8 +#define firstpower -348 /* 10 ^ -348 */ + +#define expmax -32 +#define expmin -60 + +typedef struct Fp { + uint64_t frac; + int exp; +} Fp; + +static Fp powers_ten[] = { + { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, + { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 }, + { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, + { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, + { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 }, + { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, + { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, + { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 }, + { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, + { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, + { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 }, + { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, + { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, + { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 }, + { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, + { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, + { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 }, + { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, + { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, + { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 }, + { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, + { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, + { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 }, + { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, + { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, + { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 }, + { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, + { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, + { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 }, + { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, + { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, + { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 }, + { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, + { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, + { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 }, + { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, + { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, + { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 }, + { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, + { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, + { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 }, + { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, + { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, + { 12648080533535911531U, 1066 } +}; + +static Fp find_cachedpow10(int exp, int* k) +{ + const double one_log_ten = 0.30102999566398114; -#include "fpconv.h" -#include "powers.h" + int approx = -(exp + npowers) * one_log_ten; + int idx = (approx - firstpower) / steppowers; + + while(1) { + int current = exp + powers_ten[idx].exp + 64; + + if(current < expmin) { + idx++; + continue; + } + + if(current > expmax) { + idx--; + continue; + } + + *k = (firstpower + idx * steppowers); + + return powers_ten[idx]; + } +} #define fracmask 0x000FFFFFFFFFFFFFU #define expmask 0x7FF0000000000000U @@ -317,7 +427,31 @@ static int filter_special(double fp, char* dest) return 3; } -int fpconv_dtoa(double d, char dest[24]) +/* Fast and accurate double to string conversion based on Florian Loitsch's + * Grisu-algorithm[1]. + * + * Input: + * fp -> the double to convert, dest -> destination buffer. + * The generated string will never be longer than 24 characters. + * Make sure to pass a pointer to at least 24 bytes of memory. + * The emitted string will not be null terminated. + * + * Output: + * The number of written characters. + * + * Exemplary usage: + * + * void print(double d) + * { + * char buf[24 + 1] // plus null terminator + * int str_len = fpconv_dtoa(d, buf); + * + * buf[str_len] = '\0'; + * printf("%s", buf); + * } + * + */ +static int fpconv_dtoa(double d, char dest[24]) { char digits[18]; diff --git a/ext/json/ext/vendor/fpconv/README.md b/ext/json/ext/vendor/fpconv/README.md deleted file mode 100644 index 3c555f178..000000000 --- a/ext/json/ext/vendor/fpconv/README.md +++ /dev/null @@ -1,5 +0,0 @@ -The contents of this directory is extracted from https://github.com/night-shift/fpconv - -It is licensed under the provisions of the Boost Software License - Version 1.0 - August 17th, 2003. See the ./license file for details. - -It was slightly modified to append ".0" to plain floats, for use with the https://github.com/ruby/json package. \ No newline at end of file diff --git a/ext/json/ext/vendor/fpconv/license b/ext/json/ext/vendor/fpconv/license deleted file mode 100644 index 36b7cd93c..000000000 --- a/ext/json/ext/vendor/fpconv/license +++ /dev/null @@ -1,23 +0,0 @@ -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/ext/json/ext/vendor/fpconv/src/fpconv.h b/ext/json/ext/vendor/fpconv/src/fpconv.h deleted file mode 100644 index 58bbcccbb..000000000 --- a/ext/json/ext/vendor/fpconv/src/fpconv.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef FPCONV_H -#define FPCONV_H - -/* Fast and accurate double to string conversion based on Florian Loitsch's - * Grisu-algorithm[1]. - * - * Input: - * fp -> the double to convert, dest -> destination buffer. - * The generated string will never be longer than 24 characters. - * Make sure to pass a pointer to at least 24 bytes of memory. - * The emitted string will not be null terminated. - * - * Output: - * The number of written characters. - * - * Exemplary usage: - * - * void print(double d) - * { - * char buf[24 + 1] // plus null terminator - * int str_len = fpconv_dtoa(d, buf); - * - * buf[str_len] = '\0'; - * printf("%s", buf); - * } - * - */ - -int fpconv_dtoa(double fp, char dest[24]); - -#endif - -/* [1] http://florian.loitsch.com/publications/dtoa-pldi2010.pdf */ diff --git a/ext/json/ext/vendor/fpconv/src/powers.h b/ext/json/ext/vendor/fpconv/src/powers.h deleted file mode 100644 index c707eedb0..000000000 --- a/ext/json/ext/vendor/fpconv/src/powers.h +++ /dev/null @@ -1,87 +0,0 @@ -#include - -#define npowers 87 -#define steppowers 8 -#define firstpower -348 /* 10 ^ -348 */ - -#define expmax -32 -#define expmin -60 - - -typedef struct Fp { - uint64_t frac; - int exp; -} Fp; - -static Fp powers_ten[] = { - { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, - { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 }, - { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, - { 12353653155963782858U, -1060 }, { 18408377700990114895U, -1034 }, - { 13715310171984221708U, -1007 }, { 10218702384817765436U, -980 }, - { 15227053142812498563U, -954 }, { 11345038669416679861U, -927 }, - { 16905424996341287883U, -901 }, { 12595523146049147757U, -874 }, - { 9384396036005875287U, -847 }, { 13983839803942852151U, -821 }, - { 10418772551374772303U, -794 }, { 15525180923007089351U, -768 }, - { 11567161174868858868U, -741 }, { 17236413322193710309U, -715 }, - { 12842128665889583758U, -688 }, { 9568131466127621947U, -661 }, - { 14257626930069360058U, -635 }, { 10622759856335341974U, -608 }, - { 15829145694278690180U, -582 }, { 11793632577567316726U, -555 }, - { 17573882009934360870U, -529 }, { 13093562431584567480U, -502 }, - { 9755464219737475723U, -475 }, { 14536774485912137811U, -449 }, - { 10830740992659433045U, -422 }, { 16139061738043178685U, -396 }, - { 12024538023802026127U, -369 }, { 17917957937422433684U, -343 }, - { 13349918974505688015U, -316 }, { 9946464728195732843U, -289 }, - { 14821387422376473014U, -263 }, { 11042794154864902060U, -236 }, - { 16455045573212060422U, -210 }, { 12259964326927110867U, -183 }, - { 18268770466636286478U, -157 }, { 13611294676837538539U, -130 }, - { 10141204801825835212U, -103 }, { 15111572745182864684U, -77 }, - { 11258999068426240000U, -50 }, { 16777216000000000000U, -24 }, - { 12500000000000000000U, 3 }, { 9313225746154785156U, 30 }, - { 13877787807814456755U, 56 }, { 10339757656912845936U, 83 }, - { 15407439555097886824U, 109 }, { 11479437019748901445U, 136 }, - { 17105694144590052135U, 162 }, { 12744735289059618216U, 189 }, - { 9495567745759798747U, 216 }, { 14149498560666738074U, 242 }, - { 10542197943230523224U, 269 }, { 15709099088952724970U, 295 }, - { 11704190886730495818U, 322 }, { 17440603504673385349U, 348 }, - { 12994262207056124023U, 375 }, { 9681479787123295682U, 402 }, - { 14426529090290212157U, 428 }, { 10748601772107342003U, 455 }, - { 16016664761464807395U, 481 }, { 11933345169920330789U, 508 }, - { 17782069995880619868U, 534 }, { 13248674568444952270U, 561 }, - { 9871031767461413346U, 588 }, { 14708983551653345445U, 614 }, - { 10959046745042015199U, 641 }, { 16330252207878254650U, 667 }, - { 12166986024289022870U, 694 }, { 18130221999122236476U, 720 }, - { 13508068024458167312U, 747 }, { 10064294952495520794U, 774 }, - { 14996968138956309548U, 800 }, { 11173611982879273257U, 827 }, - { 16649979327439178909U, 853 }, { 12405201291620119593U, 880 }, - { 9242595204427927429U, 907 }, { 13772540099066387757U, 933 }, - { 10261342003245940623U, 960 }, { 15290591125556738113U, 986 }, - { 11392378155556871081U, 1013 }, { 16975966327722178521U, 1039 }, - { 12648080533535911531U, 1066 } -}; - -static Fp find_cachedpow10(int exp, int* k) -{ - const double one_log_ten = 0.30102999566398114; - - int approx = -(exp + npowers) * one_log_ten; - int idx = (approx - firstpower) / steppowers; - - while(1) { - int current = exp + powers_ten[idx].exp + 64; - - if(current < expmin) { - idx++; - continue; - } - - if(current > expmax) { - idx--; - continue; - } - - *k = (firstpower + idx * steppowers); - - return powers_ten[idx]; - } -} From 930cb23f1045193de3b0eeb33f69f026c5ba4abb Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 24 Mar 2025 15:20:57 +0900 Subject: [PATCH 011/506] Removed trailing space --- ext/json/ext/generator/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 2207dc8ad..428f5e21f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1077,7 +1077,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data /* This implementation writes directly into the buffer. We reserve * the 24 characters that fpconv_dtoa states as its maximum, plus - * 2 more characters for the potential ".0" suffix. + * 2 more characters for the potential ".0" suffix. */ fbuffer_inc_capa(buffer, 26); char* d = buffer->ptr + buffer->len; From 0e60e05d994b3e909525891f2833a736278e7a01 Mon Sep 17 00:00:00 2001 From: eno Date: Sun, 16 Mar 2025 19:51:59 +0100 Subject: [PATCH 012/506] Faster integer formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit provides an alternative implementation for a long → decimal conversion. The main difference is that it uses an algorithm pulled from https://github.com/jeaiii/itoa. The source there is C++, it was converted by hand to C for inclusion with this gem. jeaiii's algorithm is covered by the MIT License, see source code. On addition this version now also generates the string directly into the fbuffer, foregoing the need to run a separate memory copy. As a result, I see a speedup of 32% on Apple Silicon M1 for an integer set of benchmarks. --- ext/json/ext/fbuffer/fbuffer.h | 56 +++++-- ext/json/ext/generator/depend | 1 + ext/json/ext/generator/generator.c | 1 + ext/json/ext/vendor/jeaiii-ltoa.h | 257 +++++++++++++++++++++++++++++ test/json/json_generator_test.rb | 12 ++ 5 files changed, 310 insertions(+), 17 deletions(-) create mode 100644 ext/json/ext/vendor/jeaiii-ltoa.h diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 4c42e14b6..b56955e8f 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -3,6 +3,7 @@ #include "ruby.h" #include "ruby/encoding.h" +#include "../vendor/jeaiii-ltoa.h" /* shims */ /* This is the fallback definition from Ruby 3.4 */ @@ -150,6 +151,13 @@ static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len) } } +/* Appends a character into a buffer. The buffer needs to have sufficient capacity, via fbuffer_inc_capa(...). */ +static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) +{ + fb->ptr[fb->len] = chr; + fb->len += 1; +} + static void fbuffer_append_str(FBuffer *fb, VALUE str) { const char *newstr = StringValuePtr(str); @@ -167,25 +175,39 @@ static inline void fbuffer_append_char(FBuffer *fb, char newchr) fb->len++; } -static long fltoa(long number, char *buf) -{ - static const char digits[] = "0123456789"; - long sign = number; - char* tmp = buf; - - if (sign < 0) number = -number; - do *tmp-- = digits[number % 10]; while (number /= 10); - if (sign < 0) *tmp-- = '-'; - return buf - tmp; -} - -#define LONG_BUFFER_SIZE 20 +/* + * Appends the decimal string representation of \a number into the buffer. + */ static void fbuffer_append_long(FBuffer *fb, long number) { - char buf[LONG_BUFFER_SIZE]; - char *buffer_end = buf + LONG_BUFFER_SIZE; - long len = fltoa(number, buffer_end - 1); - fbuffer_append(fb, buffer_end - len, len); + /* + * The to_text_from_ulong() function produces digits left-to-right, + * allowing us to write directly into the buffer, but we don't know + * the number of resulting characters. + * + * We do know, however, that the `number` argument is always in the + * range 0xc000000000000000 to 0x3fffffffffffffff, or, in decimal, + * -4611686018427387904 to 4611686018427387903. The max number of chars + * generated is therefore 20 (including a potential sign character). + */ + + static const int MAX_CHARS_FOR_LONG = 20; + + fbuffer_inc_capa(fb, MAX_CHARS_FOR_LONG); + + if (number < 0) { + fbuffer_append_reserved_char(fb, '-'); + + /* + * Since number is always > LONG_MIN, `-number` will not overflow + * and is always the positive abs() value. + */ + number = -number; + } + + char* d = fb->ptr + fb->len; + char* end = to_text_from_ulong(d, number); + fb->len += end - d; } static VALUE fbuffer_finalize(FBuffer *fb) diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 2598308cc..60697f2c4 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1,2 +1,3 @@ generator.o: generator.c $(srcdir)/../fbuffer/fbuffer.h generator.o: generator.c $(srcdir)/../vendor/fpconv.c +generator.o: generator.c $(srcdir)/../vendor/jeaiii-ltoa.h diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 428f5e21f..e0d7d0ca4 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1710,6 +1710,7 @@ void Init_generator(void) cFragment = rb_const_get(mJSON, rb_intern("Fragment")); VALUE mExt = rb_define_module_under(mJSON, "Ext"); + VALUE mGenerator = rb_define_module_under(mExt, "Generator"); rb_global_variable(&eGeneratorError); diff --git a/ext/json/ext/vendor/jeaiii-ltoa.h b/ext/json/ext/vendor/jeaiii-ltoa.h new file mode 100644 index 000000000..f83e036e7 --- /dev/null +++ b/ext/json/ext/vendor/jeaiii-ltoa.h @@ -0,0 +1,257 @@ +/* + +This file is released under the terms of the MIT License. It is based on the +work of James Edward Anhalt III, with the original license listed below. + +MIT License + +Copyright (c) 2024,2025 Enrico Thierbach - https://github.com/radiospiel +Copyright (c) 2022 James Edward Anhalt III - https://github.com/jeaiii/itoa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef JEAIII_TO_TEXT_H_ +#define JEAIII_TO_TEXT_H_ + +#include + +typedef uint_fast32_t u32_t; +typedef uint_fast64_t u64_t; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshorten-64-to-32" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmissing-braces" + +#define u32(x) ((u32_t)(x)) +#define u64(x) ((u64_t)(x)) + +struct pair +{ + char dd[2]; +}; + +#define cast_to_pair_ptr(b) ((struct pair*)(void*)(b)) + +static struct pair digits_dd[100] = +{ + { '0', '0' }, { '0', '1' }, { '0', '2' }, { '0', '3' }, { '0', '4' }, { '0', '5' }, { '0', '6' }, { '0', '7' }, { '0', '8' }, { '0', '9' }, + { '1', '0' }, { '1', '1' }, { '1', '2' }, { '1', '3' }, { '1', '4' }, { '1', '5' }, { '1', '6' }, { '1', '7' }, { '1', '8' }, { '1', '9' }, + { '2', '0' }, { '2', '1' }, { '2', '2' }, { '2', '3' }, { '2', '4' }, { '2', '5' }, { '2', '6' }, { '2', '7' }, { '2', '8' }, { '2', '9' }, + { '3', '0' }, { '3', '1' }, { '3', '2' }, { '3', '3' }, { '3', '4' }, { '3', '5' }, { '3', '6' }, { '3', '7' }, { '3', '8' }, { '3', '9' }, + { '4', '0' }, { '4', '1' }, { '4', '2' }, { '4', '3' }, { '4', '4' }, { '4', '5' }, { '4', '6' }, { '4', '7' }, { '4', '8' }, { '4', '9' }, + { '5', '0' }, { '5', '1' }, { '5', '2' }, { '5', '3' }, { '5', '4' }, { '5', '5' }, { '5', '6' }, { '5', '7' }, { '5', '8' }, { '5', '9' }, + { '6', '0' }, { '6', '1' }, { '6', '2' }, { '6', '3' }, { '6', '4' }, { '6', '5' }, { '6', '6' }, { '6', '7' }, { '6', '8' }, { '6', '9' }, + { '7', '0' }, { '7', '1' }, { '7', '2' }, { '7', '3' }, { '7', '4' }, { '7', '5' }, { '7', '6' }, { '7', '7' }, { '7', '8' }, { '7', '9' }, + { '8', '0' }, { '8', '1' }, { '8', '2' }, { '8', '3' }, { '8', '4' }, { '8', '5' }, { '8', '6' }, { '8', '7' }, { '8', '8' }, { '8', '9' }, + { '9', '0' }, { '9', '1' }, { '9', '2' }, { '9', '3' }, { '9', '4' }, { '9', '5' }, { '9', '6' }, { '9', '7' }, { '9', '8' }, { '9', '9' }, +}; + +#define NUL 'x' + +static struct pair digits_fd[100] = +{ + { '0', NUL }, { '1', NUL }, { '2', NUL }, { '3', NUL }, { '4', NUL }, { '5', NUL }, { '6', NUL }, { '7', NUL }, { '8', NUL }, { '9', NUL }, + { '1', '0' }, { '1', '1' }, { '1', '2' }, { '1', '3' }, { '1', '4' }, { '1', '5' }, { '1', '6' }, { '1', '7' }, { '1', '8' }, { '1', '9' }, + { '2', '0' }, { '2', '1' }, { '2', '2' }, { '2', '3' }, { '2', '4' }, { '2', '5' }, { '2', '6' }, { '2', '7' }, { '2', '8' }, { '2', '9' }, + { '3', '0' }, { '3', '1' }, { '3', '2' }, { '3', '3' }, { '3', '4' }, { '3', '5' }, { '3', '6' }, { '3', '7' }, { '3', '8' }, { '3', '9' }, + { '4', '0' }, { '4', '1' }, { '4', '2' }, { '4', '3' }, { '4', '4' }, { '4', '5' }, { '4', '6' }, { '4', '7' }, { '4', '8' }, { '4', '9' }, + { '5', '0' }, { '5', '1' }, { '5', '2' }, { '5', '3' }, { '5', '4' }, { '5', '5' }, { '5', '6' }, { '5', '7' }, { '5', '8' }, { '5', '9' }, + { '6', '0' }, { '6', '1' }, { '6', '2' }, { '6', '3' }, { '6', '4' }, { '6', '5' }, { '6', '6' }, { '6', '7' }, { '6', '8' }, { '6', '9' }, + { '7', '0' }, { '7', '1' }, { '7', '2' }, { '7', '3' }, { '7', '4' }, { '7', '5' }, { '7', '6' }, { '7', '7' }, { '7', '8' }, { '7', '9' }, + { '8', '0' }, { '8', '1' }, { '8', '2' }, { '8', '3' }, { '8', '4' }, { '8', '5' }, { '8', '6' }, { '8', '7' }, { '8', '8' }, { '8', '9' }, + { '9', '0' }, { '9', '1' }, { '9', '2' }, { '9', '3' }, { '9', '4' }, { '9', '5' }, { '9', '6' }, { '9', '7' }, { '9', '8' }, { '9', '9' }, +}; + +#undef NUL + +static u64_t mask24 = (u64(1) << 24) - 1; +static u64_t mask32 = (u64(1) << 32) - 1; +static u64_t mask57 = (u64(1) << 57) - 1; + +static +char* to_text_from_ulong(char* b, u64_t n) { + if (n < u32(1e2)) + { + *cast_to_pair_ptr(b) = digits_fd[n]; + return n < 10 ? b + 1 : b + 2; + } + if (n < u32(1e6)) + { + if (n < u32(1e4)) + { + u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * n; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + b -= n < u32(1e3); + u32_t f2 = (f0 & mask24) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + return b + 4; + } + u64_t f0 = u64(10 * (1ull << 32ull)/ 1e5 + 1) * n; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + b -= n < u32(1e5); + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + return b + 6; + } + if (n < u64(1ull << 32ull)) + { + if (n < u32(1e8)) + { + u64_t f0 = u64(10 * (1ull << 48ull) / 1e7 + 1) * n >> 16; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + b -= n < u32(1e7); + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + u64_t f6 = (f4 & mask32) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + return b + 8; + } + u64_t f0 = u64(10 * (1ull << 57ull) / 1e9 + 1) * n; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 57]; + b -= n < u32(1e9); + u64_t f2 = (f0 & mask57) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 57]; + u64_t f4 = (f2 & mask57) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 57]; + u64_t f6 = (f4 & mask57) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 57]; + u64_t f8 = (f6 & mask57) * 100; + *cast_to_pair_ptr(b + 8) = digits_dd[f8 >> 57]; + return b + 10; + } + + // if we get here U must be u64 but some compilers don't know that, so reassign n to a u64 to avoid warnings + u32_t z = n % u32(1e8); + u64_t u = n / u32(1e8); + + if (u < u32(1e2)) + { + // u can't be 1 digit (if u < 10 it would have been handled above as a 9 digit 32bit number) + *cast_to_pair_ptr(b) = digits_dd[u]; + b += 2; + } + else if (u < u32(1e6)) + { + if (u < u32(1e4)) + { + u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + b -= u < u32(1e3); + u32_t f2 = (f0 & mask24) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + b += 4; + } + else + { + u64_t f0 = u64(10 * (1ull << 32ull) / 1e5 + 1) * u; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + b -= u < u32(1e5); + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + b += 6; + } + } + else if (u < u32(1e8)) + { + u64_t f0 = u64(10 * (1ull << 48ull) / 1e7 + 1) * u >> 16; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + b -= u < u32(1e7); + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + u64_t f6 = (f4 & mask32) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + b += 8; + } + else if (u < u64(1ull << 32ull)) + { + u64_t f0 = u64(10 * (1ull << 57ull) / 1e9 + 1) * u; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 57]; + b -= u < u32(1e9); + u64_t f2 = (f0 & mask57) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 57]; + u64_t f4 = (f2 & mask57) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 57]; + u64_t f6 = (f4 & mask57) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 57]; + u64_t f8 = (f6 & mask57) * 100; + *cast_to_pair_ptr(b + 8) = digits_dd[f8 >> 57]; + b += 10; + } + else + { + u32_t y = u % u32(1e8); + u /= u32(1e8); + + // u is 2, 3, or 4 digits (if u < 10 it would have been handled above) + if (u < u32(1e2)) + { + *cast_to_pair_ptr(b) = digits_dd[u]; + b += 2; + } + else + { + u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; + *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + b -= u < u32(1e3); + u32_t f2 = (f0 & mask24) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + b += 4; + } + // do 8 digits + u64_t f0 = (u64((1ull << 48ull) / 1e6 + 1) * y >> 16) + 1; + *cast_to_pair_ptr(b) = digits_dd[f0 >> 32]; + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + u64_t f6 = (f4 & mask32) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + b += 8; + } + // do 8 digits + u64_t f0 = (u64((1ull << 48ull) / 1e6 + 1) * z >> 16) + 1; + *cast_to_pair_ptr(b) = digits_dd[f0 >> 32]; + u64_t f2 = (f0 & mask32) * 100; + *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + u64_t f4 = (f2 & mask32) * 100; + *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + u64_t f6 = (f4 & mask32) * 100; + *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + return b + 8; +} + +#undef u32 +#undef u64 + +#pragma clang diagnostic pop +#pragma GCC diagnostic pop + +#endif // JEAIII_TO_TEXT_H_ + diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index bf4796a6b..9cca447e2 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -707,4 +707,16 @@ def test_json_generate_float assert_equal expected, value.to_json end end + + def test_numbers_of_various_sizes + numbers = [ + 0, 1, -1, 9, -9, 13, -13, 91, -91, 513, -513, 7513, -7513, + 17591, -17591, -4611686018427387904, 4611686018427387903, + 2**62, 2**63, 2**64, -(2**62), -(2**63), -(2**64) + ] + + numbers.each do |number| + assert_equal "[#{number}]", JSON.generate([number]) + end + end end From 1bcebefa308aa5057760eaf9e007e5546cb151ec Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 25 Mar 2025 10:59:05 +0100 Subject: [PATCH 013/506] Refactor jeaiii-ltoa.h Some relatively minor change to make the library more in line with the gem. Some renaming, etc. --- ext/json/ext/fbuffer/fbuffer.h | 17 ++- ext/json/ext/generator/generator.c | 1 - ext/json/ext/vendor/jeaiii-ltoa.h | 232 ++++++++++++++++------------- 3 files changed, 139 insertions(+), 111 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index b56955e8f..18e2e963f 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -175,13 +175,23 @@ static inline void fbuffer_append_char(FBuffer *fb, char newchr) fb->len++; } +static inline char *fbuffer_cursor(FBuffer *fb) +{ + return fb->ptr + fb->len; +} + +static inline void fbuffer_advance_to(FBuffer *fb, char *end) +{ + fb->len = end - fb->ptr; +} + /* * Appends the decimal string representation of \a number into the buffer. */ static void fbuffer_append_long(FBuffer *fb, long number) { /* - * The to_text_from_ulong() function produces digits left-to-right, + * The jeaiii_ultoa() function produces digits left-to-right, * allowing us to write directly into the buffer, but we don't know * the number of resulting characters. * @@ -205,9 +215,8 @@ static void fbuffer_append_long(FBuffer *fb, long number) number = -number; } - char* d = fb->ptr + fb->len; - char* end = to_text_from_ulong(d, number); - fb->len += end - d; + char *end = jeaiii_ultoa(fbuffer_cursor(fb), number); + fbuffer_advance_to(fb, end); } static VALUE fbuffer_finalize(FBuffer *fb) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index e0d7d0ca4..428f5e21f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1710,7 +1710,6 @@ void Init_generator(void) cFragment = rb_const_get(mJSON, rb_intern("Fragment")); VALUE mExt = rb_define_module_under(mJSON, "Ext"); - VALUE mGenerator = rb_define_module_under(mExt, "Generator"); rb_global_variable(&eGeneratorError); diff --git a/ext/json/ext/vendor/jeaiii-ltoa.h b/ext/json/ext/vendor/jeaiii-ltoa.h index f83e036e7..c8e2a1288 100644 --- a/ext/json/ext/vendor/jeaiii-ltoa.h +++ b/ext/json/ext/vendor/jeaiii-ltoa.h @@ -44,102 +44,109 @@ typedef uint_fast64_t u64_t; #define u32(x) ((u32_t)(x)) #define u64(x) ((u64_t)(x)) -struct pair +struct digit_pair { char dd[2]; }; -#define cast_to_pair_ptr(b) ((struct pair*)(void*)(b)) +static const struct digit_pair *digits_dd = (struct digit_pair *)( + "00" "01" "02" "03" "04" "05" "06" "07" "08" "09" + "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" + "20" "21" "22" "23" "24" "25" "26" "27" "28" "29" + "30" "31" "32" "33" "34" "35" "36" "37" "38" "39" + "40" "41" "42" "43" "44" "45" "46" "47" "48" "49" + "50" "51" "52" "53" "54" "55" "56" "57" "58" "59" + "60" "61" "62" "63" "64" "65" "66" "67" "68" "69" + "70" "71" "72" "73" "74" "75" "76" "77" "78" "79" + "80" "81" "82" "83" "84" "85" "86" "87" "88" "89" + "90" "91" "92" "93" "94" "95" "96" "97" "98" "99" +); -static struct pair digits_dd[100] = -{ - { '0', '0' }, { '0', '1' }, { '0', '2' }, { '0', '3' }, { '0', '4' }, { '0', '5' }, { '0', '6' }, { '0', '7' }, { '0', '8' }, { '0', '9' }, - { '1', '0' }, { '1', '1' }, { '1', '2' }, { '1', '3' }, { '1', '4' }, { '1', '5' }, { '1', '6' }, { '1', '7' }, { '1', '8' }, { '1', '9' }, - { '2', '0' }, { '2', '1' }, { '2', '2' }, { '2', '3' }, { '2', '4' }, { '2', '5' }, { '2', '6' }, { '2', '7' }, { '2', '8' }, { '2', '9' }, - { '3', '0' }, { '3', '1' }, { '3', '2' }, { '3', '3' }, { '3', '4' }, { '3', '5' }, { '3', '6' }, { '3', '7' }, { '3', '8' }, { '3', '9' }, - { '4', '0' }, { '4', '1' }, { '4', '2' }, { '4', '3' }, { '4', '4' }, { '4', '5' }, { '4', '6' }, { '4', '7' }, { '4', '8' }, { '4', '9' }, - { '5', '0' }, { '5', '1' }, { '5', '2' }, { '5', '3' }, { '5', '4' }, { '5', '5' }, { '5', '6' }, { '5', '7' }, { '5', '8' }, { '5', '9' }, - { '6', '0' }, { '6', '1' }, { '6', '2' }, { '6', '3' }, { '6', '4' }, { '6', '5' }, { '6', '6' }, { '6', '7' }, { '6', '8' }, { '6', '9' }, - { '7', '0' }, { '7', '1' }, { '7', '2' }, { '7', '3' }, { '7', '4' }, { '7', '5' }, { '7', '6' }, { '7', '7' }, { '7', '8' }, { '7', '9' }, - { '8', '0' }, { '8', '1' }, { '8', '2' }, { '8', '3' }, { '8', '4' }, { '8', '5' }, { '8', '6' }, { '8', '7' }, { '8', '8' }, { '8', '9' }, - { '9', '0' }, { '9', '1' }, { '9', '2' }, { '9', '3' }, { '9', '4' }, { '9', '5' }, { '9', '6' }, { '9', '7' }, { '9', '8' }, { '9', '9' }, -}; - -#define NUL 'x' - -static struct pair digits_fd[100] = -{ - { '0', NUL }, { '1', NUL }, { '2', NUL }, { '3', NUL }, { '4', NUL }, { '5', NUL }, { '6', NUL }, { '7', NUL }, { '8', NUL }, { '9', NUL }, - { '1', '0' }, { '1', '1' }, { '1', '2' }, { '1', '3' }, { '1', '4' }, { '1', '5' }, { '1', '6' }, { '1', '7' }, { '1', '8' }, { '1', '9' }, - { '2', '0' }, { '2', '1' }, { '2', '2' }, { '2', '3' }, { '2', '4' }, { '2', '5' }, { '2', '6' }, { '2', '7' }, { '2', '8' }, { '2', '9' }, - { '3', '0' }, { '3', '1' }, { '3', '2' }, { '3', '3' }, { '3', '4' }, { '3', '5' }, { '3', '6' }, { '3', '7' }, { '3', '8' }, { '3', '9' }, - { '4', '0' }, { '4', '1' }, { '4', '2' }, { '4', '3' }, { '4', '4' }, { '4', '5' }, { '4', '6' }, { '4', '7' }, { '4', '8' }, { '4', '9' }, - { '5', '0' }, { '5', '1' }, { '5', '2' }, { '5', '3' }, { '5', '4' }, { '5', '5' }, { '5', '6' }, { '5', '7' }, { '5', '8' }, { '5', '9' }, - { '6', '0' }, { '6', '1' }, { '6', '2' }, { '6', '3' }, { '6', '4' }, { '6', '5' }, { '6', '6' }, { '6', '7' }, { '6', '8' }, { '6', '9' }, - { '7', '0' }, { '7', '1' }, { '7', '2' }, { '7', '3' }, { '7', '4' }, { '7', '5' }, { '7', '6' }, { '7', '7' }, { '7', '8' }, { '7', '9' }, - { '8', '0' }, { '8', '1' }, { '8', '2' }, { '8', '3' }, { '8', '4' }, { '8', '5' }, { '8', '6' }, { '8', '7' }, { '8', '8' }, { '8', '9' }, - { '9', '0' }, { '9', '1' }, { '9', '2' }, { '9', '3' }, { '9', '4' }, { '9', '5' }, { '9', '6' }, { '9', '7' }, { '9', '8' }, { '9', '9' }, -}; +static const struct digit_pair *digits_fd = (struct digit_pair *)( + "0_" "1_" "2_" "3_" "4_" "5_" "6_" "7_" "8_" "9_" + "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" + "20" "21" "22" "23" "24" "25" "26" "27" "28" "29" + "30" "31" "32" "33" "34" "35" "36" "37" "38" "39" + "40" "41" "42" "43" "44" "45" "46" "47" "48" "49" + "50" "51" "52" "53" "54" "55" "56" "57" "58" "59" + "60" "61" "62" "63" "64" "65" "66" "67" "68" "69" + "70" "71" "72" "73" "74" "75" "76" "77" "78" "79" + "80" "81" "82" "83" "84" "85" "86" "87" "88" "89" + "90" "91" "92" "93" "94" "95" "96" "97" "98" "99" +); -#undef NUL +static const u64_t mask24 = (u64(1) << 24) - 1; +static const u64_t mask32 = (u64(1) << 32) - 1; +static const u64_t mask57 = (u64(1) << 57) - 1; -static u64_t mask24 = (u64(1) << 24) - 1; -static u64_t mask32 = (u64(1) << 32) - 1; -static u64_t mask57 = (u64(1) << 57) - 1; +#define COPY(buffer, digits) memcpy(buffer, &(digits), sizeof(struct digit_pair)) -static -char* to_text_from_ulong(char* b, u64_t n) { - if (n < u32(1e2)) - { - *cast_to_pair_ptr(b) = digits_fd[n]; +static char * +jeaiii_ultoa(char *b, u64_t n) +{ + if (n < u32(1e2)) { + COPY(b, digits_fd[n]); return n < 10 ? b + 1 : b + 2; } - if (n < u32(1e6)) - { - if (n < u32(1e4)) - { + + if (n < u32(1e6)) { + if (n < u32(1e4)) { u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * n; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + COPY(b, digits_fd[f0 >> 24]); + b -= n < u32(1e3); u32_t f2 = (f0 & mask24) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + COPY(b + 2, digits_dd[f2 >> 24]); + return b + 4; } + u64_t f0 = u64(10 * (1ull << 32ull)/ 1e5 + 1) * n; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + COPY(b, digits_fd[f0 >> 32]); + b -= n < u32(1e5); u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); return b + 6; } - if (n < u64(1ull << 32ull)) - { - if (n < u32(1e8)) - { + + if (n < u64(1ull << 32ull)) { + if (n < u32(1e8)) { u64_t f0 = u64(10 * (1ull << 48ull) / 1e7 + 1) * n >> 16; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + COPY(b, digits_fd[f0 >> 32]); + b -= n < u32(1e7); u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); + u64_t f6 = (f4 & mask32) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + COPY(b + 6, digits_dd[f6 >> 32]); + return b + 8; } + u64_t f0 = u64(10 * (1ull << 57ull) / 1e9 + 1) * n; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 57]; + COPY(b, digits_fd[f0 >> 57]); + b -= n < u32(1e9); u64_t f2 = (f0 & mask57) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 57]; + COPY(b + 2, digits_dd[f2 >> 57]); + u64_t f4 = (f2 & mask57) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 57]; + COPY(b + 4, digits_dd[f4 >> 57]); + u64_t f6 = (f4 & mask57) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 57]; + COPY(b + 6, digits_dd[f6 >> 57]); + u64_t f8 = (f6 & mask57) * 100; - *cast_to_pair_ptr(b + 8) = digits_dd[f8 >> 57]; + COPY(b + 8, digits_dd[f8 >> 57]); + return b + 10; } @@ -147,108 +154,121 @@ char* to_text_from_ulong(char* b, u64_t n) { u32_t z = n % u32(1e8); u64_t u = n / u32(1e8); - if (u < u32(1e2)) - { + if (u < u32(1e2)) { // u can't be 1 digit (if u < 10 it would have been handled above as a 9 digit 32bit number) - *cast_to_pair_ptr(b) = digits_dd[u]; + COPY(b, digits_dd[u]); b += 2; } - else if (u < u32(1e6)) - { - if (u < u32(1e4)) - { + else if (u < u32(1e6)) { + if (u < u32(1e4)) { u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + COPY(b, digits_fd[f0 >> 24]); + b -= u < u32(1e3); u32_t f2 = (f0 & mask24) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + COPY(b + 2, digits_dd[f2 >> 24]); b += 4; } - else - { + else { u64_t f0 = u64(10 * (1ull << 32ull) / 1e5 + 1) * u; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + COPY(b, digits_fd[f0 >> 32]); + b -= u < u32(1e5); u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); b += 6; } } - else if (u < u32(1e8)) - { + else if (u < u32(1e8)) { u64_t f0 = u64(10 * (1ull << 48ull) / 1e7 + 1) * u >> 16; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 32]; + COPY(b, digits_fd[f0 >> 32]); + b -= u < u32(1e7); u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); + u64_t f6 = (f4 & mask32) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + COPY(b + 6, digits_dd[f6 >> 32]); + b += 8; } - else if (u < u64(1ull << 32ull)) - { + else if (u < u64(1ull << 32ull)) { u64_t f0 = u64(10 * (1ull << 57ull) / 1e9 + 1) * u; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 57]; + COPY(b, digits_fd[f0 >> 57]); + b -= u < u32(1e9); u64_t f2 = (f0 & mask57) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 57]; + COPY(b + 2, digits_dd[f2 >> 57]); + u64_t f4 = (f2 & mask57) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 57]; + COPY(b + 4, digits_dd[f4 >> 57]); + u64_t f6 = (f4 & mask57) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 57]; + COPY(b + 6, digits_dd[f6 >> 57]); + u64_t f8 = (f6 & mask57) * 100; - *cast_to_pair_ptr(b + 8) = digits_dd[f8 >> 57]; + COPY(b + 8, digits_dd[f8 >> 57]); b += 10; } - else - { + else { u32_t y = u % u32(1e8); u /= u32(1e8); // u is 2, 3, or 4 digits (if u < 10 it would have been handled above) - if (u < u32(1e2)) - { - *cast_to_pair_ptr(b) = digits_dd[u]; + if (u < u32(1e2)) { + COPY(b, digits_dd[u]); b += 2; } - else - { + else { u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; - *cast_to_pair_ptr(b) = digits_fd[f0 >> 24]; + COPY(b, digits_fd[f0 >> 24]); + b -= u < u32(1e3); u32_t f2 = (f0 & mask24) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 24]; + COPY(b + 2, digits_dd[f2 >> 24]); + b += 4; } // do 8 digits u64_t f0 = (u64((1ull << 48ull) / 1e6 + 1) * y >> 16) + 1; - *cast_to_pair_ptr(b) = digits_dd[f0 >> 32]; + COPY(b, digits_dd[f0 >> 32]); + u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); + u64_t f6 = (f4 & mask32) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + COPY(b + 6, digits_dd[f6 >> 32]); b += 8; } + // do 8 digits u64_t f0 = (u64((1ull << 48ull) / 1e6 + 1) * z >> 16) + 1; - *cast_to_pair_ptr(b) = digits_dd[f0 >> 32]; + COPY(b, digits_dd[f0 >> 32]); + u64_t f2 = (f0 & mask32) * 100; - *cast_to_pair_ptr(b + 2) = digits_dd[f2 >> 32]; + COPY(b + 2, digits_dd[f2 >> 32]); + u64_t f4 = (f2 & mask32) * 100; - *cast_to_pair_ptr(b + 4) = digits_dd[f4 >> 32]; + COPY(b + 4, digits_dd[f4 >> 32]); + u64_t f6 = (f4 & mask32) * 100; - *cast_to_pair_ptr(b + 6) = digits_dd[f6 >> 32]; + COPY(b + 6, digits_dd[f6 >> 32]); + return b + 8; } #undef u32 #undef u64 +#undef COPY #pragma clang diagnostic pop #pragma GCC diagnostic pop From 172762c6e4a928cb9afe8abd2bba9e563ed0e23a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 26 Mar 2025 11:47:51 +0100 Subject: [PATCH 014/506] Deprecate all `*_default_options` Globally changing the behavior of the library is a bad idea, as many different libraries may rely on `json` and may not expect it and likely never tested that a different default config works for them. If you need to change the behavior of JSON, it's best to do it only locally, and not globally. In addition the new `JSON::Coder` interface is much more suited for that. Another reason for the deprecation is that it's impossible to make `JSON.load` and `JSON.dump` Ractor-safe with such API. --- CHANGES.md | 4 ++ lib/json/common.rb | 77 +++++++++++++++---------- test/json/json_common_interface_test.rb | 8 ++- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eca83cdaa..cd11229cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Changes +* Deprecate `JSON.load_default_options`. +* Deprecate `JSON.unsafe_load_default_options`. +* Deprecate `JSON.dump_default_options`. + ### 2025-03-12 (2.10.2) * Fix a potential crash in the C extension parser. diff --git a/lib/json/common.rb b/lib/json/common.rb index 9094df00a..800056c49 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -101,6 +101,29 @@ def create_pretty_state # Sets or Returns the JSON generator state class that is used by JSON. attr_accessor :state + + private + + def deprecated_singleton_attr_accessor(*attrs) + args = RUBY_VERSION >= "3.0" ? ", category: :deprecated" : "" + attrs.each do |attr| + singleton_class.class_eval <<~RUBY + def #{attr} + warn "JSON.#{attr} is deprecated and will be removed in json 3.0.0", uplevel: 1 #{args} + @#{attr} + end + + def #{attr}=(val) + warn "JSON.#{attr}= is deprecated and will be removed in json 3.0.0", uplevel: 1 #{args} + @#{attr} = val + end + + def _#{attr} + @#{attr} + end + RUBY + end + end end # Sets create identifier, which is used to decide if the _json_create_ @@ -424,28 +447,26 @@ def pretty_generate(obj, opts = nil) module_function :pretty_unparse # :startdoc: - class << self - # Sets or returns default options for the JSON.unsafe_load method. - # Initially: - # opts = JSON.load_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} - attr_accessor :unsafe_load_default_options - end - self.unsafe_load_default_options = { + # Sets or returns default options for the JSON.unsafe_load method. + # Initially: + # opts = JSON.load_default_options + # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} + deprecated_singleton_attr_accessor :unsafe_load_default_options + + @unsafe_load_default_options = { :max_nesting => false, :allow_nan => true, :allow_blank => true, :create_additions => true, } - class << self - # Sets or returns default options for the JSON.load method. - # Initially: - # opts = JSON.load_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} - attr_accessor :load_default_options - end - self.load_default_options = { + # Sets or returns default options for the JSON.load method. + # Initially: + # opts = JSON.load_default_options + # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} + deprecated_singleton_attr_accessor :load_default_options + + @load_default_options = { :allow_nan => true, :allow_blank => true, :create_additions => nil, @@ -581,9 +602,9 @@ class << self # def unsafe_load(source, proc = nil, options = nil) opts = if options.nil? - unsafe_load_default_options + _unsafe_load_default_options else - unsafe_load_default_options.merge(options) + _unsafe_load_default_options.merge(options) end unless source.is_a?(String) @@ -741,9 +762,9 @@ def unsafe_load(source, proc = nil, options = nil) # def load(source, proc = nil, options = nil) opts = if options.nil? - load_default_options + _load_default_options else - load_default_options.merge(options) + _load_default_options.merge(options) end unless source.is_a?(String) @@ -781,14 +802,12 @@ def recurse_proc(result, &proc) # :nodoc: alias restore load module_function :restore - class << self - # Sets or returns the default options for the JSON.dump method. - # Initially: - # opts = JSON.dump_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true} - attr_accessor :dump_default_options - end - self.dump_default_options = { + # Sets or returns the default options for the JSON.dump method. + # Initially: + # opts = JSON.dump_default_options + # opts # => {:max_nesting=>false, :allow_nan=>true} + deprecated_singleton_attr_accessor :dump_default_options + @dump_default_options = { :max_nesting => false, :allow_nan => true, } @@ -841,7 +860,7 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end end - opts = JSON.dump_default_options + opts = JSON._dump_default_options opts = opts.merge(:max_nesting => limit) if limit opts = opts.merge(kwargs) if kwargs diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 1f157da02..96f7ccdf9 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -174,9 +174,9 @@ def test_dump_in_io end def test_dump_should_modify_defaults - max_nesting = JSON.dump_default_options[:max_nesting] + max_nesting = JSON._dump_default_options[:max_nesting] dump([], StringIO.new, 10) - assert_equal max_nesting, JSON.dump_default_options[:max_nesting] + assert_equal max_nesting, JSON._dump_default_options[:max_nesting] end def test_JSON @@ -211,6 +211,10 @@ def test_load_file_with_bad_default_external_encoding end end + def test_deprecated_dump_default_options + assert JSON.dump_default_options + end + private def with_external_encoding(encoding) From a88d825a9169a9b778e598cf7e7626a19059b4dd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 26 Mar 2025 15:23:42 +0100 Subject: [PATCH 015/506] Remove `unparse` and `restore` aliases. These were deprecated 16 years ago. --- CHANGES.md | 4 ++++ lib/json/common.rb | 22 ---------------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index cd11229cc..159c4f00f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,10 @@ * Deprecate `JSON.load_default_options`. * Deprecate `JSON.unsafe_load_default_options`. * Deprecate `JSON.dump_default_options`. +* Remove deprecated `JSON.restore` method. +* Remove deprecated `JSON.unparse` method. +* Remove deprecated `JSON.fast_unparse` method. +* Remove deprecated `JSON.pretty_unparse` method. ### 2025-03-12 (2.10.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index 800056c49..af130dc00 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -357,13 +357,6 @@ def generate(obj, opts = nil) end end - # :stopdoc: - # I want to deprecate these later, so I'll first be silent about them, and - # later delete them. - alias unparse generate - module_function :unparse - # :startdoc: - # :call-seq: # JSON.fast_generate(obj, opts) -> new_string # @@ -386,12 +379,6 @@ def fast_generate(obj, opts = nil) state.generate(obj) end - # :stopdoc: - # I want to deprecate these later, so I'll first be silent about them, and later delete them. - alias fast_unparse fast_generate - module_function :fast_unparse - # :startdoc: - # :call-seq: # JSON.pretty_generate(obj, opts = nil) -> new_string # @@ -441,12 +428,6 @@ def pretty_generate(obj, opts = nil) state.generate(obj) end - # :stopdoc: - # I want to deprecate these later, so I'll first be silent about them, and later delete them. - alias pretty_unparse pretty_generate - module_function :pretty_unparse - # :startdoc: - # Sets or returns default options for the JSON.unsafe_load method. # Initially: # opts = JSON.load_default_options @@ -799,9 +780,6 @@ def recurse_proc(result, &proc) # :nodoc: end end - alias restore load - module_function :restore - # Sets or returns the default options for the JSON.dump method. # Initially: # opts = JSON.dump_default_options From 18e5c3c67cdce502c8120a18d49ba9c9f53f0a9c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 26 Mar 2025 15:44:43 +0100 Subject: [PATCH 016/506] Remove unused `FAST_STATE_PROTOTYPE` and `PRETTY_STATE_PROTOTYPE` --- lib/json/common.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index af130dc00..0498a28b9 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -71,8 +71,6 @@ def generator=(generator) # :nodoc: self.state = generator::State const_set :State, self.state const_set :SAFE_STATE_PROTOTYPE, State.new # for JRuby - const_set :FAST_STATE_PROTOTYPE, create_fast_state - const_set :PRETTY_STATE_PROTOTYPE, create_pretty_state ensure $VERBOSE = old end From 8449ad3abf278bfe7056f6007d61008a34f2d07e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 26 Mar 2025 16:37:03 +0100 Subject: [PATCH 017/506] Get rid of SAFE_STATE_PROTOTYPE It was only used by JRuby and TruffleRuby to call `SAFE_STATE_PROTOTYPE.dup` instead of `State.new` which isn't an worthy optimization. --- java/src/json/ext/GeneratorState.java | 3 +-- java/src/json/ext/RuntimeInfo.java | 13 ------------- lib/json/common.rb | 1 - lib/json/truffle_ruby/generator.rb | 2 +- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 524d6b057..9c15699fb 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -158,8 +158,7 @@ static GeneratorState fromState(ThreadContext context, RuntimeInfo info, } } - // for other values, return the safe prototype - return (GeneratorState)info.getSafeStatePrototype(context).dup(); + return (GeneratorState)klass.newInstance(context, context.nil); } /** diff --git a/java/src/json/ext/RuntimeInfo.java b/java/src/json/ext/RuntimeInfo.java index f712dc251..9906a5d95 100644 --- a/java/src/json/ext/RuntimeInfo.java +++ b/java/src/json/ext/RuntimeInfo.java @@ -30,8 +30,6 @@ final class RuntimeInfo { WeakReference jsonModule; /** JSON::Ext::Generator::State */ WeakReference generatorStateClass; - /** JSON::SAFE_STATE_PROTOTYPE */ - WeakReference safeStatePrototype; private RuntimeInfo() { } @@ -67,15 +65,4 @@ public static RuntimeInfo forRuntime(Ruby runtime) { return cache; } } - - public GeneratorState getSafeStatePrototype(ThreadContext context) { - if (safeStatePrototype == null) { - IRubyObject value = jsonModule.get().getConstant("SAFE_STATE_PROTOTYPE"); - if (!(value instanceof GeneratorState)) { - throw context.runtime.newTypeError(value, generatorStateClass.get()); - } - safeStatePrototype = new WeakReference<>((GeneratorState) value); - } - return safeStatePrototype.get(); - } } diff --git a/lib/json/common.rb b/lib/json/common.rb index 0498a28b9..b2f11afe5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -70,7 +70,6 @@ def generator=(generator) # :nodoc: end self.state = generator::State const_set :State, self.state - const_set :SAFE_STATE_PROTOTYPE, State.new # for JRuby ensure $VERBOSE = old end diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index bf6289f1f..c814106de 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -117,7 +117,7 @@ def self.from_state(opts) return new(opts.to_h) end end - SAFE_STATE_PROTOTYPE.dup + new end # Instantiates a new State object, configured by _opts_. From 93a62a06d27fa12ebe8d04a0d038be64fd9067af Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 27 Mar 2025 10:12:53 +0900 Subject: [PATCH 018/506] Removed trailing spaces --- ext/json/ext/fbuffer/fbuffer.h | 4 ++-- test/json/json_generator_test.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 18e2e963f..b8a4e983d 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -196,7 +196,7 @@ static void fbuffer_append_long(FBuffer *fb, long number) * the number of resulting characters. * * We do know, however, that the `number` argument is always in the - * range 0xc000000000000000 to 0x3fffffffffffffff, or, in decimal, + * range 0xc000000000000000 to 0x3fffffffffffffff, or, in decimal, * -4611686018427387904 to 4611686018427387903. The max number of chars * generated is therefore 20 (including a potential sign character). */ @@ -208,7 +208,7 @@ static void fbuffer_append_long(FBuffer *fb, long number) if (number < 0) { fbuffer_append_reserved_char(fb, '-'); - /* + /* * Since number is always > LONG_MIN, `-number` will not overflow * and is always the positive abs() value. */ diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 9cca447e2..f0dec452f 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -707,7 +707,7 @@ def test_json_generate_float assert_equal expected, value.to_json end end - + def test_numbers_of_various_sizes numbers = [ 0, 1, -1, 9, -9, 13, -13, 91, -91, 513, -513, 7513, -7513, From 6508455d821d8bad5b73830c93010f37e4ce6651 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 09:54:51 +0100 Subject: [PATCH 019/506] Deprecate `JSON.fast_generate` --- CHANGES.md | 1 + lib/json/common.rb | 18 ++++-------------- test/json/json_generator_test.rb | 19 ------------------- 3 files changed, 5 insertions(+), 33 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 159c4f00f..9e00cd543 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,6 @@ # Changes +* Deprecate `JSON.fast_generate` (it's not any faster, so pointless). * Deprecate `JSON.load_default_options`. * Deprecate `JSON.unsafe_load_default_options`. * Deprecate `JSON.dump_default_options`. diff --git a/lib/json/common.rb b/lib/json/common.rb index b2f11afe5..c9f6d7505 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -74,16 +74,6 @@ def generator=(generator) # :nodoc: $VERBOSE = old end - def create_fast_state - State.new( - :indent => '', - :space => '', - :object_nl => "", - :array_nl => "", - :max_nesting => false - ) - end - def create_pretty_state State.new( :indent => ' ', @@ -368,12 +358,12 @@ def generate(obj, opts = nil) # # Raises SystemStackError (stack level too deep): # JSON.fast_generate(a) def fast_generate(obj, opts = nil) - if State === opts - state = opts + if RUBY_VERSION >= "3.0" + warn "JSON.fast_generate is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated else - state = JSON.create_fast_state.configure(opts) + warn "JSON.fast_generate is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 end - state.generate(obj) + generate(obj, opts) end # :call-seq: diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f0dec452f..64636c12c 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -237,25 +237,6 @@ def test_safe_state }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end - def test_fast_state - state = JSON.create_fast_state - assert_equal({ - :allow_nan => false, - :array_nl => "", - :as_json => false, - :ascii_only => false, - :buffer_initial_length => 1024, - :depth => 0, - :script_safe => false, - :strict => false, - :indent => "", - :max_nesting => 0, - :object_nl => "", - :space => "", - :space_before => "", - }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) - end - def test_allow_nan error = assert_raise(GeneratorError) { generate([JSON::NaN]) } assert_same JSON::NaN, error.invalid_object From 01c47a055574d86619f5687b9928a14147cba4ee Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 10:16:10 +0100 Subject: [PATCH 020/506] Cleanup JSON.pretty_generate --- lib/json/common.rb | 45 ++++++++++++++++---------------- test/json/json_generator_test.rb | 21 +-------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index c9f6d7505..1d939b7d1 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -74,15 +74,6 @@ def generator=(generator) # :nodoc: $VERBOSE = old end - def create_pretty_state - State.new( - :indent => ' ', - :space => ' ', - :object_nl => "\n", - :array_nl => "\n" - ) - end - # Returns the JSON generator module that is used by JSON. attr_reader :generator @@ -366,6 +357,14 @@ def fast_generate(obj, opts = nil) generate(obj, opts) end + PRETTY_GENERATE_OPTIONS = { + indent: ' ', + space: ' ', + object_nl: "\n", + array_nl: "\n", + }.freeze + private_constant :PRETTY_GENERATE_OPTIONS + # :call-seq: # JSON.pretty_generate(obj, opts = nil) -> new_string # @@ -397,22 +396,24 @@ def fast_generate(obj, opts = nil) # } # def pretty_generate(obj, opts = nil) - if State === opts - state, opts = opts, nil - else - state = JSON.create_pretty_state - end + return state.generate(obj) if State === opts + + options = PRETTY_GENERATE_OPTIONS + if opts - if opts.respond_to? :to_hash - opts = opts.to_hash - elsif opts.respond_to? :to_h - opts = opts.to_h - else - raise TypeError, "can't convert #{opts.class} into Hash" + unless opts.is_a?(Hash) + if opts.respond_to? :to_hash + opts = opts.to_hash + elsif opts.respond_to? :to_h + opts = opts.to_h + else + raise TypeError, "can't convert #{opts.class} into Hash" + end end - state.configure(opts) + options = options.merge(opts) end - state.generate(obj) + + State.generate(obj, options, nil) end # Sets or returns default options for the JSON.unsafe_load method. diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 64636c12c..05006ebd0 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -199,26 +199,7 @@ def test_falsy_state ) end - def test_pretty_state - state = JSON.create_pretty_state - assert_equal({ - :allow_nan => false, - :array_nl => "\n", - :as_json => false, - :ascii_only => false, - :buffer_initial_length => 1024, - :depth => 0, - :script_safe => false, - :strict => false, - :indent => " ", - :max_nesting => 100, - :object_nl => "\n", - :space => " ", - :space_before => "", - }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) - end - - def test_safe_state + def test_state_defaults state = JSON::State.new assert_equal({ :allow_nan => false, From 3de8702354395121357b94aff3d68935a9ac709f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 10:16:14 +0100 Subject: [PATCH 021/506] Remove outdated JSON.iconv --- CHANGES.md | 1 + lib/json/common.rb | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9e00cd543..b1d705426 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * Deprecate `JSON.load_default_options`. * Deprecate `JSON.unsafe_load_default_options`. * Deprecate `JSON.dump_default_options`. +* Remove outdated `JSON.iconv`. * Remove deprecated `JSON.restore` method. * Remove deprecated `JSON.unparse` method. * Remove deprecated `JSON.fast_unparse` method. diff --git a/lib/json/common.rb b/lib/json/common.rb index 1d939b7d1..24f124baa 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -837,11 +837,6 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end end - # Encodes string using String.encode. - def self.iconv(to, from, string) - string.encode(to, from) - end - # JSON::Coder holds a parser and generator configuration. # # module MyApp From 760d9222657a7d6625f11aa1c6eba4058ae29722 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 10:36:31 +0100 Subject: [PATCH 022/506] Get rid of JSON::NOT_SET --- lib/json/common.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 24f124baa..30fdbe1db 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -5,9 +5,6 @@ module JSON autoload :GenericObject, 'json/generic_object' - NOT_SET = Object.new.freeze - private_constant :NOT_SET - class << self # :call-seq: # JSON[object] -> new_array or new_string From 1ca7efed1f6e13d48934cd12c403d2ed08ac9fc1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 10:34:17 +0100 Subject: [PATCH 023/506] Remove `Class#json_creatable?` monkey patch. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 13 ++- java/src/json/ext/ParserConfig.java | 160 +++++++++++++++------------- java/src/json/ext/ParserConfig.rl | 14 ++- lib/json/common.rb | 11 -- test/json/json_addition_test.rb | 14 +-- 6 files changed, 106 insertions(+), 107 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b1d705426..743b9e5a2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ * Deprecate `JSON.unsafe_load_default_options`. * Deprecate `JSON.dump_default_options`. * Remove outdated `JSON.iconv`. +* Remove `Class#json_creatable?` monkey patch. * Remove deprecated `JSON.restore` method. * Remove deprecated `JSON.unparse` method. * Remove deprecated `JSON.fast_unparse` method. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d990612a2..d2d0d38d8 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -790,6 +790,15 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig return array; } +static bool json_obj_creatable_p(VALUE klass) +{ + if (rb_respond_to(klass, i_json_creatable_p)) { + return RTEST(rb_funcall(klass, i_json_creatable_p, 0)); + } else { + return rb_respond_to(klass, i_json_create); + } +} + static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, long count) { VALUE object; @@ -818,7 +827,7 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi } if (!NIL_P(klassname)) { VALUE klass = rb_funcall(mJSON, i_deep_const_get, 1, klassname); - if (RTEST(rb_funcall(klass, i_json_creatable_p, 0))) { + if (json_obj_creatable_p(klass)) { if (config->deprecated_create_additions) { json_deprecated(deprecated_create_additions_warning); } @@ -837,7 +846,7 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi static int match_i(VALUE regexp, VALUE klass, VALUE memo) { if (regexp == Qundef) return ST_STOP; - if (RTEST(rb_funcall(klass, i_json_creatable_p, 0)) && + if (json_obj_creatable_p(klass) && RTEST(rb_funcall(regexp, i_match, 1, rb_ary_entry(memo, 0)))) { rb_ary_push(memo, klass); return ST_STOP; diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 6596f97f0..83251fc8d 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -266,6 +266,14 @@ private IRubyObject createCustomDecimal(final ThreadContext context, final ByteL return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); } + private static boolean isObjCreateable(final ThreadContext context, IRubyObject klass) { + if (klass.respondsTo("json_creatable?")) { + return klass.callMethod(context, "json_creatable?").isTrue(); + } else { + return klass.respondsTo("json_create"); + } + } + /** * A string parsing session. * @@ -303,11 +311,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 329 "ParserConfig.rl" +// line 337 "ParserConfig.rl" -// line 311 "ParserConfig.java" +// line 319 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -421,7 +429,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 435 "ParserConfig.rl" +// line 443 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -429,14 +437,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 433 "ParserConfig.java" +// line 441 "ParserConfig.java" { cs = JSON_value_start; } -// line 442 "ParserConfig.rl" +// line 450 "ParserConfig.rl" -// line 440 "ParserConfig.java" +// line 448 "ParserConfig.java" { int _klen; int _trans = 0; @@ -462,13 +470,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 420 "ParserConfig.rl" +// line 428 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 472 "ParserConfig.java" +// line 480 "ParserConfig.java" } } @@ -531,25 +539,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 337 "ParserConfig.rl" +// line 345 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 340 "ParserConfig.rl" +// line 348 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 343 "ParserConfig.rl" +// line 351 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 346 "ParserConfig.rl" +// line 354 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -559,7 +567,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 353 "ParserConfig.rl" +// line 361 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -569,7 +577,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 360 "ParserConfig.rl" +// line 368 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -598,7 +606,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 386 "ParserConfig.rl" +// line 394 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -611,7 +619,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 396 "ParserConfig.rl" +// line 404 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -626,7 +634,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 408 "ParserConfig.rl" +// line 416 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -640,7 +648,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 644 "ParserConfig.java" +// line 652 "ParserConfig.java" } } } @@ -660,7 +668,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 443 "ParserConfig.rl" +// line 451 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -673,7 +681,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 677 "ParserConfig.java" +// line 685 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -772,7 +780,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 465 "ParserConfig.rl" +// line 473 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -789,15 +797,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 793 "ParserConfig.java" +// line 801 "ParserConfig.java" { cs = JSON_integer_start; } -// line 481 "ParserConfig.rl" +// line 489 "ParserConfig.rl" int memo = p; -// line 801 "ParserConfig.java" +// line 809 "ParserConfig.java" { int _klen; int _trans = 0; @@ -878,13 +886,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 459 "ParserConfig.rl" +// line 467 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 888 "ParserConfig.java" +// line 896 "ParserConfig.java" } } } @@ -904,7 +912,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 483 "ParserConfig.rl" +// line 491 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -924,7 +932,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 928 "ParserConfig.java" +// line 936 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1026,7 +1034,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 516 "ParserConfig.rl" +// line 524 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1045,15 +1053,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1049 "ParserConfig.java" +// line 1057 "ParserConfig.java" { cs = JSON_float_start; } -// line 534 "ParserConfig.rl" +// line 542 "ParserConfig.rl" int memo = p; -// line 1057 "ParserConfig.java" +// line 1065 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1134,13 +1142,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 507 "ParserConfig.rl" +// line 515 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1144 "ParserConfig.java" +// line 1152 "ParserConfig.java" } } } @@ -1160,7 +1168,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 536 "ParserConfig.rl" +// line 544 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1170,7 +1178,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1174 "ParserConfig.java" +// line 1182 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1272,7 +1280,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 575 "ParserConfig.rl" +// line 583 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1280,15 +1288,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1284 "ParserConfig.java" +// line 1292 "ParserConfig.java" { cs = JSON_string_start; } -// line 582 "ParserConfig.rl" +// line 590 "ParserConfig.rl" int memo = p; -// line 1292 "ParserConfig.java" +// line 1300 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1369,7 +1377,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 550 "ParserConfig.rl" +// line 558 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1384,13 +1392,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 563 "ParserConfig.rl" +// line 571 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1394 "ParserConfig.java" +// line 1402 "ParserConfig.java" } } } @@ -1410,7 +1418,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 584 "ParserConfig.rl" +// line 592 "ParserConfig.rl" if (config.createAdditions) { RubyHash matchString = config.match_string; @@ -1421,8 +1429,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } catch (JumpException e) { } if (memoArray[1] != null) { RubyClass klass = (RubyClass) memoArray[1]; - if (klass.respondsTo("json_creatable?") && - klass.callMethod(context, "json_creatable?").isTrue()) { + if (isObjCreateable(context, klass)) { if (config.deprecatedCreateAdditions) { context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); } @@ -1451,7 +1458,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1455 "ParserConfig.java" +// line 1462 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1618,7 +1625,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 662 "ParserConfig.rl" +// line 669 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1638,14 +1645,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { } -// line 1642 "ParserConfig.java" +// line 1649 "ParserConfig.java" { cs = JSON_array_start; } -// line 681 "ParserConfig.rl" +// line 688 "ParserConfig.rl" -// line 1649 "ParserConfig.java" +// line 1656 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1688,7 +1695,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 629 "ParserConfig.rl" +// line 636 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1758,7 +1765,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 631 "ParserConfig.rl" +// line 638 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1775,13 +1782,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 646 "ParserConfig.rl" +// line 653 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1785 "ParserConfig.java" +// line 1792 "ParserConfig.java" } } } @@ -1801,7 +1808,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 682 "ParserConfig.rl" +// line 689 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(result, p + 1); @@ -1811,7 +1818,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1815 "ParserConfig.java" +// line 1822 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1988,7 +1995,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 743 "ParserConfig.rl" +// line 750 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -2013,14 +2020,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { } -// line 2017 "ParserConfig.java" +// line 2024 "ParserConfig.java" { cs = JSON_object_start; } -// line 767 "ParserConfig.rl" +// line 774 "ParserConfig.rl" -// line 2024 "ParserConfig.java" +// line 2031 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2063,7 +2070,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 696 "ParserConfig.rl" +// line 703 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2133,7 +2140,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 698 "ParserConfig.rl" +// line 705 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2150,7 +2157,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 713 "ParserConfig.rl" +// line 720 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2168,13 +2175,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 729 "ParserConfig.rl" +// line 736 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2178 "ParserConfig.java" +// line 2185 "ParserConfig.java" } } } @@ -2194,7 +2201,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 768 "ParserConfig.rl" +// line 775 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2216,8 +2223,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) // might throw ArgumentError, we let it propagate IRubyObject klass = config.info.jsonModule.get(). callMethod(context, "deep_const_get", vKlassName); - if (klass.respondsTo("json_creatable?") && - klass.callMethod(context, "json_creatable?").isTrue()) { + if (isObjCreateable(context, klass)) { if (config.deprecatedCreateAdditions) { context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); } @@ -2230,7 +2236,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2234 "ParserConfig.java" +// line 2240 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2333,7 +2339,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 822 "ParserConfig.rl" +// line 828 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2343,16 +2349,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2347 "ParserConfig.java" +// line 2353 "ParserConfig.java" { cs = JSON_start; } -// line 831 "ParserConfig.rl" +// line 837 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2356 "ParserConfig.java" +// line 2362 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2433,7 +2439,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 808 "ParserConfig.rl" +// line 814 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2445,7 +2451,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2449 "ParserConfig.java" +// line 2455 "ParserConfig.java" } } } @@ -2465,7 +2471,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 834 "ParserConfig.rl" +// line 840 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 0382a7c5d..8688a1016 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -264,6 +264,14 @@ public class ParserConfig extends RubyObject { return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); } + private static boolean isObjCreateable(final ThreadContext context, IRubyObject klass) { + if (klass.respondsTo("json_creatable?")) { + return klass.callMethod(context, "json_creatable?").isTrue(); + } else { + return klass.respondsTo("json_create"); + } + } + /** * A string parsing session. * @@ -591,8 +599,7 @@ public class ParserConfig extends RubyObject { } catch (JumpException e) { } if (memoArray[1] != null) { RubyClass klass = (RubyClass) memoArray[1]; - if (klass.respondsTo("json_creatable?") && - klass.callMethod(context, "json_creatable?").isTrue()) { + if (isObjCreateable(context, klass)) { if (config.deprecatedCreateAdditions) { context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); } @@ -786,8 +793,7 @@ public class ParserConfig extends RubyObject { // might throw ArgumentError, we let it propagate IRubyObject klass = config.info.jsonModule.get(). callMethod(context, "deep_const_get", vKlassName); - if (klass.respondsTo("json_creatable?") && - klass.callMethod(context, "json_creatable?").isTrue()) { + if (isObjCreateable(context, klass)) { if (config.deprecatedCreateAdditions) { context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); } diff --git a/lib/json/common.rb b/lib/json/common.rb index 30fdbe1db..c47152e38 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -951,14 +951,3 @@ def JSON(object, *args) JSON.generate(object, args.first) end end - -# Extends any Class to include _json_creatable?_ method. -class ::Class - # Returns true if this class can be used to create an instance - # from a serialised JSON string. The class has to implement a class - # method _json_create_ that expects a hash as first parameter. The hash - # should include the required data. - def json_creatable? - respond_to?(:json_create) - end -end diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index 1eb269c2f..4d8d18687 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -44,10 +44,6 @@ def to_json(*args) end class B - def self.json_creatable? - false - end - def to_json(*args) { 'json_class' => self.class.name, @@ -56,10 +52,6 @@ def to_json(*args) end class C - def self.json_creatable? - false - end - def to_json(*args) { 'json_class' => 'JSONAdditionTest::Nix', @@ -69,7 +61,6 @@ def to_json(*args) def test_extended_json a = A.new(666) - assert A.json_creatable? json = generate(a) a_again = parse(json, :create_additions => true) assert_kind_of a.class, a_again @@ -78,7 +69,7 @@ def test_extended_json def test_extended_json_default a = A.new(666) - assert A.json_creatable? + assert A.respond_to?(:json_create) json = generate(a) a_hash = parse(json) assert_kind_of Hash, a_hash @@ -86,7 +77,6 @@ def test_extended_json_default def test_extended_json_disabled a = A.new(666) - assert A.json_creatable? json = generate(a) a_again = parse(json, :create_additions => true) assert_kind_of a.class, a_again @@ -101,14 +91,12 @@ def test_extended_json_disabled def test_extended_json_fail1 b = B.new - assert !B.json_creatable? json = generate(b) assert_equal({ "json_class"=>"JSONAdditionTest::B" }, parse(json)) end def test_extended_json_fail2 c = C.new - assert !C.json_creatable? json = generate(c) assert_raise(ArgumentError, NameError) { parse(json, :create_additions => true) } end From eda239ad348ac912477144e322faee0f8bb48534 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 11:01:45 +0100 Subject: [PATCH 024/506] Remove more dead code --- CHANGES.md | 2 ++ lib/json/common.rb | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 743b9e5a2..6a71f0015 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,8 @@ * Remove deprecated `JSON.unparse` method. * Remove deprecated `JSON.fast_unparse` method. * Remove deprecated `JSON.pretty_unparse` method. +* Remove deprecated `JSON::UnparserError` constant. +* Remove outdated `JSON::MissingUnicodeSupport` constant. ### 2025-03-12 (2.10.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index c47152e38..9e9d6fb3e 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -161,13 +161,6 @@ def detailed_message(...) end end - # For backwards compatibility - UnparserError = GeneratorError # :nodoc: - - # This exception is raised if the required unicode support is missing on the - # system. Usually this means that the iconv library is not installed. - class MissingUnicodeSupport < JSONError; end - # Fragment of JSON document that is to be included as is: # fragment = JSON::Fragment.new("[1, 2, 3]") # JSON.generate({ count: 3, items: fragments }) From 73d2137fd3ad18e2a524553e80531f434cf26dde Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 26 Mar 2025 15:19:26 +0100 Subject: [PATCH 025/506] JSON.load invoke the proc callback directly from the parser. And substitute the return value like `Marshal.load` doesm which I can only assume was the intent. This also open the door to re-implement all the `create_addition` logic in `json/common.rb`. --- CHANGES.md | 2 + ext/json/ext/parser/parser.c | 41 +++--- java/src/json/ext/ParserConfig.java | 169 +++++++++++++----------- java/src/json/ext/ParserConfig.rl | 23 +++- lib/json/common.rb | 20 +-- test/json/json_common_interface_test.rb | 2 +- 6 files changed, 141 insertions(+), 116 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a71f0015..1bde9fc3b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Fix `JSON.load` proc argument to substitute the parsed object with the return value. + This better match `Marshal.load` behavior. * Deprecate `JSON.fast_generate` (it's not any faster, so pointless). * Deprecate `JSON.load_default_options`. * Deprecate `JSON.unsafe_load_default_options`. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d2d0d38d8..b08fadd7d 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -37,7 +37,7 @@ static ID i_json_creatable_p, i_json_create, i_create_id, static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze, sym_create_additions, sym_create_id, sym_object_class, sym_array_class, - sym_decimal_class, sym_match_string; + sym_decimal_class, sym_match_string, sym_on_load; static int binary_encindex; static int utf8_encindex; @@ -444,6 +444,7 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) } typedef struct JSON_ParserStruct { + VALUE on_load_proc; VALUE create_id; VALUE object_class; VALUE array_class; @@ -879,7 +880,14 @@ static inline VALUE json_decode_string(JSON_ParserState *state, JSON_ParserConfi return string; } -#define PUSH(result) rvalue_stack_push(state->stack, result, &state->stack_handle, &state->stack) +static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *config, VALUE value) +{ + if (RB_UNLIKELY(config->on_load_proc)) { + value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil); + } + rvalue_stack_push(state->stack, value, &state->stack_handle, &state->stack); + return value; +} static const bool string_scan[256] = { // ASCII Control Characters @@ -906,7 +914,7 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig case '"': { VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name); state->cursor++; - return PUSH(string); + return json_push_value(state, config, string); } case '\\': { state->cursor++; @@ -940,7 +948,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case 'n': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "null", 4) == 0)) { state->cursor += 4; - return PUSH(Qnil); + return json_push_value(state, config, Qnil); } raise_parse_error("unexpected token at '%s'", state->cursor); @@ -948,7 +956,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case 't': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { state->cursor += 4; - return PUSH(Qtrue); + return json_push_value(state, config, Qtrue); } raise_parse_error("unexpected token at '%s'", state->cursor); @@ -957,7 +965,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) // Note: memcmp with a small power of two compile to an integer comparison if ((state->end - state->cursor >= 5) && (memcmp(state->cursor + 1, "alse", 4) == 0)) { state->cursor += 5; - return PUSH(Qfalse); + return json_push_value(state, config, Qfalse); } raise_parse_error("unexpected token at '%s'", state->cursor); @@ -966,7 +974,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) // Note: memcmp with a small power of two compile to an integer comparison if (config->allow_nan && (state->end - state->cursor >= 3) && (memcmp(state->cursor + 1, "aN", 2) == 0)) { state->cursor += 3; - return PUSH(CNaN); + return json_push_value(state, config, CNaN); } raise_parse_error("unexpected token at '%s'", state->cursor); @@ -974,7 +982,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case 'I': if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { state->cursor += 8; - return PUSH(CInfinity); + return json_push_value(state, config, CInfinity); } raise_parse_error("unexpected token at '%s'", state->cursor); @@ -984,7 +992,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if ((state->end - state->cursor >= 9) && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { if (config->allow_nan) { state->cursor += 9; - return PUSH(CMinusInfinity); + return json_push_value(state, config, CMinusInfinity); } else { raise_parse_error("unexpected token at '%s'", state->cursor); } @@ -1041,9 +1049,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (integer) { - return PUSH(json_decode_integer(start, state->cursor)); + return json_push_value(state, config, json_decode_integer(start, state->cursor)); } - return PUSH(json_decode_float(config, start, state->cursor)); + return json_push_value(state, config, json_decode_float(config, start, state->cursor)); } case '"': { // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} @@ -1057,7 +1065,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if ((state->cursor < state->end) && (*state->cursor == ']')) { state->cursor++; - return PUSH(json_decode_array(state, config, 0)); + return json_push_value(state, config, json_decode_array(state, config, 0)); } else { state->current_nesting++; if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { @@ -1076,7 +1084,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) long count = state->stack->head - stack_head; state->current_nesting--; state->in_array--; - return PUSH(json_decode_array(state, config, count)); + return json_push_value(state, config, json_decode_array(state, config, count)); } if (*state->cursor == ',') { @@ -1103,7 +1111,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if ((state->cursor < state->end) && (*state->cursor == '}')) { state->cursor++; - return PUSH(json_decode_object(state, config, 0)); + return json_push_value(state, config, json_decode_object(state, config, 0)); } else { state->current_nesting++; if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { @@ -1132,7 +1140,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; state->current_nesting--; long count = state->stack->head - stack_head; - return PUSH(json_decode_object(state, config, count)); + return json_push_value(state, config, json_decode_object(state, config, count)); } if (*state->cursor == ',') { @@ -1220,6 +1228,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } + else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } else if (key == sym_create_id) { config->create_id = RTEST(val) ? val : Qfalse; } else if (key == sym_object_class) { config->object_class = RTEST(val) ? val : Qfalse; } else if (key == sym_array_class) { config->array_class = RTEST(val) ? val : Qfalse; } @@ -1396,6 +1405,7 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts) static void JSON_ParserConfig_mark(void *ptr) { JSON_ParserConfig *config = ptr; + rb_gc_mark(config->on_load_proc); rb_gc_mark(config->create_id); rb_gc_mark(config->object_class); rb_gc_mark(config->array_class); @@ -1468,6 +1478,7 @@ void Init_parser(void) sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma")); sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); sym_freeze = ID2SYM(rb_intern("freeze")); + sym_on_load = ID2SYM(rb_intern("on_load")); sym_create_additions = ID2SYM(rb_intern("create_additions")); sym_create_id = ID2SYM(rb_intern("create_id")); sym_object_class = ID2SYM(rb_intern("object_class")); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 83251fc8d..a0b88349b 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -17,6 +17,7 @@ import org.jruby.RubyHash; import org.jruby.RubyInteger; import org.jruby.RubyObject; +import org.jruby.RubyProc; import org.jruby.RubyString; import org.jruby.anno.JRubyMethod; import org.jruby.exceptions.JumpException; @@ -58,6 +59,7 @@ public class ParserConfig extends RubyObject { private boolean allowTrailingComma; private boolean symbolizeNames; private boolean freeze; + private RubyProc onLoadProc; private RubyClass objectClass; private RubyClass arrayClass; private RubyClass decimalClass; @@ -181,6 +183,7 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); this.freeze = opts.getBool("freeze", false); + this.onLoadProc = opts.getProc("on_load"); this.createId = opts.getString("create_id", getCreateId(context)); IRubyObject additions = opts.get("create_additions"); @@ -216,6 +219,14 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { return this; } + public IRubyObject onLoad(ThreadContext context, IRubyObject object) { + if (onLoadProc == null) { + return object; + } else { + return onLoadProc.call(context, object); + } + } + /** * Checks the given string's encoding. If a non-UTF-8 encoding is detected, * a converted copy is returned. @@ -311,11 +322,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 337 "ParserConfig.rl" +// line 348 "ParserConfig.rl" -// line 319 "ParserConfig.java" +// line 330 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -429,7 +440,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 443 "ParserConfig.rl" +// line 454 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -437,14 +448,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 441 "ParserConfig.java" +// line 452 "ParserConfig.java" { cs = JSON_value_start; } -// line 450 "ParserConfig.rl" +// line 461 "ParserConfig.rl" -// line 448 "ParserConfig.java" +// line 459 "ParserConfig.java" { int _klen; int _trans = 0; @@ -470,13 +481,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 428 "ParserConfig.rl" +// line 439 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 480 "ParserConfig.java" +// line 491 "ParserConfig.java" } } @@ -539,25 +550,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 345 "ParserConfig.rl" +// line 356 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 348 "ParserConfig.rl" +// line 359 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 351 "ParserConfig.rl" +// line 362 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 354 "ParserConfig.rl" +// line 365 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -567,7 +578,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 361 "ParserConfig.rl" +// line 372 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -577,7 +588,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 368 "ParserConfig.rl" +// line 379 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -606,7 +617,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 394 "ParserConfig.rl" +// line 405 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -619,7 +630,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 404 "ParserConfig.rl" +// line 415 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -634,7 +645,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 416 "ParserConfig.rl" +// line 427 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -648,7 +659,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 652 "ParserConfig.java" +// line 663 "ParserConfig.java" } } } @@ -668,7 +679,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 451 "ParserConfig.rl" +// line 462 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -681,7 +692,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 685 "ParserConfig.java" +// line 696 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -780,7 +791,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 473 "ParserConfig.rl" +// line 484 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -790,22 +801,22 @@ void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { return; } RubyInteger number = createInteger(context, p, new_p); - res.update(number, new_p + 1); + res.update(config.onLoad(context, number), new_p + 1); } int parseIntegerInternal(int p, int pe) { int cs; -// line 801 "ParserConfig.java" +// line 812 "ParserConfig.java" { cs = JSON_integer_start; } -// line 489 "ParserConfig.rl" +// line 500 "ParserConfig.rl" int memo = p; -// line 809 "ParserConfig.java" +// line 820 "ParserConfig.java" { int _klen; int _trans = 0; @@ -886,13 +897,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 467 "ParserConfig.rl" +// line 478 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 896 "ParserConfig.java" +// line 907 "ParserConfig.java" } } } @@ -912,7 +923,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 491 "ParserConfig.rl" +// line 502 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -932,7 +943,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 936 "ParserConfig.java" +// line 947 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1034,7 +1045,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 524 "ParserConfig.rl" +// line 535 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1046,22 +1057,22 @@ void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { final ByteList num = absSubSequence(p, new_p); IRubyObject number = config.decimalFactory.apply(context, num); - res.update(number, new_p + 1); + res.update(config.onLoad(context, number), new_p + 1); } int parseFloatInternal(int p, int pe) { int cs; -// line 1057 "ParserConfig.java" +// line 1068 "ParserConfig.java" { cs = JSON_float_start; } -// line 542 "ParserConfig.rl" +// line 553 "ParserConfig.rl" int memo = p; -// line 1065 "ParserConfig.java" +// line 1076 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1142,13 +1153,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 515 "ParserConfig.rl" +// line 526 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1152 "ParserConfig.java" +// line 1163 "ParserConfig.java" } } } @@ -1168,7 +1179,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 544 "ParserConfig.rl" +// line 555 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1178,7 +1189,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1182 "ParserConfig.java" +// line 1193 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1280,7 +1291,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 583 "ParserConfig.rl" +// line 594 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1288,15 +1299,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1292 "ParserConfig.java" +// line 1303 "ParserConfig.java" { cs = JSON_string_start; } -// line 590 "ParserConfig.rl" +// line 601 "ParserConfig.rl" int memo = p; -// line 1300 "ParserConfig.java" +// line 1311 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1377,7 +1388,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 558 "ParserConfig.rl" +// line 569 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1392,13 +1403,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 571 "ParserConfig.rl" +// line 582 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1402 "ParserConfig.java" +// line 1413 "ParserConfig.java" } } } @@ -1418,7 +1429,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 592 "ParserConfig.rl" +// line 603 "ParserConfig.rl" if (config.createAdditions) { RubyHash matchString = config.match_string; @@ -1448,9 +1459,9 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) string.setFrozen(true); string = context.runtime.freezeAndDedupString(string); } - res.update(string, p + 1); + res.update(config.onLoad(context, string), p + 1); } else { - res.update(result, p + 1); + res.update(config.onLoad(context, result), p + 1); } } else { res.update(null, p + 1); @@ -1458,7 +1469,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1462 "ParserConfig.java" +// line 1473 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1625,7 +1636,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 669 "ParserConfig.rl" +// line 680 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1645,14 +1656,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { } -// line 1649 "ParserConfig.java" +// line 1660 "ParserConfig.java" { cs = JSON_array_start; } -// line 688 "ParserConfig.rl" +// line 699 "ParserConfig.rl" -// line 1656 "ParserConfig.java" +// line 1667 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1695,7 +1706,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 636 "ParserConfig.rl" +// line 647 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1765,7 +1776,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 638 "ParserConfig.rl" +// line 649 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1782,13 +1793,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 653 "ParserConfig.rl" +// line 664 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1792 "ParserConfig.java" +// line 1803 "ParserConfig.java" } } } @@ -1808,17 +1819,17 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 689 "ParserConfig.rl" +// line 700 "ParserConfig.rl" if (cs >= JSON_array_first_final) { - res.update(result, p + 1); + res.update(config.onLoad(context, result), p + 1); } else { throw unexpectedToken(context, p, pe); } } -// line 1822 "ParserConfig.java" +// line 1833 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1995,7 +2006,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 750 "ParserConfig.rl" +// line 761 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -2020,14 +2031,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { } -// line 2024 "ParserConfig.java" +// line 2035 "ParserConfig.java" { cs = JSON_object_start; } -// line 774 "ParserConfig.rl" +// line 785 "ParserConfig.rl" -// line 2031 "ParserConfig.java" +// line 2042 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2070,7 +2081,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 703 "ParserConfig.rl" +// line 714 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2140,7 +2151,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 705 "ParserConfig.rl" +// line 716 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2157,7 +2168,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 720 "ParserConfig.rl" +// line 731 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2175,13 +2186,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 736 "ParserConfig.rl" +// line 747 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2185 "ParserConfig.java" +// line 2196 "ParserConfig.java" } } } @@ -2201,7 +2212,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 775 "ParserConfig.rl" +// line 786 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2232,11 +2243,11 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } } } - res.update(returnedResult, p + 1); + res.update(config.onLoad(context, returnedResult), p + 1); } -// line 2240 "ParserConfig.java" +// line 2251 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2339,7 +2350,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 828 "ParserConfig.rl" +// line 839 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2349,16 +2360,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2353 "ParserConfig.java" +// line 2364 "ParserConfig.java" { cs = JSON_start; } -// line 837 "ParserConfig.rl" +// line 848 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2362 "ParserConfig.java" +// line 2373 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2439,7 +2450,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 814 "ParserConfig.rl" +// line 825 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2451,7 +2462,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2455 "ParserConfig.java" +// line 2466 "ParserConfig.java" } } } @@ -2471,7 +2482,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 840 "ParserConfig.rl" +// line 851 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 8688a1016..652a27955 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -15,6 +15,7 @@ import org.jruby.RubyFloat; import org.jruby.RubyHash; import org.jruby.RubyInteger; import org.jruby.RubyObject; +import org.jruby.RubyProc; import org.jruby.RubyString; import org.jruby.anno.JRubyMethod; import org.jruby.exceptions.JumpException; @@ -56,6 +57,7 @@ public class ParserConfig extends RubyObject { private boolean allowTrailingComma; private boolean symbolizeNames; private boolean freeze; + private RubyProc onLoadProc; private RubyClass objectClass; private RubyClass arrayClass; private RubyClass decimalClass; @@ -179,6 +181,7 @@ public class ParserConfig extends RubyObject { this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); this.freeze = opts.getBool("freeze", false); + this.onLoadProc = opts.getProc("on_load"); this.createId = opts.getString("create_id", getCreateId(context)); IRubyObject additions = opts.get("create_additions"); @@ -214,6 +217,14 @@ public class ParserConfig extends RubyObject { return this; } + public IRubyObject onLoad(ThreadContext context, IRubyObject object) { + if (onLoadProc == null) { + return object; + } else { + return onLoadProc.call(context, object); + } + } + /** * Checks the given string's encoding. If a non-UTF-8 encoding is detected, * a converted copy is returned. @@ -479,7 +490,7 @@ public class ParserConfig extends RubyObject { return; } RubyInteger number = createInteger(context, p, new_p); - res.update(number, new_p + 1); + res.update(config.onLoad(context, number), new_p + 1); } int parseIntegerInternal(int p, int pe) { @@ -532,7 +543,7 @@ public class ParserConfig extends RubyObject { final ByteList num = absSubSequence(p, new_p); IRubyObject number = config.decimalFactory.apply(context, num); - res.update(number, new_p + 1); + res.update(config.onLoad(context, number), new_p + 1); } int parseFloatInternal(int p, int pe) { @@ -618,9 +629,9 @@ public class ParserConfig extends RubyObject { string.setFrozen(true); string = context.runtime.freezeAndDedupString(string); } - res.update(string, p + 1); + res.update(config.onLoad(context, string), p + 1); } else { - res.update(result, p + 1); + res.update(config.onLoad(context, result), p + 1); } } else { res.update(null, p + 1); @@ -688,7 +699,7 @@ public class ParserConfig extends RubyObject { %% write exec; if (cs >= JSON_array_first_final) { - res.update(result, p + 1); + res.update(config.onLoad(context, result), p + 1); } else { throw unexpectedToken(context, p, pe); } @@ -802,7 +813,7 @@ public class ParserConfig extends RubyObject { } } } - res.update(returnedResult, p + 1); + res.update(config.onLoad(context, returnedResult), p + 1); } %%{ diff --git a/lib/json/common.rb b/lib/json/common.rb index 9e9d6fb3e..17c7e3e24 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -739,23 +739,13 @@ def load(source, proc = nil, options = nil) if opts[:allow_blank] && (source.nil? || source.empty?) source = 'null' end - result = parse(source, opts) - recurse_proc(result, &proc) if proc - result - end - # Recursively calls passed _Proc_ if the parsed data structure is an _Array_ or _Hash_ - def recurse_proc(result, &proc) # :nodoc: - case result - when Array - result.each { |x| recurse_proc x, &proc } - proc.call result - when Hash - result.each { |x, y| recurse_proc x, &proc; recurse_proc y, &proc } - proc.call result - else - proc.call result + if proc + opts = opts.dup + opts[:on_load] = proc.to_proc end + + parse(source, opts) end # Sets or returns the default options for the JSON.dump method. diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 96f7ccdf9..3f4f27439 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -110,7 +110,7 @@ def test_load def test_load_with_proc visited = [] - JSON.load('{"foo": [1, 2, 3], "bar": {"baz": "plop"}}', proc { |o| visited << JSON.dump(o) }) + JSON.load('{"foo": [1, 2, 3], "bar": {"baz": "plop"}}', proc { |o| visited << JSON.dump(o); o }) expected = [ '"foo"', From f411ddf1ceaf1ea82c8470c94da522919a4fb3c9 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 12:25:08 +0100 Subject: [PATCH 026/506] Move `create_addtions` logic in Ruby. By leveraging the `on_load` callback we can move all this logic out of the parser. Which mean we no longer have to duplicate that logic in both parser and that we'll later be able to extract it entirely from the gem. --- ext/json/ext/parser/extconf.rb | 1 - ext/json/ext/parser/parser.c | 151 +--------------- java/src/json/ext/ParserConfig.java | 255 +++++++++------------------- java/src/json/ext/ParserConfig.rl | 109 +----------- lib/json/common.rb | 127 +++++++++++++- test/json/json_addition_test.rb | 7 +- 6 files changed, 213 insertions(+), 437 deletions(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index a8e21aed4..09c963778 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -4,7 +4,6 @@ have_func("rb_enc_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby -have_func("rb_category_warn", "ruby.h") # Missing on TruffleRuby have_func("strnlen", "string.h") # Missing on Solaris 10 append_cflags("-std=c99") diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b08fadd7d..f20769a36 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -31,28 +31,15 @@ typedef unsigned char _Bool; static VALUE mJSON, eNestingError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; -static ID i_json_creatable_p, i_json_create, i_create_id, - i_chr, i_deep_const_get, i_match, i_aset, i_aref, +static ID i_chr, i_aset, i_aref, i_leftshift, i_new, i_try_convert, i_uminus, i_encode; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze, - sym_create_additions, sym_create_id, sym_object_class, sym_array_class, - sym_decimal_class, sym_match_string, sym_on_load; + sym_decimal_class, sym_on_load; static int binary_encindex; static int utf8_encindex; -#ifdef HAVE_RB_CATEGORY_WARN -# define json_deprecated(message) rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, message) -#else -# define json_deprecated(message) rb_warn(message) -#endif - -static const char deprecated_create_additions_warning[] = - "JSON.load implicit support for `create_additions: true` is deprecated " - "and will be removed in 3.0, use JSON.unsafe_load or explicitly " - "pass `create_additions: true`"; - #ifndef HAVE_RB_HASH_BULK_INSERT // For TruffleRuby void @@ -445,20 +432,14 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) typedef struct JSON_ParserStruct { VALUE on_load_proc; - VALUE create_id; - VALUE object_class; - VALUE array_class; VALUE decimal_class; ID decimal_method_id; - VALUE match_string; int max_nesting; bool allow_nan; bool allow_trailing_comma; bool parsing_name; bool symbolize_names; bool freeze; - bool create_additions; - bool deprecated_create_additions; } JSON_ParserConfig; typedef struct JSON_ParserStateStruct { @@ -770,18 +751,7 @@ static VALUE json_decode_float(JSON_ParserConfig *config, const char *start, con static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count) { - VALUE array; - if (RB_UNLIKELY(config->array_class)) { - array = rb_class_new_instance(0, 0, config->array_class); - VALUE *items = rvalue_stack_peek(state->stack, count); - long index; - for (index = 0; index < count; index++) { - rb_funcall(array, i_leftshift, 1, items[index]); - } - } else { - array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count)); - } - + VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count)); rvalue_stack_pop(state->stack, count); if (config->freeze) { @@ -791,52 +761,13 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig return array; } -static bool json_obj_creatable_p(VALUE klass) -{ - if (rb_respond_to(klass, i_json_creatable_p)) { - return RTEST(rb_funcall(klass, i_json_creatable_p, 0)); - } else { - return rb_respond_to(klass, i_json_create); - } -} - static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, long count) { - VALUE object; - if (RB_UNLIKELY(config->object_class)) { - object = rb_class_new_instance(0, 0, config->object_class); - long index = 0; - VALUE *items = rvalue_stack_peek(state->stack, count); - while (index < count) { - VALUE name = items[index++]; - VALUE value = items[index++]; - rb_funcall(object, i_aset, 2, name, value); - } - } else { - object = rb_hash_new_capa(count); - rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object); - } + VALUE object = rb_hash_new_capa(count); + rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object); rvalue_stack_pop(state->stack, count); - if (RB_UNLIKELY(config->create_additions)) { - VALUE klassname; - if (config->object_class) { - klassname = rb_funcall(object, i_aref, 1, config->create_id); - } else { - klassname = rb_hash_aref(object, config->create_id); - } - if (!NIL_P(klassname)) { - VALUE klass = rb_funcall(mJSON, i_deep_const_get, 1, klassname); - if (json_obj_creatable_p(klass)) { - if (config->deprecated_create_additions) { - json_deprecated(deprecated_create_additions_warning); - } - object = rb_funcall(klass, i_json_create, 1, object); - } - } - } - if (config->freeze) { RB_OBJ_FREEZE(object); } @@ -844,17 +775,6 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi return object; } -static int match_i(VALUE regexp, VALUE klass, VALUE memo) -{ - if (regexp == Qundef) return ST_STOP; - if (json_obj_creatable_p(klass) && - RTEST(rb_funcall(regexp, i_match, 1, rb_ary_entry(memo, 0)))) { - rb_ary_push(memo, klass); - return ST_STOP; - } - return ST_CONTINUE; -} - static inline VALUE json_decode_string(JSON_ParserState *state, JSON_ParserConfig *config, const char *start, const char *end, bool escaped, bool is_name) { VALUE string; @@ -866,17 +786,6 @@ static inline VALUE json_decode_string(JSON_ParserState *state, JSON_ParserConfi string = json_string_fastpath(state, start, end, is_name, intern, symbolize); } - if (RB_UNLIKELY(config->create_additions && RTEST(config->match_string))) { - VALUE klass; - VALUE memo = rb_ary_new2(2); - rb_ary_push(memo, string); - rb_hash_foreach(config->match_string, match_i, memo); - klass = rb_ary_entry(memo, 1); - if (RTEST(klass)) { - string = rb_funcall(klass, i_json_create, 1, string); - } - } - return string; } @@ -1229,10 +1138,6 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } - else if (key == sym_create_id) { config->create_id = RTEST(val) ? val : Qfalse; } - else if (key == sym_object_class) { config->object_class = RTEST(val) ? val : Qfalse; } - else if (key == sym_array_class) { config->array_class = RTEST(val) ? val : Qfalse; } - else if (key == sym_match_string) { config->match_string = RTEST(val) ? val : Qfalse; } else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { @@ -1262,15 +1167,6 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) } } } - else if (key == sym_create_additions) { - if (NIL_P(val)) { - config->create_additions = true; - config->deprecated_create_additions = true; - } else { - config->create_additions = RTEST(val); - config->deprecated_create_additions = false; - } - } return ST_CONTINUE; } @@ -1285,16 +1181,6 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts) // We assume in most cases few keys are set so it's faster to go over // the provided keys than to check all possible keys. rb_hash_foreach(opts, parser_config_init_i, (VALUE)config); - - if (config->symbolize_names && config->create_additions) { - rb_raise(rb_eArgError, - "options :symbolize_names and :create_additions cannot be " - " used in conjunction"); - } - - if (config->create_additions && !config->create_id) { - config->create_id = rb_funcall(mJSON, i_create_id, 0); - } } } @@ -1319,15 +1205,6 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts) * (keys) in a JSON object. Otherwise strings are returned, which is * also the default. It's not possible to use this option in * conjunction with the *create_additions* option. - * * *create_additions*: If set to false, the Parser doesn't create - * additions even if a matching class and create_id was found. This option - * defaults to false. - * * *object_class*: Defaults to Hash. If another type is provided, it will be used - * instead of Hash to represent JSON objects. The type must respond to - * +new+ without arguments, and return an object that respond to +[]=+. - * * *array_class*: Defaults to Array If another type is provided, it will be used - * instead of Hash to represent JSON arrays. The type must respond to - * +new+ without arguments, and return an object that respond to +<<+. * * *decimal_class*: Specifies which class to use instead of the default * (Float) when parsing decimal numbers. This class must accept a single * string argument in its constructor. @@ -1338,11 +1215,7 @@ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) parser_config_init(config, opts); - RB_OBJ_WRITTEN(self, Qundef, config->create_id); - RB_OBJ_WRITTEN(self, Qundef, config->object_class); - RB_OBJ_WRITTEN(self, Qundef, config->array_class); RB_OBJ_WRITTEN(self, Qundef, config->decimal_class); - RB_OBJ_WRITTEN(self, Qundef, config->match_string); return self; } @@ -1406,11 +1279,7 @@ static void JSON_ParserConfig_mark(void *ptr) { JSON_ParserConfig *config = ptr; rb_gc_mark(config->on_load_proc); - rb_gc_mark(config->create_id); - rb_gc_mark(config->object_class); - rb_gc_mark(config->array_class); rb_gc_mark(config->decimal_class); - rb_gc_mark(config->match_string); } static void JSON_ParserConfig_free(void *ptr) @@ -1479,19 +1348,9 @@ void Init_parser(void) sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); sym_freeze = ID2SYM(rb_intern("freeze")); sym_on_load = ID2SYM(rb_intern("on_load")); - sym_create_additions = ID2SYM(rb_intern("create_additions")); - sym_create_id = ID2SYM(rb_intern("create_id")); - sym_object_class = ID2SYM(rb_intern("object_class")); - sym_array_class = ID2SYM(rb_intern("array_class")); sym_decimal_class = ID2SYM(rb_intern("decimal_class")); - sym_match_string = ID2SYM(rb_intern("match_string")); - i_create_id = rb_intern("create_id"); - i_json_creatable_p = rb_intern("json_creatable?"); - i_json_create = rb_intern("json_create"); i_chr = rb_intern("chr"); - i_match = rb_intern("match"); - i_deep_const_get = rb_intern("deep_const_get"); i_aset = rb_intern("[]="); i_aref = rb_intern("[]"); i_leftshift = rb_intern("<<"); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index a0b88349b..692b3dfaa 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -51,17 +51,12 @@ */ public class ParserConfig extends RubyObject { private final RuntimeInfo info; - private RubyString createId; - private boolean createAdditions; - private boolean deprecatedCreateAdditions; private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; - private RubyClass objectClass; - private RubyClass arrayClass; private RubyClass decimalClass; BiFunction decimalFactory; private RubyHash match_string; @@ -184,25 +179,8 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.symbolizeNames = opts.getBool("symbolize_names", false); this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); - this.createId = opts.getString("create_id", getCreateId(context)); - IRubyObject additions = opts.get("create_additions"); - this.createAdditions = false; - this.deprecatedCreateAdditions = false; - - if (additions != null) { - if (additions.isNil()) { - this.createAdditions = true; - this.deprecatedCreateAdditions = true; - } else { - this.createAdditions = opts.getBool("create_additions", false); - } - } - - this.objectClass = opts.getClass("object_class", runtime.getHash()); - this.arrayClass = opts.getClass("array_class", runtime.getArray()); this.decimalClass = opts.getClass("decimal_class", null); - this.match_string = opts.getHash("match_string"); if (decimalClass == null) { this.decimalFactory = this::createFloat; @@ -212,10 +190,6 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.decimalFactory = this::createCustomDecimal; } - if(symbolizeNames && createAdditions) { - throw runtime.newArgumentError("options :symbolize_names and :create_additions cannot be used in conjunction"); - } - return this; } @@ -277,14 +251,6 @@ private IRubyObject createCustomDecimal(final ThreadContext context, final ByteL return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); } - private static boolean isObjCreateable(final ThreadContext context, IRubyObject klass) { - if (klass.respondsTo("json_creatable?")) { - return klass.callMethod(context, "json_creatable?").isTrue(); - } else { - return klass.respondsTo("json_create"); - } - } - /** * A string parsing session. * @@ -322,11 +288,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 348 "ParserConfig.rl" +// line 314 "ParserConfig.rl" -// line 330 "ParserConfig.java" +// line 296 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -440,7 +406,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 454 "ParserConfig.rl" +// line 420 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -448,14 +414,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 452 "ParserConfig.java" +// line 418 "ParserConfig.java" { cs = JSON_value_start; } -// line 461 "ParserConfig.rl" +// line 427 "ParserConfig.rl" -// line 459 "ParserConfig.java" +// line 425 "ParserConfig.java" { int _klen; int _trans = 0; @@ -481,13 +447,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 439 "ParserConfig.rl" +// line 405 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 491 "ParserConfig.java" +// line 457 "ParserConfig.java" } } @@ -550,25 +516,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 356 "ParserConfig.rl" +// line 322 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 359 "ParserConfig.rl" +// line 325 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 362 "ParserConfig.rl" +// line 328 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 365 "ParserConfig.rl" +// line 331 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -578,7 +544,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 372 "ParserConfig.rl" +// line 338 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -588,7 +554,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 379 "ParserConfig.rl" +// line 345 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -617,7 +583,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 405 "ParserConfig.rl" +// line 371 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -630,7 +596,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 415 "ParserConfig.rl" +// line 381 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -645,7 +611,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 427 "ParserConfig.rl" +// line 393 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -659,7 +625,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 663 "ParserConfig.java" +// line 629 "ParserConfig.java" } } } @@ -679,7 +645,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 462 "ParserConfig.rl" +// line 428 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -692,7 +658,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 696 "ParserConfig.java" +// line 662 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -791,7 +757,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 484 "ParserConfig.rl" +// line 450 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -808,15 +774,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 812 "ParserConfig.java" +// line 778 "ParserConfig.java" { cs = JSON_integer_start; } -// line 500 "ParserConfig.rl" +// line 466 "ParserConfig.rl" int memo = p; -// line 820 "ParserConfig.java" +// line 786 "ParserConfig.java" { int _klen; int _trans = 0; @@ -897,13 +863,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 478 "ParserConfig.rl" +// line 444 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 907 "ParserConfig.java" +// line 873 "ParserConfig.java" } } } @@ -923,7 +889,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 502 "ParserConfig.rl" +// line 468 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -943,7 +909,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 947 "ParserConfig.java" +// line 913 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1045,7 +1011,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 535 "ParserConfig.rl" +// line 501 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1064,15 +1030,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1068 "ParserConfig.java" +// line 1034 "ParserConfig.java" { cs = JSON_float_start; } -// line 553 "ParserConfig.rl" +// line 519 "ParserConfig.rl" int memo = p; -// line 1076 "ParserConfig.java" +// line 1042 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1153,13 +1119,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 526 "ParserConfig.rl" +// line 492 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1163 "ParserConfig.java" +// line 1129 "ParserConfig.java" } } } @@ -1179,7 +1145,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 555 "ParserConfig.rl" +// line 521 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1189,7 +1155,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1193 "ParserConfig.java" +// line 1159 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1291,7 +1257,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 594 "ParserConfig.rl" +// line 560 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1299,15 +1265,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1303 "ParserConfig.java" +// line 1269 "ParserConfig.java" { cs = JSON_string_start; } -// line 601 "ParserConfig.rl" +// line 567 "ParserConfig.rl" int memo = p; -// line 1311 "ParserConfig.java" +// line 1277 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1388,7 +1354,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 569 "ParserConfig.rl" +// line 535 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1403,13 +1369,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 582 "ParserConfig.rl" +// line 548 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1413 "ParserConfig.java" +// line 1379 "ParserConfig.java" } } } @@ -1429,26 +1395,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 603 "ParserConfig.rl" - - if (config.createAdditions) { - RubyHash matchString = config.match_string; - if (matchString != null) { - final IRubyObject[] memoArray = { result, null }; - try { - matchString.visitAll(context, MATCH_VISITOR, memoArray); - } catch (JumpException e) { } - if (memoArray[1] != null) { - RubyClass klass = (RubyClass) memoArray[1]; - if (isObjCreateable(context, klass)) { - if (config.deprecatedCreateAdditions) { - context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); - } - result = klass.callMethod(context, "json_create", result); - } - } - } - } +// line 569 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1469,7 +1416,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1473 "ParserConfig.java" +// line 1420 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1636,7 +1583,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 680 "ParserConfig.rl" +// line 623 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1647,23 +1594,17 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { "nesting of " + currentNesting + " is too deep"); } - IRubyObject result; - if (config.arrayClass == context.runtime.getArray()) { - result = RubyArray.newArray(context.runtime); - } else { - result = config.arrayClass.newInstance(context, - IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); - } + IRubyObject result = RubyArray.newArray(context.runtime); -// line 1660 "ParserConfig.java" +// line 1601 "ParserConfig.java" { cs = JSON_array_start; } -// line 699 "ParserConfig.rl" +// line 636 "ParserConfig.rl" -// line 1667 "ParserConfig.java" +// line 1608 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1706,7 +1647,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 647 "ParserConfig.rl" +// line 594 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1776,30 +1717,26 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 649 "ParserConfig.rl" +// line 596 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } else { - if (config.arrayClass == context.runtime.getArray()) { - ((RubyArray)result).append(res.result); - } else { - result.callMethod(context, "<<", res.result); - } + ((RubyArray)result).append(res.result); {p = (( res.p))-1;} } } break; case 1: -// line 664 "ParserConfig.rl" +// line 607 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1803 "ParserConfig.java" +// line 1740 "ParserConfig.java" } } } @@ -1819,7 +1756,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 700 "ParserConfig.rl" +// line 637 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1829,7 +1766,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1833 "ParserConfig.java" +// line 1770 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -2006,13 +1943,12 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 761 "ParserConfig.rl" +// line 694 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject lastName = null; - boolean objectDefault = true; if (config.maxNesting > 0 && currentNesting > config.maxNesting) { throw newException(context, Utils.M_NESTING_ERROR, @@ -2021,24 +1957,17 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { // this is guaranteed to be a RubyHash due to the earlier // allocator test at OptionsReader#getClass - IRubyObject result; - if (config.objectClass == context.runtime.getHash()) { - result = RubyHash.newHash(context.runtime); - } else { - objectDefault = false; - result = config.objectClass.newInstance(context, - IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); - } + IRubyObject result = RubyHash.newHash(context.runtime); -// line 2035 "ParserConfig.java" +// line 1964 "ParserConfig.java" { cs = JSON_object_start; } -// line 785 "ParserConfig.rl" +// line 710 "ParserConfig.rl" -// line 2042 "ParserConfig.java" +// line 1971 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2081,7 +2010,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 714 "ParserConfig.rl" +// line 651 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2151,24 +2080,20 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 716 "ParserConfig.rl" +// line 653 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } else { - if (config.objectClass == context.runtime.getHash()) { - ((RubyHash)result).op_aset(context, lastName, res.result); - } else { - Helpers.invoke(context, result, "[]=", lastName, res.result); - } + ((RubyHash)result).op_aset(context, lastName, res.result); {p = (( res.p))-1;} } } break; case 1: -// line 731 "ParserConfig.rl" +// line 664 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2186,13 +2111,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 747 "ParserConfig.rl" +// line 680 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2196 "ParserConfig.java" +// line 2121 "ParserConfig.java" } } } @@ -2212,42 +2137,18 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 786 "ParserConfig.rl" +// line 711 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); return; } - IRubyObject returnedResult = result; - - // attempt to de-serialize object - if (config.createAdditions) { - IRubyObject vKlassName; - if (objectDefault) { - vKlassName = ((RubyHash)result).op_aref(context, config.createId); - } else { - vKlassName = result.callMethod(context, "[]", config.createId); - } - - if (!vKlassName.isNil()) { - // might throw ArgumentError, we let it propagate - IRubyObject klass = config.info.jsonModule.get(). - callMethod(context, "deep_const_get", vKlassName); - if (isObjCreateable(context, klass)) { - if (config.deprecatedCreateAdditions) { - context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); - } - - returnedResult = klass.callMethod(context, "json_create", result); - } - } - } - res.update(config.onLoad(context, returnedResult), p + 1); + res.update(config.onLoad(context, result), p + 1); } -// line 2251 "ParserConfig.java" +// line 2152 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2350,7 +2251,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 839 "ParserConfig.rl" +// line 740 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2360,16 +2261,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2364 "ParserConfig.java" +// line 2265 "ParserConfig.java" { cs = JSON_start; } -// line 848 "ParserConfig.rl" +// line 749 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2373 "ParserConfig.java" +// line 2274 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2450,7 +2351,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 825 "ParserConfig.rl" +// line 726 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2462,7 +2363,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2466 "ParserConfig.java" +// line 2367 "ParserConfig.java" } } } @@ -2482,7 +2383,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 851 "ParserConfig.rl" +// line 752 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 652a27955..ce0d9438e 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -49,17 +49,12 @@ import static org.jruby.util.ConvertDouble.DoubleConverter; */ public class ParserConfig extends RubyObject { private final RuntimeInfo info; - private RubyString createId; - private boolean createAdditions; - private boolean deprecatedCreateAdditions; private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; - private RubyClass objectClass; - private RubyClass arrayClass; private RubyClass decimalClass; BiFunction decimalFactory; private RubyHash match_string; @@ -182,25 +177,8 @@ public class ParserConfig extends RubyObject { this.symbolizeNames = opts.getBool("symbolize_names", false); this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); - this.createId = opts.getString("create_id", getCreateId(context)); - IRubyObject additions = opts.get("create_additions"); - this.createAdditions = false; - this.deprecatedCreateAdditions = false; - - if (additions != null) { - if (additions.isNil()) { - this.createAdditions = true; - this.deprecatedCreateAdditions = true; - } else { - this.createAdditions = opts.getBool("create_additions", false); - } - } - - this.objectClass = opts.getClass("object_class", runtime.getHash()); - this.arrayClass = opts.getClass("array_class", runtime.getArray()); this.decimalClass = opts.getClass("decimal_class", null); - this.match_string = opts.getHash("match_string"); if (decimalClass == null) { this.decimalFactory = this::createFloat; @@ -210,10 +188,6 @@ public class ParserConfig extends RubyObject { this.decimalFactory = this::createCustomDecimal; } - if(symbolizeNames && createAdditions) { - throw runtime.newArgumentError("options :symbolize_names and :create_additions cannot be used in conjunction"); - } - return this; } @@ -275,14 +249,6 @@ public class ParserConfig extends RubyObject { return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); } - private static boolean isObjCreateable(final ThreadContext context, IRubyObject klass) { - if (klass.respondsTo("json_creatable?")) { - return klass.callMethod(context, "json_creatable?").isTrue(); - } else { - return klass.respondsTo("json_create"); - } - } - /** * A string parsing session. * @@ -601,25 +567,6 @@ public class ParserConfig extends RubyObject { int memo = p; %% write exec; - if (config.createAdditions) { - RubyHash matchString = config.match_string; - if (matchString != null) { - final IRubyObject[] memoArray = { result, null }; - try { - matchString.visitAll(context, MATCH_VISITOR, memoArray); - } catch (JumpException e) { } - if (memoArray[1] != null) { - RubyClass klass = (RubyClass) memoArray[1]; - if (isObjCreateable(context, klass)) { - if (config.deprecatedCreateAdditions) { - context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); - } - result = klass.callMethod(context, "json_create", result); - } - } - } - } - if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { RubyString string = (RubyString)result; @@ -652,11 +599,7 @@ public class ParserConfig extends RubyObject { fhold; fbreak; } else { - if (config.arrayClass == context.runtime.getArray()) { - ((RubyArray)result).append(res.result); - } else { - result.callMethod(context, "<<", res.result); - } + ((RubyArray)result).append(res.result); fexec res.p; } } @@ -687,13 +630,7 @@ public class ParserConfig extends RubyObject { "nesting of " + currentNesting + " is too deep"); } - IRubyObject result; - if (config.arrayClass == context.runtime.getArray()) { - result = RubyArray.newArray(context.runtime); - } else { - result = config.arrayClass.newInstance(context, - IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); - } + IRubyObject result = RubyArray.newArray(context.runtime); %% write init; %% write exec; @@ -719,11 +656,7 @@ public class ParserConfig extends RubyObject { fhold; fbreak; } else { - if (config.objectClass == context.runtime.getHash()) { - ((RubyHash)result).op_aset(context, lastName, res.result); - } else { - Helpers.invoke(context, result, "[]=", lastName, res.result); - } + ((RubyHash)result).op_aset(context, lastName, res.result); fexec res.p; } } @@ -763,7 +696,6 @@ public class ParserConfig extends RubyObject { void parseObject(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject lastName = null; - boolean objectDefault = true; if (config.maxNesting > 0 && currentNesting > config.maxNesting) { throw newException(context, Utils.M_NESTING_ERROR, @@ -772,14 +704,7 @@ public class ParserConfig extends RubyObject { // this is guaranteed to be a RubyHash due to the earlier // allocator test at OptionsReader#getClass - IRubyObject result; - if (config.objectClass == context.runtime.getHash()) { - result = RubyHash.newHash(context.runtime); - } else { - objectDefault = false; - result = config.objectClass.newInstance(context, - IRubyObject.NULL_ARRAY, Block.NULL_BLOCK); - } + IRubyObject result = RubyHash.newHash(context.runtime); %% write init; %% write exec; @@ -789,31 +714,7 @@ public class ParserConfig extends RubyObject { return; } - IRubyObject returnedResult = result; - - // attempt to de-serialize object - if (config.createAdditions) { - IRubyObject vKlassName; - if (objectDefault) { - vKlassName = ((RubyHash)result).op_aref(context, config.createId); - } else { - vKlassName = result.callMethod(context, "[]", config.createId); - } - - if (!vKlassName.isNil()) { - // might throw ArgumentError, we let it propagate - IRubyObject klass = config.info.jsonModule.get(). - callMethod(context, "deep_const_get", vKlassName); - if (isObjCreateable(context, klass)) { - if (config.deprecatedCreateAdditions) { - context.runtime.getWarnings().warn("JSON.load implicit support for `create_additions: true` is deprecated and will be removed in 3.0, use JSON.unsafe_load or explicitly pass `create_additions: true`"); - } - - returnedResult = klass.callMethod(context, "json_create", result); - } - } - } - res.update(config.onLoad(context, returnedResult), p + 1); + res.update(config.onLoad(context, result), p + 1); } %%{ diff --git a/lib/json/common.rb b/lib/json/common.rb index 17c7e3e24..a542bdb36 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -5,6 +5,112 @@ module JSON autoload :GenericObject, 'json/generic_object' + module ParserOptions # :nodoc: + class << self + def prepare(opts) + if opts[:object_class] || opts[:array_class] + opts = opts.dup + on_load = opts[:on_load] + + on_load = object_class_proc(opts[:object_class], on_load) if opts[:object_class] + on_load = array_class_proc(opts[:array_class], on_load) if opts[:array_class] + opts[:on_load] = on_load + end + + if opts.fetch(:create_additions, false) != false + opts = create_additions_proc(opts) + end + + opts + end + + private + + def object_class_proc(object_class, on_load) + ->(obj) do + if Hash === obj + object = object_class.new + obj.each { |k, v| object[k] = v } + obj = object + end + on_load.nil? ? obj : on_load.call(obj) + end + end + + def array_class_proc(array_class, on_load) + ->(obj) do + if Array === obj + array = array_class.new + obj.each { |v| array << v } + obj = array + end + on_load.nil? ? obj : on_load.call(obj) + end + end + + # TODO: exact :create_additions support to another gem for version 3.0 + def create_additions_proc(opts) + if opts[:symbolize_names] + raise ArgumentError, "options :symbolize_names and :create_additions cannot be used in conjunction" + end + + opts = opts.dup + create_additions = opts.fetch(:create_additions, false) + on_load = opts[:on_load] + object_class = opts[:object_class] || Hash + + opts[:on_load] = ->(object) do + case object + when String + opts[:match_string]&.each do |pattern, klass| + if match = pattern.match(object) + create_additions_warning if create_additions.nil? + object = klass.json_create(object) + break + end + end + when object_class + if opts[:create_additions] != false + if class_name = object[JSON.create_id] + klass = JSON.deep_const_get(class_name) + if (klass.respond_to?(:json_creatable?) && klass.json_creatable?) || klass.respond_to?(:json_create) + create_additions_warning if create_additions.nil? + object = klass.json_create(object) + end + end + end + end + + on_load.nil? ? object : on_load.call(object) + end + + opts + end + + GEM_ROOT = File.expand_path("../../../", __FILE__) + "/" + def create_additions_warning + message = "JSON.load implicit support for `create_additions: true` is deprecated " \ + "and will be removed in 3.0, use JSON.unsafe_load or explicitly " \ + "pass `create_additions: true`" + + uplevel = 4 + caller_locations(uplevel, 10).each do |frame| + if frame.path.nil? || frame.path.start_with?(GEM_ROOT) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") + uplevel += 1 + else + break + end + end + + if RUBY_VERSION >= "3.0" + warn(message, uplevel: uplevel - 1, category: :deprecated) + else + warn(message, uplevel: uplevel - 1) + end + end + end + end + class << self # :call-seq: # JSON[object] -> new_array or new_string @@ -236,9 +342,16 @@ def to_json(state = nil, *) # JSON.parse('') # def parse(source, opts = nil) + opts = ParserOptions.prepare(opts) unless opts.nil? Parser.parse(source, opts) end + PARSE_L_OPTIONS = { + max_nesting: false, + allow_nan: true, + }.freeze + private_constant :PARSE_L_OPTIONS + # :call-seq: # JSON.parse!(source, opts) -> object # @@ -251,12 +364,11 @@ def parse(source, opts = nil) # which disables checking for nesting depth. # - Option +allow_nan+, if not provided, defaults to +true+. def parse!(source, opts = nil) - options = { - :max_nesting => false, - :allow_nan => true - } - options.merge!(opts) if opts - Parser.new(source, options).parse + if opts.nil? + parse(source, PARSE_L_OPTIONS) + else + parse(source, PARSE_L_OPTIONS.merge(opts)) + end end # :call-seq: @@ -859,10 +971,9 @@ def initialize(options = nil, &as_json) options[:strict] = true end options[:as_json] = as_json if as_json - options[:create_additions] = false unless options.key?(:create_additions) @state = State.new(options).freeze - @parser_config = Ext::Parser::Config.new(options) + @parser_config = Ext::Parser::Config.new(ParserOptions.prepare(options)) end # call-seq: diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index 4d8d18687..a98e06df8 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -151,7 +151,12 @@ def test_core end def test_deprecated_load_create_additions - assert_deprecated_warning(/use JSON\.unsafe_load/) do + pattern = /json_addition_test\.rb.*use JSON\.unsafe_load/ + if RUBY_ENGINE == 'truffleruby' + pattern = /use JSON\.unsafe_load/ + end + + assert_deprecated_warning(pattern) do JSON.load(JSON.dump(Time.now)) end end From eeb4efc8c1acbbee8fd7c7fca1d6231f983c1687 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 13:31:14 +0100 Subject: [PATCH 027/506] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1bde9fc3b..9dbc253ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Optimize Integer generation to be ~1.8x faster. +* Optimize Float generation to be ~10x faster. * Fix `JSON.load` proc argument to substitute the parsed object with the return value. This better match `Marshal.load` behavior. * Deprecate `JSON.fast_generate` (it's not any faster, so pointless). From 1b16a82980745b5990c4895bbb2711cd6227603a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 27 Mar 2025 15:07:01 +0100 Subject: [PATCH 028/506] Add missing assert_deprecated_warning --- test/json/json_common_interface_test.rb | 4 +- test/json/json_generator_test.rb | 58 +++++++++++++------------ 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 3f4f27439..9c1255130 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -212,7 +212,9 @@ def test_load_file_with_bad_default_external_encoding end def test_deprecated_dump_default_options - assert JSON.dump_default_options + assert_deprecated_warning(/dump_default_options/) do + JSON.dump_default_options + end end private diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 05006ebd0..f87006ac7 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -136,15 +136,17 @@ def test_generate_custom end def test_fast_generate - json = fast_generate(@hash) - assert_equal(parse(@json2), parse(json)) - parsed_json = parse(json) - assert_equal(@hash, parsed_json) - json = fast_generate({1=>2}) - assert_equal('{"1":2}', json) - parsed_json = parse(json) - assert_equal({"1"=>2}, parsed_json) - assert_equal '666', fast_generate(666) + assert_deprecated_warning(/fast_generate/) do + json = fast_generate(@hash) + assert_equal(parse(@json2), parse(json)) + parsed_json = parse(json) + assert_equal(@hash, parsed_json) + json = fast_generate({1=>2}) + assert_equal('{"1":2}', json) + parsed_json = parse(json) + assert_equal({"1"=>2}, parsed_json) + assert_equal '666', fast_generate(666) + end end def test_own_state @@ -219,24 +221,26 @@ def test_state_defaults end def test_allow_nan - error = assert_raise(GeneratorError) { generate([JSON::NaN]) } - assert_same JSON::NaN, error.invalid_object - assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::NaN]) } - assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) } - assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true) - error = assert_raise(GeneratorError) { generate([JSON::Infinity]) } - assert_same JSON::Infinity, error.invalid_object - assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::Infinity]) } - assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) } - assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true) - error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } - assert_same JSON::MinusInfinity, error.invalid_object - assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::MinusInfinity]) } - assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) } - assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true) + assert_deprecated_warning(/fast_generate/) do + error = assert_raise(GeneratorError) { generate([JSON::NaN]) } + assert_same JSON::NaN, error.invalid_object + assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true) + assert_raise(GeneratorError) { fast_generate([JSON::NaN]) } + assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) } + assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true) + error = assert_raise(GeneratorError) { generate([JSON::Infinity]) } + assert_same JSON::Infinity, error.invalid_object + assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true) + assert_raise(GeneratorError) { fast_generate([JSON::Infinity]) } + assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) } + assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true) + error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } + assert_same JSON::MinusInfinity, error.invalid_object + assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true) + assert_raise(GeneratorError) { fast_generate([JSON::MinusInfinity]) } + assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) } + assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true) + end end def test_depth From 0d4fc669edc01f9bd5447c723cda05ca8de56ba8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 28 Mar 2025 08:53:25 +0100 Subject: [PATCH 029/506] Remove caller location assertions Fix: https://github.com/ruby/json/pull/781 Tests are ran a bit differently in ruby/ruby and this assertion breaks. --- test/json/json_addition_test.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index a98e06df8..4d8d18687 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -151,12 +151,7 @@ def test_core end def test_deprecated_load_create_additions - pattern = /json_addition_test\.rb.*use JSON\.unsafe_load/ - if RUBY_ENGINE == 'truffleruby' - pattern = /use JSON\.unsafe_load/ - end - - assert_deprecated_warning(pattern) do + assert_deprecated_warning(/use JSON\.unsafe_load/) do JSON.load(JSON.dump(Time.now)) end end From 97f47d1cfdb93103c6381acf36c11ee7c024a8f3 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 7 Apr 2025 19:44:45 +0900 Subject: [PATCH 030/506] Suppress -Wunknown-pragmas warnings --- ext/json/ext/vendor/jeaiii-ltoa.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/vendor/jeaiii-ltoa.h b/ext/json/ext/vendor/jeaiii-ltoa.h index c8e2a1288..1a1337ad7 100644 --- a/ext/json/ext/vendor/jeaiii-ltoa.h +++ b/ext/json/ext/vendor/jeaiii-ltoa.h @@ -35,11 +35,13 @@ SOFTWARE. typedef uint_fast32_t u32_t; typedef uint_fast64_t u64_t; +#if defined __clang__ #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wshorten-64-to-32" - +#elif defined __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-braces" +#endif #define u32(x) ((u32_t)(x)) #define u64(x) ((u64_t)(x)) @@ -270,8 +272,10 @@ jeaiii_ultoa(char *b, u64_t n) #undef u64 #undef COPY +#if defined __clang__ #pragma clang diagnostic pop +#elif defined __GNUC__ #pragma GCC diagnostic pop +#endif #endif // JEAIII_TO_TEXT_H_ - From 97f2924d0aa8c81f91475fae420d4ea0b504510d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 7 Apr 2025 13:10:16 +0200 Subject: [PATCH 031/506] Cleanup jeaiii-ltoa to not need pragmas --- ext/json/ext/vendor/jeaiii-ltoa.h | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/ext/json/ext/vendor/jeaiii-ltoa.h b/ext/json/ext/vendor/jeaiii-ltoa.h index 1a1337ad7..ba4f497fc 100644 --- a/ext/json/ext/vendor/jeaiii-ltoa.h +++ b/ext/json/ext/vendor/jeaiii-ltoa.h @@ -35,14 +35,6 @@ SOFTWARE. typedef uint_fast32_t u32_t; typedef uint_fast64_t u64_t; -#if defined __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshorten-64-to-32" -#elif defined __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmissing-braces" -#endif - #define u32(x) ((u32_t)(x)) #define u64(x) ((u64_t)(x)) @@ -93,7 +85,7 @@ jeaiii_ultoa(char *b, u64_t n) if (n < u32(1e6)) { if (n < u32(1e4)) { - u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * n; + u32_t f0 = u32((10 * (1 << 24) / 1e3 + 1) * n); COPY(b, digits_fd[f0 >> 24]); b -= n < u32(1e3); @@ -163,7 +155,7 @@ jeaiii_ultoa(char *b, u64_t n) } else if (u < u32(1e6)) { if (u < u32(1e4)) { - u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; + u32_t f0 = u32((10 * (1 << 24) / 1e3 + 1) * u); COPY(b, digits_fd[f0 >> 24]); b -= u < u32(1e3); @@ -228,7 +220,7 @@ jeaiii_ultoa(char *b, u64_t n) b += 2; } else { - u32_t f0 = u32(10 * (1 << 24) / 1e3 + 1) * u; + u32_t f0 = u32((10 * (1 << 24) / 1e3 + 1) * u); COPY(b, digits_fd[f0 >> 24]); b -= u < u32(1e3); @@ -272,10 +264,4 @@ jeaiii_ultoa(char *b, u64_t n) #undef u64 #undef COPY -#if defined __clang__ -#pragma clang diagnostic pop -#elif defined __GNUC__ -#pragma GCC diagnostic pop -#endif - #endif // JEAIII_TO_TEXT_H_ From ff8e18392d18395d29fbf9bdebceeacb70ed99be Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 15 Apr 2025 13:34:03 +0900 Subject: [PATCH 032/506] Don't create custom NaN and Infinity objects. --- lib/json/common.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index a542bdb36..91c89ef23 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -48,7 +48,7 @@ def array_class_proc(array_class, on_load) end end - # TODO: exact :create_additions support to another gem for version 3.0 + # TODO: exctract :create_additions support to another gem for version 3.0 def create_additions_proc(opts) if opts[:symbolize_names] raise ArgumentError, "options :symbolize_names and :create_additions cannot be used in conjunction" @@ -220,9 +220,9 @@ def self.create_id Thread.current[:"JSON.create_id"] || 'json_class' end - NaN = 0.0/0 + NaN = Float::NAN - Infinity = 1.0/0 + Infinity = Float::INFINITY MinusInfinity = -Infinity From 3e7e661ef520642a64b581e8799b5312a8770923 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 15 Apr 2025 13:35:50 +0900 Subject: [PATCH 033/506] Get rid of JSONError.wrap --- lib/json/common.rb | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 91c89ef23..8f34f1f05 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -227,13 +227,7 @@ def self.create_id MinusInfinity = -Infinity # The base exception for JSON errors. - class JSONError < StandardError - def self.wrap(exception) - obj = new("Wrapped(#{exception.class}): #{exception.message.inspect}") - obj.set_backtrace exception.backtrace - obj - end - end + class JSONError < StandardError; end # This exception is raised if a parser error occurs. class ParserError < JSONError; end From 27155b650066f941fe886419f63c43826188aa86 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 15 Apr 2025 13:43:02 +0900 Subject: [PATCH 034/506] Deprecate Kernel#j and Kernel#jj --- CHANGES.md | 2 ++ lib/json/common.rb | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 9dbc253ed..05b2ea157 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,8 @@ * Deprecate `JSON.load_default_options`. * Deprecate `JSON.unsafe_load_default_options`. * Deprecate `JSON.dump_default_options`. +* Deprecate `Kernel#j` +* Deprecate `Kernel#jj` * Remove outdated `JSON.iconv`. * Remove `Class#json_creatable?` monkey patch. * Remove deprecated `JSON.restore` method. diff --git a/lib/json/common.rb b/lib/json/common.rb index 8f34f1f05..37427534e 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1005,6 +1005,12 @@ module ::Kernel # Outputs _objs_ to STDOUT as JSON strings in the shortest form, that is in # one line. def j(*objs) + if RUBY_VERSION >= "3.0" + warn "Kernel#j is deprecated and will be removed in json 3.0.0", uplevel: 1, category: :deprecated + else + warn "Kernel#j is deprecated and will be removed in json 3.0.0", uplevel: 1 + end + objs.each do |obj| puts JSON::generate(obj, :allow_nan => true, :max_nesting => false) end @@ -1014,6 +1020,12 @@ def j(*objs) # Outputs _objs_ to STDOUT as JSON strings in a pretty format, with # indentation and over many lines. def jj(*objs) + if RUBY_VERSION >= "3.0" + warn "Kernel#jj is deprecated and will be removed in json 3.0.0", uplevel: 1, category: :deprecated + else + warn "Kernel#jj is deprecated and will be removed in json 3.0.0", uplevel: 1 + end + objs.each do |obj| puts JSON::pretty_generate(obj, :allow_nan => true, :max_nesting => false) end From 573ebdca899d45cf3de757e5e91f7fe199d3fe2b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 15 Apr 2025 13:43:23 +0900 Subject: [PATCH 035/506] Cleanup Kernel#JSON --- lib/json/common.rb | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 37427534e..b5fbfcf05 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -123,7 +123,7 @@ class << self # Otherwise, calls JSON.generate with +object+ and +opts+ (see method #generate): # ruby = [0, 1, nil] # JSON[ruby] # => '[0,1,null]' - def [](object, opts = {}) + def [](object, opts = nil) if object.is_a?(String) return JSON.parse(object, opts) elsif object.respond_to?(:to_str) @@ -1038,16 +1038,7 @@ def jj(*objs) # # The _opts_ argument is passed through to generate/parse respectively. See # generate and parse for their documentation. - def JSON(object, *args) - if object.is_a?(String) - return JSON.parse(object, args.first) - elsif object.respond_to?(:to_str) - str = object.to_str - if str.is_a?(String) - return JSON.parse(object.to_str, args.first) - end - end - - JSON.generate(object, args.first) + def JSON(object, opts = nil) + JSON[object, opts] end end From 92beca803238ac956508da9ccb94fdc609e2209b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 17 Apr 2025 07:37:14 +0900 Subject: [PATCH 036/506] Get rid of unused CircularDatastructure class --- lib/json/common.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index b5fbfcf05..254ed0920 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -236,10 +236,6 @@ class ParserError < JSONError; end # deep. class NestingError < ParserError; end - # :stopdoc: - class CircularDatastructure < NestingError; end - # :startdoc: - # This exception is raised if a generator or unparser error occurs. class GeneratorError < JSONError attr_reader :invalid_object From a6949f8656abd533451491ba9f5c851f5cb5d070 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Apr 2025 10:11:28 +0200 Subject: [PATCH 037/506] Release 2.11.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 05b2ea157..ece8e0bb1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +### 2025-04-24 (2.11.0) + * Optimize Integer generation to be ~1.8x faster. * Optimize Float generation to be ~10x faster. * Fix `JSON.load` proc argument to substitute the parsed object with the return value. diff --git a/lib/json/version.rb b/lib/json/version.rb index 9c21f18b3..59adffc4d 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.10.2' + VERSION = '2.11.0' end From 0dee9bdad937a74a5667b7781c10e6aab22abee2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Apr 2025 14:59:52 +0200 Subject: [PATCH 038/506] Bring back the numerous deprecated alias This time with explicit deprecation warnings. --- CHANGES.md | 6 ++++++ lib/json/common.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ece8e0bb1..26fed9e41 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Changes +### 2025-04-24 (2.11.1) + +* Add back `JSON.restore`, `JSON.unparse`, `JSON.fast_unparse` and `JSON.pretty_unparse`. + These were deprecated 16 years ago, but never emited warnings, only undocumented, so are + still used by a few gems. + ### 2025-04-24 (2.11.0) * Optimize Integer generation to be ~1.8x faster. diff --git a/lib/json/common.rb b/lib/json/common.rb index 254ed0920..77d024dea 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -919,6 +919,50 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end end + # :stopdoc: + # All these were meant to be deprecated circa 2009, but were just set as undocumented + # so usage still exist in the wild. + def unparse(...) + if RUBY_VERSION >= "3.0" + warn "JSON.unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated + else + warn "JSON.unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 + end + generate(...) + end + module_function :unparse + + def fast_unparse(...) + if RUBY_VERSION >= "3.0" + warn "JSON.fast_unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated + else + warn "JSON.fast_unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 + end + generate(...) + end + module_function :fast_unparse + + def pretty_unparse(...) + if RUBY_VERSION >= "3.0" + warn "JSON.pretty_unparse is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1, category: :deprecated + else + warn "JSON.pretty_unparse is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1 + end + pretty_generate(...) + end + module_function :fast_unparse + + def restore(...) + if RUBY_VERSION >= "3.0" + warn "JSON.restore is deprecated and will be removed in json 3.0.0, just use JSON.load", uplevel: 1, category: :deprecated + else + warn "JSON.restore is deprecated and will be removed in json 3.0.0, just use JSON.load", uplevel: 1 + end + load(...) + end + module_function :restore + # :startdoc: + # JSON::Coder holds a parser and generator configuration. # # module MyApp From 84443e881df0a2a5f86abf95aaec1e43d5fba957 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Apr 2025 15:09:27 +0200 Subject: [PATCH 039/506] Release 2.11.1 --- lib/json/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/version.rb b/lib/json/version.rb index 59adffc4d..1fa83370b 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.11.0' + VERSION = '2.11.1' end From 123121bba2bd1bd01acdb2a778e84caa881613c2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Apr 2025 18:21:23 +0200 Subject: [PATCH 040/506] Bring back JSON::PRETTY_STATE_PROTOTYPE with a deprecation Fix: https://github.com/ruby/json/issues/788 `multi_json` rely on it, even though it was never documented as public API. Bringing it back as a method so it can emit a deprecation warning. --- CHANGES.md | 3 +++ lib/json/common.rb | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 26fed9e41..1ad2ea69b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,8 @@ # Changes +* Add back `JSON::PRETTY_STATE_PROTOTYPE`. This constant was private API but is used by popular gems like `multi_json`. + It now emits a deprecation warning. + ### 2025-04-24 (2.11.1) * Add back `JSON.restore`, `JSON.unparse`, `JSON.fast_unparse` and `JSON.pretty_unparse`. diff --git a/lib/json/common.rb b/lib/json/common.rb index 77d024dea..98bc0ac85 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -961,6 +961,24 @@ def restore(...) load(...) end module_function :restore + + class << self + private + + def const_missing(const_name) + case const_name + when :PRETTY_STATE_PROTOTYPE + if RUBY_VERSION >= "3.0" + warn "JSON::PRETTY_STATE_PROTOTYPE is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1, category: :deprecated + else + warn "JSON::PRETTY_STATE_PROTOTYPE is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1 + end + state.new(PRETTY_GENERATE_OPTIONS) + else + super + end + end + end # :startdoc: # JSON::Coder holds a parser and generator configuration. From c985e8c6ea1b1bfe1e571edce0a32066254c2a11 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Apr 2025 18:34:06 +0200 Subject: [PATCH 041/506] Release 2.11.2 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1ad2ea69b..f9044d640 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +### 2025-04-24 (2.11.2) + * Add back `JSON::PRETTY_STATE_PROTOTYPE`. This constant was private API but is used by popular gems like `multi_json`. It now emits a deprecation warning. diff --git a/lib/json/version.rb b/lib/json/version.rb index 1fa83370b..63c532d69 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.11.1' + VERSION = '2.11.2' end From 2ffa4ea46bb9dd4ac19eb2ea20c17119b6065feb Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 25 Apr 2025 07:35:05 +0200 Subject: [PATCH 042/506] Stop caching the generator state pointer Fix: https://github.com/ruby/json/issues/790 If we end up calling something that spills the state on the heap, the pointer we received is outdated and may be out of sync. --- CHANGES.md | 2 + ext/json/ext/generator/generator.c | 179 ++++++++++++------------ test/json/json_common_interface_test.rb | 24 ++++ 3 files changed, 116 insertions(+), 89 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f9044d640..ff3f79778 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Fix a regression in `JSON.pretty_generate` that could cause indentation to be off once some `#to_json` has been called. + ### 2025-04-24 (2.11.2) * Add back `JSON::PRETTY_STATE_PROTOTYPE`. This constant was private API but is used by popular gems like `multi_json`. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 428f5e21f..981bf96bb 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -45,7 +45,7 @@ static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_n struct generate_json_data; -typedef void (*generator_func)(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); +typedef void (*generator_func)(FBuffer *buffer, struct generate_json_data *data, VALUE obj); struct generate_json_data { FBuffer *buffer; @@ -57,20 +57,20 @@ struct generate_json_data { static VALUE cState_from_state_s(VALUE self, VALUE opts); static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io); -static void generate_json(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); +static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj); #ifdef RUBY_INTEGER_UNIFICATION -static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); +static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj); #endif -static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); -static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj); +static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj); +static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static int usascii_encindex, utf8_encindex, binary_encindex; @@ -802,12 +802,12 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) int j; if (arg->iter > 0) fbuffer_append_char(buffer, ','); - if (RB_UNLIKELY(state->object_nl)) { - fbuffer_append_str(buffer, state->object_nl); + if (RB_UNLIKELY(data->state->object_nl)) { + fbuffer_append_str(buffer, data->state->object_nl); } - if (RB_UNLIKELY(state->indent)) { + if (RB_UNLIKELY(data->state->indent)) { for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, state->indent); + fbuffer_append_str(buffer, data->state->indent); } } @@ -829,21 +829,22 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) } if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) { - generate_json_string(buffer, data, state, key_to_s); + generate_json_string(buffer, data, key_to_s); } else { - generate_json(buffer, data, state, key_to_s); + generate_json(buffer, data, key_to_s); } - if (RB_UNLIKELY(state->space_before)) fbuffer_append_str(buffer, state->space_before); + if (RB_UNLIKELY(state->space_before)) fbuffer_append_str(buffer, data->state->space_before); fbuffer_append_char(buffer, ':'); - if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, state->space); - generate_json(buffer, data, state, val); + if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, data->state->space); + generate_json(buffer, data, val); arg->iter++; return ST_CONTINUE; } -static inline long increase_depth(JSON_Generator_State *state) +static inline long increase_depth(struct generate_json_data *data) { + JSON_Generator_State *state = data->state; long depth = ++state->depth; if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) { rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); @@ -851,14 +852,14 @@ static inline long increase_depth(JSON_Generator_State *state) return depth; } -static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { int j; - long depth = increase_depth(state); + long depth = increase_depth(data); if (RHASH_SIZE(obj) == 0) { fbuffer_append(buffer, "{}", 2); - --state->depth; + --data->state->depth; return; } @@ -870,49 +871,49 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat }; rb_hash_foreach(obj, json_object_i, (VALUE)&arg); - depth = --state->depth; - if (RB_UNLIKELY(state->object_nl)) { - fbuffer_append_str(buffer, state->object_nl); - if (RB_UNLIKELY(state->indent)) { + depth = --data->state->depth; + if (RB_UNLIKELY(data->state->object_nl)) { + fbuffer_append_str(buffer, data->state->object_nl); + if (RB_UNLIKELY(data->state->indent)) { for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, state->indent); + fbuffer_append_str(buffer, data->state->indent); } } } fbuffer_append_char(buffer, '}'); } -static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { int i, j; - long depth = increase_depth(state); + long depth = increase_depth(data); if (RARRAY_LEN(obj) == 0) { fbuffer_append(buffer, "[]", 2); - --state->depth; + --data->state->depth; return; } fbuffer_append_char(buffer, '['); - if (RB_UNLIKELY(state->array_nl)) fbuffer_append_str(buffer, state->array_nl); + if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); for(i = 0; i < RARRAY_LEN(obj); i++) { if (i > 0) { fbuffer_append_char(buffer, ','); - if (RB_UNLIKELY(state->array_nl)) fbuffer_append_str(buffer, state->array_nl); + if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); } - if (RB_UNLIKELY(state->indent)) { + if (RB_UNLIKELY(data->state->indent)) { for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, state->indent); + fbuffer_append_str(buffer, data->state->indent); } } - generate_json(buffer, data, state, RARRAY_AREF(obj, i)); + generate_json(buffer, data, RARRAY_AREF(obj, i)); } - state->depth = --depth; - if (RB_UNLIKELY(state->array_nl)) { - fbuffer_append_str(buffer, state->array_nl); - if (RB_UNLIKELY(state->indent)) { + data->state->depth = --depth; + if (RB_UNLIKELY(data->state->array_nl)) { + fbuffer_append_str(buffer, data->state->array_nl); + if (RB_UNLIKELY(data->state->indent)) { for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, state->indent); + fbuffer_append_str(buffer, data->state->indent); } } } @@ -961,7 +962,7 @@ static inline VALUE ensure_valid_encoding(VALUE str) return str; } -static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { obj = ensure_valid_encoding(obj); @@ -977,9 +978,9 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat switch(rb_enc_str_coderange(obj)) { case ENC_CODERANGE_7BIT: case ENC_CODERANGE_VALID: - if (RB_UNLIKELY(state->ascii_only)) { - convert_UTF8_to_ASCII_only_JSON(&search, state->script_safe ? script_safe_escape_table : ascii_only_escape_table); - } else if (RB_UNLIKELY(state->script_safe)) { + if (RB_UNLIKELY(data->state->ascii_only)) { + convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table); + } else if (RB_UNLIKELY(data->state->script_safe)) { convert_UTF8_to_script_safe_JSON(&search); } else { convert_UTF8_to_JSON(&search); @@ -992,7 +993,7 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat fbuffer_append_char(buffer, '"'); } -static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { VALUE tmp; if (rb_respond_to(obj, i_to_json)) { @@ -1002,68 +1003,68 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d } else { tmp = rb_funcall(obj, i_to_s, 0); Check_Type(tmp, T_STRING); - generate_json_string(buffer, data, state, tmp); + generate_json_string(buffer, data, tmp); } } -static inline void generate_json_symbol(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static inline void generate_json_symbol(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { - if (state->strict) { - generate_json_string(buffer, data, state, rb_sym2str(obj)); + if (data->state->strict) { + generate_json_string(buffer, data, rb_sym2str(obj)); } else { - generate_json_fallback(buffer, data, state, obj); + generate_json_fallback(buffer, data, obj); } } -static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append(buffer, "null", 4); } -static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append(buffer, "false", 5); } -static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append(buffer, "true", 4); } -static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append_long(buffer, FIX2LONG(obj)); } -static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { VALUE tmp = rb_funcall(obj, i_to_s, 0); fbuffer_append_str(buffer, tmp); } #ifdef RUBY_INTEGER_UNIFICATION -static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { if (FIXNUM_P(obj)) - generate_json_fixnum(buffer, data, state, obj); + generate_json_fixnum(buffer, data, obj); else - generate_json_bignum(buffer, data, state, obj); + generate_json_bignum(buffer, data, obj); } #endif -static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { double value = RFLOAT_VALUE(obj); - char allow_nan = state->allow_nan; + char allow_nan = data->state->allow_nan; if (isinf(value) || isnan(value)) { /* for NaN and Infinity values we either raise an error or rely on Float#to_s. */ if (!allow_nan) { - if (state->strict && state->as_json) { - VALUE casted_obj = rb_proc_call_with_block(state->as_json, 1, &obj, Qnil); + if (data->state->strict && data->state->as_json) { + VALUE casted_obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil); if (casted_obj != obj) { - increase_depth(state); - generate_json(buffer, data, state, casted_obj); - state->depth--; + increase_depth(data); + generate_json(buffer, data, casted_obj); + data->state->depth--; return; } } @@ -1089,30 +1090,30 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data buffer->len += len; } -static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { VALUE fragment = RSTRUCT_GET(obj, 0); Check_Type(fragment, T_STRING); fbuffer_append_str(buffer, fragment); } -static void generate_json(FBuffer *buffer, struct generate_json_data *data, JSON_Generator_State *state, VALUE obj) +static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { bool as_json_called = false; start: if (obj == Qnil) { - generate_json_null(buffer, data, state, obj); + generate_json_null(buffer, data, obj); } else if (obj == Qfalse) { - generate_json_false(buffer, data, state, obj); + generate_json_false(buffer, data, obj); } else if (obj == Qtrue) { - generate_json_true(buffer, data, state, obj); + generate_json_true(buffer, data, obj); } else if (RB_SPECIAL_CONST_P(obj)) { if (RB_FIXNUM_P(obj)) { - generate_json_fixnum(buffer, data, state, obj); + generate_json_fixnum(buffer, data, obj); } else if (RB_FLONUM_P(obj)) { - generate_json_float(buffer, data, state, obj); + generate_json_float(buffer, data, obj); } else if (RB_STATIC_SYM_P(obj)) { - generate_json_symbol(buffer, data, state, obj); + generate_json_symbol(buffer, data, obj); } else { goto general; } @@ -1120,43 +1121,43 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, JSON VALUE klass = RBASIC_CLASS(obj); switch (RB_BUILTIN_TYPE(obj)) { case T_BIGNUM: - generate_json_bignum(buffer, data, state, obj); + generate_json_bignum(buffer, data, obj); break; case T_HASH: if (klass != rb_cHash) goto general; - generate_json_object(buffer, data, state, obj); + generate_json_object(buffer, data, obj); break; case T_ARRAY: if (klass != rb_cArray) goto general; - generate_json_array(buffer, data, state, obj); + generate_json_array(buffer, data, obj); break; case T_STRING: if (klass != rb_cString) goto general; - generate_json_string(buffer, data, state, obj); + generate_json_string(buffer, data, obj); break; case T_SYMBOL: - generate_json_symbol(buffer, data, state, obj); + generate_json_symbol(buffer, data, obj); break; case T_FLOAT: if (klass != rb_cFloat) goto general; - generate_json_float(buffer, data, state, obj); + generate_json_float(buffer, data, obj); break; case T_STRUCT: if (klass != cFragment) goto general; - generate_json_fragment(buffer, data, state, obj); + generate_json_fragment(buffer, data, obj); break; default: general: - if (state->strict) { - if (RTEST(state->as_json) && !as_json_called) { - obj = rb_proc_call_with_block(state->as_json, 1, &obj, Qnil); + if (data->state->strict) { + if (RTEST(data->state->as_json) && !as_json_called) { + obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil); as_json_called = true; goto start; } else { raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj)); } } else { - generate_json_fallback(buffer, data, state, obj); + generate_json_fallback(buffer, data, obj); } } } @@ -1166,7 +1167,7 @@ static VALUE generate_json_try(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; - data->func(data->buffer, data, data->state, data->obj); + data->func(data->buffer, data, data->obj); return Qnil; } diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 9c1255130..745400faa 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -91,6 +91,30 @@ def test_fast_generate def test_pretty_generate assert_equal "[\n 1,\n 2,\n 3\n]", JSON.pretty_generate([ 1, 2, 3 ]) + assert_equal <<~JSON.strip, JSON.pretty_generate({ a: { b: "f"}, c: "d"}) + { + "a": { + "b": "f" + }, + "c": "d" + } + JSON + + # Cause the state to be spilled on the heap. + o = Object.new + def o.to_s + "Object" + end + actual = JSON.pretty_generate({ a: { b: o}, c: "d", e: "f"}) + assert_equal <<~JSON.strip, actual + { + "a": { + "b": "Object" + }, + "c": "d", + "e": "f" + } + JSON end def test_load From 3e025f76d77e323b30f6f6d2d8d06e787d497a0c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 25 Apr 2025 08:02:43 +0200 Subject: [PATCH 043/506] Release 2.11.3 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ff3f79778..ea59edf7e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +### 2025-04-25 (2.11.3) + * Fix a regression in `JSON.pretty_generate` that could cause indentation to be off once some `#to_json` has been called. ### 2025-04-24 (2.11.2) diff --git a/lib/json/version.rb b/lib/json/version.rb index 63c532d69..c8744c973 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.11.2' + VERSION = '2.11.3' end From 12dc394d115d10168f647acb7218a922c0f5e7ad Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 27 Apr 2025 11:10:00 +0200 Subject: [PATCH 044/506] Handle non-string keys returning immediate values via `to_s` We can't directly call `RBASIC_CLASS` as the return value of `to_s` may be an immediate. --- CHANGES.md | 2 ++ ext/json/ext/generator/generator.c | 17 ++++++++++++++++- test/json/json_generator_test.rb | 16 ++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ea59edf7e..fc055710b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Handle non-string hash keys with broken `to_s` implementations. + ### 2025-04-25 (2.11.3) * Fix a regression in `JSON.pretty_generate` that could cause indentation to be off once some `#to_json` has been called. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 981bf96bb..ba1056643 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -789,6 +789,21 @@ struct hash_foreach_arg { int iter; }; +static VALUE +convert_string_subclass(VALUE key) +{ + VALUE key_to_s = rb_funcall(key, i_to_s, 0); + + if (RB_UNLIKELY(rb_type(key_to_s) != T_STRING)) { + VALUE cname = rb_obj_class(key); + rb_raise(rb_eTypeError, + "can't convert %"PRIsVALUE" to %s (%"PRIsVALUE"#%s gives %"PRIsVALUE")", + cname, "String", cname, "to_s", rb_obj_class(key_to_s)); + } + + return key_to_s; +} + static int json_object_i(VALUE key, VALUE val, VALUE _arg) { @@ -817,7 +832,7 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) { key_to_s = key; } else { - key_to_s = rb_funcall(key, i_to_s, 0); + key_to_s = convert_string_subclass(key); } break; case T_SYMBOL: diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f87006ac7..47c94bd39 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -605,6 +605,22 @@ def test_string_subclass_with_to_s assert_equal '{"JSONGeneratorTest::StringWithToS#to_s":1}', JSON.generate(StringWithToS.new => 1) end + def test_string_subclass_with_broken_to_s + klass = Class.new(String) do + def to_s + false + end + end + s = klass.new("test") + assert_equal '["test"]', JSON.generate([s]) + + omit("Can't figure out how to match behavior in java code") if RUBY_PLATFORM == "java" + + assert_raise TypeError do + JSON.generate(s => 1) + end + end + if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java" def test_valid_utf8_in_different_encoding utf8_string = "€™" From b14250f1daf7e246952d37110908c537debe3ca0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 27 Apr 2025 12:35:01 +0200 Subject: [PATCH 045/506] Use RB_TYPE_P --- ext/json/ext/generator/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index ba1056643..7159180c0 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -794,7 +794,7 @@ convert_string_subclass(VALUE key) { VALUE key_to_s = rb_funcall(key, i_to_s, 0); - if (RB_UNLIKELY(rb_type(key_to_s) != T_STRING)) { + if (RB_UNLIKELY(!RB_TYPE_P(key_to_s, T_STRING))) { VALUE cname = rb_obj_class(key); rb_raise(rb_eTypeError, "can't convert %"PRIsVALUE" to %s (%"PRIsVALUE"#%s gives %"PRIsVALUE")", From 49003523dab352fa1af7237b30f98dc3056cc90b Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Mon, 28 Apr 2025 07:57:10 -0500 Subject: [PATCH 046/506] Introduce ARM Neon and SSE2 SIMD. (#743) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See the pull request for the long development history: https://github.com/ruby/json/pull/743 ``` == Encoding activitypub.json (52595 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 2.913k i/100ms Calculating ------------------------------------- after 29.377k (± 2.0%) i/s (34.04 μs/i) - 148.563k in 5.059169s Comparison: before: 23314.1 i/s after: 29377.3 i/s - 1.26x faster == Encoding citm_catalog.json (500298 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 152.000 i/100ms Calculating ------------------------------------- after 1.569k (± 0.8%) i/s (637.49 μs/i) - 7.904k in 5.039001s Comparison: before: 1485.6 i/s after: 1568.7 i/s - 1.06x faster == Encoding twitter.json (466906 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 309.000 i/100ms Calculating ------------------------------------- after 3.115k (± 3.1%) i/s (321.01 μs/i) - 15.759k in 5.063776s Comparison: before: 2508.3 i/s after: 3115.2 i/s - 1.24x faster ``` --- .github/workflows/ci.yml | 31 +-- ext/json/ext/generator/extconf.rb | 31 +++ ext/json/ext/generator/generator.c | 366 ++++++++++++++++++++++++++++- ext/json/ext/generator/simd.h | 112 +++++++++ test/json/json_generator_test.rb | 90 ++++++- 5 files changed, 602 insertions(+), 28 deletions(-) create mode 100644 ext/json/ext/generator/simd.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9574d30c..b6f82ae90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,26 +14,29 @@ jobs: host: needs: ruby-versions - name: ${{ matrix.os }} ${{ matrix.ruby }} + name: ${{ matrix.os }} ${{ matrix.ruby }} ${{ matrix.env }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: - - ubuntu-latest - - macos-14 - - windows-latest + - ubuntu-latest + - macos-14 + - windows-latest ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }} + env: + - "" include: - - { os: ubuntu-24.04-arm, ruby: 3.4 } - - { os: macos-13, ruby: 3.4 } - - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI - - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 - - { os: macos-latest , ruby: truffleruby-head } - - { os: ubuntu-latest , ruby: truffleruby-head } + - { os: ubuntu-24.04-arm, ruby: 3.4 } + - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } + - { os: macos-13, ruby: 3.4 } + - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI + - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 + - { os: macos-latest , ruby: truffleruby-head } + - { os: ubuntu-latest , ruby: truffleruby-head } exclude: - - { os: windows-latest, ruby: jruby } - - { os: windows-latest, ruby: jruby-head } + - { os: windows-latest, ruby: jruby } + - { os: windows-latest, ruby: jruby-head } steps: - uses: actions/checkout@v4 @@ -49,9 +52,9 @@ jobs: bundle config --without benchmark bundle install - - run: rake compile + - run: rake compile ${{ matrix.env }} - - run: rake test JSON_COMPACT=1 + - run: rake test JSON_COMPACT=1 ${{ matrix.env }} - run: rake build diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 078068cf6..4fbeb5f33 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -6,5 +6,36 @@ else append_cflags("-std=c99") $defs << "-DJSON_GENERATOR" + + if enable_config('generator-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) + if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ + # Try to compile a small program using NEON instructions + if have_header('arm_neon.h') + have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') + #include + int main() { + uint8x16_t test = vdupq_n_u8(32); + return 0; + } + SRC + $defs.push("-DENABLE_SIMD") + end + end + + if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC', opt='-msse2') + #include + int main() { + __m128i test = _mm_set1_epi8(32); + return 0; + } + SRC + $defs.push("-DENABLE_SIMD") + end + + have_header('cpuid.h') + end + + create_header + create_makefile 'json/ext/generator' end diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 7159180c0..c09061728 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -5,6 +5,8 @@ #include #include +#include "simd.h" + /* ruby api and some helpers */ typedef struct JSON_Generator_StateStruct { @@ -109,12 +111,40 @@ typedef struct _search_state { const char *end; const char *cursor; FBuffer *buffer; + +#ifdef ENABLE_SIMD + const char *chunk_base; + const char *chunk_end; + bool has_matches; + +#ifdef HAVE_SIMD_NEON + uint64_t matches_mask; +#elif HAVE_SIMD_SSE2 + int matches_mask; +#else +#error "Unknown SIMD Implementation." +#endif /* HAVE_SIMD_NEON */ +#endif /* ENABLE_SIMD */ } search_state; -static inline void search_flush(search_state *search) -{ - fbuffer_append(search->buffer, search->cursor, search->ptr - search->cursor); - search->cursor = search->ptr; +#if (defined(__GNUC__ ) || defined(__clang__)) +#define FORCE_INLINE __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + +static inline FORCE_INLINE void search_flush(search_state *search) +{ + // Do not remove this conditional without profiling, specifically escape-heavy text. + // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush). + // For back-to-back characters that need to be escaped, specifcally for the SIMD code paths, this method + // will be called just before calling escape_UTF8_char_basic. There will be no characers to append for the + // consecutive characters that need to be escaped. While the fbuffer_append is a no-op if + // nothing needs to be flushed, we can save a few memory references with this conditional. + if (search->ptr > search->cursor) { + fbuffer_append(search->buffer, search->cursor, search->ptr - search->cursor); + search->cursor = search->ptr; + } } static const unsigned char escape_table_basic[256] = { @@ -130,6 +160,8 @@ static const unsigned char escape_table_basic[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; +static unsigned char (*search_escape_basic_impl)(search_state *); + static inline unsigned char search_escape_basic(search_state *search) { while (search->ptr < search->end) { @@ -144,7 +176,8 @@ static inline unsigned char search_escape_basic(search_state *search) return 0; } -static inline void escape_UTF8_char_basic(search_state *search) { +static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search) +{ const unsigned char ch = (unsigned char)*search->ptr; switch (ch) { case '"': fbuffer_append(search->buffer, "\\\"", 2); break; @@ -156,11 +189,15 @@ static inline void escape_UTF8_char_basic(search_state *search) { case '\r': fbuffer_append(search->buffer, "\\r", 2); break; case '\t': fbuffer_append(search->buffer, "\\t", 2); break; default: { - const char *hexdig = "0123456789abcdef"; - char scratch[6] = { '\\', 'u', '0', '0', 0, 0 }; - scratch[4] = hexdig[(ch >> 4) & 0xf]; - scratch[5] = hexdig[ch & 0xf]; - fbuffer_append(search->buffer, scratch, 6); + if (ch < ' ') { + const char *hexdig = "0123456789abcdef"; + char scratch[6] = { '\\', 'u', '0', '0', 0, 0 }; + scratch[4] = hexdig[(ch >> 4) & 0xf]; + scratch[5] = hexdig[ch & 0xf]; + fbuffer_append(search->buffer, scratch, 6); + } else { + fbuffer_append_char(search->buffer, ch); + } break; } } @@ -186,12 +223,13 @@ static inline void escape_UTF8_char_basic(search_state *search) { */ static inline void convert_UTF8_to_JSON(search_state *search) { - while (search_escape_basic(search)) { + while (search_escape_basic_impl(search)) { escape_UTF8_char_basic(search); } } -static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) { +static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) +{ const unsigned char ch = (unsigned char)*search->ptr; switch (ch_len) { case 1: { @@ -227,6 +265,285 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) search->cursor = (search->ptr += ch_len); } +#ifdef ENABLE_SIMD + +static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) +{ + // Flush the buffer so everything up until the last 'len' characters are unflushed. + search_flush(search); + + FBuffer *buf = search->buffer; + fbuffer_inc_capa(buf, vec_len); + + char *s = (buf->ptr + buf->len); + + // Pad the buffer with dummy characters that won't need escaping. + // This seem wateful at first sight, but memset of vector length is very fast. + memset(s, 'X', vec_len); + + // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters + // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage. + MEMCPY(s, search->ptr, char, len); + + return s; +} + +#ifdef HAVE_SIMD_NEON + +static inline FORCE_INLINE unsigned char neon_next_match(search_state *search) +{ + uint64_t mask = search->matches_mask; + uint32_t index = trailing_zeros64(mask) >> 2; + + // It is assumed escape_UTF8_char_basic will only ever increase search->ptr by at most one character. + // If we want to use a similar approach for full escaping we'll need to ensure: + // search->chunk_base + index >= search->ptr + // However, since we know escape_UTF8_char_basic only increases search->ptr by one, if the next match + // is one byte after the previous match then: + // search->chunk_base + index == search->ptr + search->ptr = search->chunk_base + index; + mask &= mask - 1; + search->matches_mask = mask; + search_flush(search); + return 1; +} + +// See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon +static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches) +{ + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + return mask & 0x8888888888888888ull; +} + +static inline FORCE_INLINE uint64_t neon_rules_update(const char *ptr) +{ + uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); + + const uint8x16_t lower_bound = vdupq_n_u8(' '); + const uint8x16_t backslash = vdupq_n_u8('\\'); + const uint8x16_t dblquote = vdupq_n_u8('\"'); + + uint8x16_t too_low = vcltq_u8(chunk, lower_bound); + uint8x16_t has_backslash = vceqq_u8(chunk, backslash); + uint8x16_t has_dblquote = vceqq_u8(chunk, dblquote); + uint8x16_t needs_escape = vorrq_u8(too_low, vorrq_u8(has_backslash, has_dblquote)); + + return neon_match_mask(needs_escape); +} + +static inline unsigned char search_escape_basic_neon(search_state *search) +{ + if (RB_UNLIKELY(search->has_matches)) { + // There are more matches if search->matches_mask > 0. + if (search->matches_mask > 0) { + return neon_next_match(search); + } else { + // neon_next_match will only advance search->ptr up to the last matching character. + // Skip over any characters in the last chunk that occur after the last match. + search->has_matches = false; + search->ptr = search->chunk_end; + } + } + + /* + * The code below implements an SIMD-based algorithm to determine if N bytes at a time + * need to be escaped. + * + * Assume the ptr = "Te\sting!" (the double quotes are included in the string) + * + * The explanation will be limited to the first 8 bytes of the string for simplicity. However + * the vector insructions may work on larger vectors. + * + * First, we load three constants 'lower_bound', 'backslash' and 'dblquote" in vector registers. + * + * lower_bound: [20 20 20 20 20 20 20 20] + * backslash: [5C 5C 5C 5C 5C 5C 5C 5C] + * dblquote: [22 22 22 22 22 22 22 22] + * + * Next we load the first chunk of the ptr: + * [22 54 65 5C 73 74 69 6E] (" T e \ s t i n) + * + * First we check if any byte in chunk is less than 32 (0x20). This returns the following vector + * as no bytes are less than 32 (0x20): + * [0 0 0 0 0 0 0 0] + * + * Next, we check if any byte in chunk is equal to a backslash: + * [0 0 0 FF 0 0 0 0] + * + * Finally we check if any byte in chunk is equal to a double quote: + * [FF 0 0 0 0 0 0 0] + * + * Now we have three vectors where each byte indicates if the corresponding byte in chunk + * needs to be escaped. We combine these vectors with a series of logical OR instructions. + * This is the needs_escape vector and it is equal to: + * [FF 0 0 FF 0 0 0 0] + * + * Next we compute the bitwise AND between each byte and 0x1 and compute the horizontal sum of + * the values in the vector. This computes how many bytes need to be escaped within this chunk. + * + * Finally we compute a mask that indicates which bytes need to be escaped. If the mask is 0 then, + * no bytes need to be escaped and we can continue to the next chunk. If the mask is not 0 then we + * have at least one byte that needs to be escaped. + */ + while (search->ptr + sizeof(uint8x16_t) <= search->end) { + uint64_t mask = neon_rules_update(search->ptr); + + if (!mask) { + search->ptr += sizeof(uint8x16_t); + continue; + } + search->matches_mask = mask; + search->has_matches = true; + search->chunk_base = search->ptr; + search->chunk_end = search->ptr + sizeof(uint8x16_t); + return neon_next_match(search); + } + + // There are fewer than 16 bytes left. + unsigned long remaining = (search->end - search->ptr); + if (remaining >= SIMD_MINIMUM_THRESHOLD) { + char *s = copy_remaining_bytes(search, sizeof(uint8x16_t), remaining); + + uint64_t mask = neon_rules_update(s); + + if (!mask) { + // Nothing to escape, ensure search_flush doesn't do anything by setting + // search->cursor to search->ptr. + search->buffer->len += remaining; + search->ptr = search->end; + search->cursor = search->end; + return 0; + } + + search->matches_mask = mask; + search->has_matches = true; + search->chunk_end = search->end; + search->chunk_base = search->ptr; + return neon_next_match(search); + } + + if (search->ptr < search->end) { + return search_escape_basic(search); + } + + search_flush(search); + return 0; +} +#endif /* HAVE_SIMD_NEON */ + +#ifdef HAVE_SIMD_SSE2 + +#define _mm_cmpge_epu8(a, b) _mm_cmpeq_epi8(_mm_max_epu8(a, b), a) +#define _mm_cmple_epu8(a, b) _mm_cmpge_epu8(b, a) +#define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) +#define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) + +static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search) +{ + int mask = search->matches_mask; + int index = trailing_zeros(mask); + + // It is assumed escape_UTF8_char_basic will only ever increase search->ptr by at most one character. + // If we want to use a similar approach for full escaping we'll need to ensure: + // search->chunk_base + index >= search->ptr + // However, since we know escape_UTF8_char_basic only increases search->ptr by one, if the next match + // is one byte after the previous match then: + // search->chunk_base + index == search->ptr + search->ptr = search->chunk_base + index; + mask &= mask - 1; + search->matches_mask = mask; + search_flush(search); + return 1; +} + +#if defined(__clang__) || defined(__GNUC__) +#define TARGET_SSE2 __attribute__((target("sse2"))) +#else +#define TARGET_SSE2 +#endif + +static inline TARGET_SSE2 FORCE_INLINE int sse2_update(const char *ptr) +{ + __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); + + const __m128i lower_bound = _mm_set1_epi8(' '); + const __m128i backslash = _mm_set1_epi8('\\'); + const __m128i dblquote = _mm_set1_epi8('\"'); + + __m128i too_low = _mm_cmplt_epu8(chunk, lower_bound); + __m128i has_backslash = _mm_cmpeq_epi8(chunk, backslash); + __m128i has_dblquote = _mm_cmpeq_epi8(chunk, dblquote); + __m128i needs_escape = _mm_or_si128(too_low, _mm_or_si128(has_backslash, has_dblquote)); + return _mm_movemask_epi8(needs_escape); +} + +static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(search_state *search) +{ + if (RB_UNLIKELY(search->has_matches)) { + // There are more matches if search->matches_mask > 0. + if (search->matches_mask > 0) { + return sse2_next_match(search); + } else { + // sse2_next_match will only advance search->ptr up to the last matching character. + // Skip over any characters in the last chunk that occur after the last match. + search->has_matches = false; + if (RB_UNLIKELY(search->chunk_base + sizeof(__m128i) >= search->end)) { + search->ptr = search->end; + } else { + search->ptr = search->chunk_base + sizeof(__m128i); + } + } + } + + while (search->ptr + sizeof(__m128i) <= search->end) { + int needs_escape_mask = sse2_update(search->ptr); + + if (needs_escape_mask == 0) { + search->ptr += sizeof(__m128i); + continue; + } + + search->has_matches = true; + search->matches_mask = needs_escape_mask; + search->chunk_base = search->ptr; + return sse2_next_match(search); + } + + // There are fewer than 16 bytes left. + unsigned long remaining = (search->end - search->ptr); + if (remaining >= SIMD_MINIMUM_THRESHOLD) { + char *s = copy_remaining_bytes(search, sizeof(__m128i), remaining); + + int needs_escape_mask = sse2_update(s); + + if (needs_escape_mask == 0) { + // Nothing to escape, ensure search_flush doesn't do anything by setting + // search->cursor to search->ptr. + search->buffer->len += remaining; + search->ptr = search->end; + search->cursor = search->end; + return 0; + } + + search->has_matches = true; + search->matches_mask = needs_escape_mask; + search->chunk_base = search->ptr; + return sse2_next_match(search); + } + + if (search->ptr < search->end) { + return search_escape_basic(search); + } + + search_flush(search); + return 0; +} + +#endif /* HAVE_SIMD_SSE2 */ + +#endif /* ENABLE_SIMD */ + static const unsigned char script_safe_escape_table[256] = { // ASCII Control Characters 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, @@ -990,6 +1307,12 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat search.cursor = search.ptr; search.end = search.ptr + len; +#ifdef ENABLE_SIMD + search.matches_mask = 0; + search.has_matches = false; + search.chunk_base = NULL; +#endif /* ENABLE_SIMD */ + switch(rb_enc_str_coderange(obj)) { case ENC_CODERANGE_7BIT: case ENC_CODERANGE_VALID: @@ -1853,4 +2176,23 @@ void Init_generator(void) binary_encindex = rb_ascii8bit_encindex(); rb_require("json/ext/generator/state"); + + + switch(find_simd_implementation()) { +#ifdef ENABLE_SIMD +#ifdef HAVE_SIMD_NEON + case SIMD_NEON: + search_escape_basic_impl = search_escape_basic_neon; + break; +#endif /* HAVE_SIMD_NEON */ +#ifdef HAVE_SIMD_SSE2 + case SIMD_SSE2: + search_escape_basic_impl = search_escape_basic_sse2; + break; +#endif /* HAVE_SIMD_SSE2 */ +#endif /* ENABLE_SIMD */ + default: + search_escape_basic_impl = search_escape_basic; + break; + } } diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h new file mode 100644 index 000000000..4deb97b4d --- /dev/null +++ b/ext/json/ext/generator/simd.h @@ -0,0 +1,112 @@ +#include "extconf.h" + +typedef enum { + SIMD_NONE, + SIMD_NEON, + SIMD_SSE2 +} SIMD_Implementation; + +#ifdef ENABLE_SIMD + +#ifdef __clang__ + #if __has_builtin(__builtin_ctzll) + #define HAVE_BUILTIN_CTZLL 1 + #else + #define HAVE_BUILTIN_CTZLL 0 + #endif +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define HAVE_BUILTIN_CTZLL 1 +#else + #define HAVE_BUILTIN_CTZLL 0 +#endif + +static inline uint32_t trailing_zeros64(uint64_t input) { +#if HAVE_BUILTIN_CTZLL + return __builtin_ctzll(input); +#else + uint32_t trailing_zeros = 0; + uint64_t temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; +#endif +} + +static inline int trailing_zeros(int input) { + #if HAVE_BUILTIN_CTZLL + return __builtin_ctz(input); + #else + int trailing_zeros = 0; + int temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; + #endif +} + +#define SIMD_MINIMUM_THRESHOLD 6 + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) +#include + +#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 +static SIMD_Implementation find_simd_implementation(void) { + return SIMD_NEON; +} + +#define HAVE_SIMD_NEON 1 + +uint8x16x4_t load_uint8x16_4(const unsigned char *table) { + uint8x16x4_t tab; + tab.val[0] = vld1q_u8(table); + tab.val[1] = vld1q_u8(table+16); + tab.val[2] = vld1q_u8(table+32); + tab.val[3] = vld1q_u8(table+48); + return tab; +} + +#endif /* ARM Neon Support.*/ + +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) + +#ifdef HAVE_X86INTRIN_H +#include + +#define HAVE_SIMD_SSE2 1 + +#ifdef HAVE_CPUID_H +#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 + +#include +#endif /* HAVE_CPUID_H */ + +static SIMD_Implementation find_simd_implementation(void) { + +#if defined(__GNUC__ ) || defined(__clang__) +#ifdef __GNUC__ + __builtin_cpu_init(); +#endif /* __GNUC__ */ + + // TODO Revisit. I think the SSE version now only uses SSE2 instructions. + if (__builtin_cpu_supports("sse2")) { + return SIMD_SSE2; + } +#endif /* __GNUC__ || __clang__*/ + + return SIMD_NONE; +} + +#endif /* HAVE_X86INTRIN_H */ +#endif /* X86_64 Support */ + +#endif /* ENABLE_SIMD */ + +#ifndef FIND_SIMD_IMPLEMENTATION_DEFINED +static SIMD_Implementation find_simd_implementation(void) { + return SIMD_NONE; +} +#endif diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 47c94bd39..3192b555a 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -410,18 +410,34 @@ def test_backslash json = '["\\\\.(?i:gif|jpe?g|png)$"]' assert_equal json, generate(data) # - data = [ '\\"' ] - json = '["\\\\\""]' + data = [ '\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$\\.(?i:gif|jpe?g|png)$' ] + json = '["\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$\\\\.(?i:gif|jpe?g|png)$"]' + assert_equal json, generate(data) + # + data = [ '\\"\\"\\"\\"\\"\\"\\"\\"\\"\\"\\"' ] + json = '["\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\"\\\\\""]' assert_equal json, generate(data) # data = [ '/' ] json = '["/"]' assert_equal json, generate(data) # + data = [ '////////////////////////////////////////////////////////////////////////////////////' ] + json = '["////////////////////////////////////////////////////////////////////////////////////"]' + assert_equal json, generate(data) + # data = [ '/' ] json = '["\/"]' assert_equal json, generate(data, :script_safe => true) # + data = [ '///////////' ] + json = '["\/\/\/\/\/\/\/\/\/\/\/"]' + assert_equal json, generate(data, :script_safe => true) + # + data = [ '///////////////////////////////////////////////////////' ] + json = '["\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/"]' + assert_equal json, generate(data, :script_safe => true) + # data = [ "\u2028\u2029" ] json = '["\u2028\u2029"]' assert_equal json, generate(data, :script_safe => true) @@ -438,6 +454,10 @@ def test_backslash json = '["\""]' assert_equal json, generate(data) # + data = ['"""""""""""""""""""""""""'] + json = '["\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""]' + assert_equal json, generate(data) + # data = ["'"] json = '["\\\'"]' assert_equal '["\'"]', generate(data) @@ -445,6 +465,72 @@ def test_backslash data = ["倩", "瀨"] json = '["倩","瀨"]' assert_equal json, generate(data, script_safe: true) + # + data = '["This is a "test" of the emergency broadcast system."]' + json = "\"[\\\"This is a \\\"test\\\" of the emergency broadcast system.\\\"]\"" + assert_equal json, generate(data) + # + data = '\tThis is a test of the emergency broadcast system.' + json = "\"\\\\tThis is a test of the emergency broadcast system.\"" + assert_equal json, generate(data) + # + data = 'This\tis a test of the emergency broadcast system.' + json = "\"This\\\\tis a test of the emergency broadcast system.\"" + assert_equal json, generate(data) + # + data = 'This is\ta test of the emergency broadcast system.' + json = "\"This is\\\\ta test of the emergency broadcast system.\"" + assert_equal json, generate(data) + # + data = 'This is a test of the emergency broadcast\tsystem.' + json = "\"This is a test of the emergency broadcast\\\\tsystem.\"" + assert_equal json, generate(data) + # + data = 'This is a test of the emergency broadcast\tsystem.\n' + json = "\"This is a test of the emergency broadcast\\\\tsystem.\\\\n\"" + assert_equal json, generate(data) + data = '"' * 15 + json = "\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\"" + assert_equal json, generate(data) + data = "\"\"\"\"\"\"\"\"\"\"\"\"\"\"a" + json = "\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"a\"" + assert_equal json, generate(data) + data = "\u0001\u0001\u0001\u0001" + json = "\"\\u0001\\u0001\\u0001\\u0001\"" + assert_equal json, generate(data) + data = "\u0001a\u0001a\u0001a\u0001a" + json = "\"\\u0001a\\u0001a\\u0001a\\u0001a\"" + assert_equal json, generate(data) + data = "\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\"" + assert_equal json, generate(data) + data = "\u0001aa\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\\u0001aa\"" + assert_equal json, generate(data) + data = "\u0001aa\u0001aa\u0001aa\u0001aa\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\\u0001aa\\u0001aa\\u0001aa\\u0001aa\"" + assert_equal json, generate(data) + data = "\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002" + json = "\"\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\"" + assert_equal json, generate(data) + data = "ab\u0002c" + json = "\"ab\\u0002c\"" + assert_equal json, generate(data) + data = "ab\u0002cab\u0002cab\u0002cab\u0002c" + json = "\"ab\\u0002cab\\u0002cab\\u0002cab\\u0002c\"" + assert_equal json, generate(data) + data = "ab\u0002cab\u0002cab\u0002cab\u0002cab\u0002cab\u0002c" + json = "\"ab\\u0002cab\\u0002cab\\u0002cab\\u0002cab\\u0002cab\\u0002c\"" + assert_equal json, generate(data) + data = "\n\t\f\b\n\t\f\b\n\t\f\b\n\t\f" + json = "\"\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\"" + assert_equal json, generate(data) + data = "\n\t\f\b\n\t\f\b\n\t\f\b\n\t\f\b" + json = "\"\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\"" + assert_equal json, generate(data) + data = "a\n\t\f\b\n\t\f\b\n\t\f\b\n\t" + json = "\"a\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\"" + assert_equal json, generate(data) end def test_string_subclass From 2739de4fcd423172e71410b66c33e5ecdeb81e70 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 15:06:36 +0200 Subject: [PATCH 047/506] Update ext/json/ext/generator/depend --- ext/json/ext/generator/depend | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 60697f2c4..21bdc420f 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1,3 +1,4 @@ generator.o: generator.c $(srcdir)/../fbuffer/fbuffer.h generator.o: generator.c $(srcdir)/../vendor/fpconv.c generator.o: generator.c $(srcdir)/../vendor/jeaiii-ltoa.h +generator.o: generator.c $(srcdir)/simd.h From 034c5debd8e4b36cd663220faabfa178c7ea91b5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 15:23:48 +0200 Subject: [PATCH 048/506] SIMD: Match control char and double quote in one pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `c < 32 || c == 34` is equivalent to `c ^ 2 < 33`. Found in: https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ The gain seem mostly present on micro-benchmark, and even there aren't very consistent, but it's never slower. ``` == Encoding long string (124001 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 5.295k i/100ms Calculating ------------------------------------- after 55.796k (± 3.4%) i/s (17.92 μs/i) - 280.635k in 5.035690s Comparison: before: 49840.7 i/s after: 55795.8 i/s - 1.12x faster ``` --- ext/json/ext/generator/generator.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index c09061728..3112ef1a4 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -320,14 +320,12 @@ static inline FORCE_INLINE uint64_t neon_rules_update(const char *ptr) { uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); - const uint8x16_t lower_bound = vdupq_n_u8(' '); - const uint8x16_t backslash = vdupq_n_u8('\\'); - const uint8x16_t dblquote = vdupq_n_u8('\"'); + // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 + // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ + const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33)); - uint8x16_t too_low = vcltq_u8(chunk, lower_bound); - uint8x16_t has_backslash = vceqq_u8(chunk, backslash); - uint8x16_t has_dblquote = vceqq_u8(chunk, dblquote); - uint8x16_t needs_escape = vorrq_u8(too_low, vorrq_u8(has_backslash, has_dblquote)); + uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\')); + uint8x16_t needs_escape = vorrq_u8(too_low_or_dbl_quote, has_backslash); return neon_match_mask(needs_escape); } @@ -467,14 +465,11 @@ static inline TARGET_SSE2 FORCE_INLINE int sse2_update(const char *ptr) { __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); - const __m128i lower_bound = _mm_set1_epi8(' '); - const __m128i backslash = _mm_set1_epi8('\\'); - const __m128i dblquote = _mm_set1_epi8('\"'); - - __m128i too_low = _mm_cmplt_epu8(chunk, lower_bound); - __m128i has_backslash = _mm_cmpeq_epi8(chunk, backslash); - __m128i has_dblquote = _mm_cmpeq_epi8(chunk, dblquote); - __m128i needs_escape = _mm_or_si128(too_low, _mm_or_si128(has_backslash, has_dblquote)); + // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 + // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ + __m128i too_low_or_dbl_quote = _mm_cmplt_epu8(_mm_xor_si128(chunk, _mm_set1_epi8(2)), _mm_set1_epi8(33)); + __m128i has_backslash = _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\')); + __m128i needs_escape = _mm_or_si128(too_low_or_dbl_quote, has_backslash); return _mm_movemask_epi8(needs_escape); } From 0a871365db420ef056546545c13a069fd28ae17c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 16:15:36 +0200 Subject: [PATCH 049/506] Fix i686 builds We should test compilation with `-msse2` because we need to test with whatever arguments Ruby will be compiled with. --- ext/json/ext/generator/extconf.rb | 2 +- ext/json/ext/generator/generator.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 4fbeb5f33..374f95a1a 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -22,7 +22,7 @@ end end - if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC', opt='-msse2') + if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') #include int main() { __m128i test = _mm_set1_epi8(32); diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 3112ef1a4..536c2aa1b 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -117,9 +117,9 @@ typedef struct _search_state { const char *chunk_end; bool has_matches; -#ifdef HAVE_SIMD_NEON +#if defined(HAVE_SIMD_NEON) uint64_t matches_mask; -#elif HAVE_SIMD_SSE2 +#elif defined(HAVE_SIMD_SSE2) int matches_mask; #else #error "Unknown SIMD Implementation." From d7d60cccb067d332d29a988c045b551afc40efd2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 18:01:18 +0200 Subject: [PATCH 050/506] Fix --with-static-linked-ext builds --- ext/json/ext/generator/extconf.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 374f95a1a..e44890e2e 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -35,7 +35,5 @@ have_header('cpuid.h') end - create_header - create_makefile 'json/ext/generator' end From 6b059900de98edcb946314f8d319a17119e64d64 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Apr 2025 18:50:24 +0200 Subject: [PATCH 051/506] Remove explicit include of extconf.h --- ext/json/ext/generator/simd.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h index 4deb97b4d..2fbc93169 100644 --- a/ext/json/ext/generator/simd.h +++ b/ext/json/ext/generator/simd.h @@ -1,5 +1,3 @@ -#include "extconf.h" - typedef enum { SIMD_NONE, SIMD_NEON, From 410d3ed19193fb8d0b3288152859da6aa81c798a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 29 Apr 2025 11:53:38 +0200 Subject: [PATCH 052/506] Update changelog --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index fc055710b..c2ee0daff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ # Changes +### Unreleased + * Handle non-string hash keys with broken `to_s` implementations. +* `JSON.generate` now uses SSE2 (x86) or NEON (arm64) instructions when available to escape strings. ### 2025-04-25 (2.11.3) From d9a42c2ec0da5cca166ee0aeff8d5a1d190eabee Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 29 Apr 2025 06:54:38 -0500 Subject: [PATCH 053/506] Remove unnecessary conditional in escape_UTF8_char_basic. --- ext/json/ext/generator/generator.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 536c2aa1b..633fa87cc 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -189,15 +189,11 @@ static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search) case '\r': fbuffer_append(search->buffer, "\\r", 2); break; case '\t': fbuffer_append(search->buffer, "\\t", 2); break; default: { - if (ch < ' ') { - const char *hexdig = "0123456789abcdef"; - char scratch[6] = { '\\', 'u', '0', '0', 0, 0 }; - scratch[4] = hexdig[(ch >> 4) & 0xf]; - scratch[5] = hexdig[ch & 0xf]; - fbuffer_append(search->buffer, scratch, 6); - } else { - fbuffer_append_char(search->buffer, ch); - } + const char *hexdig = "0123456789abcdef"; + char scratch[6] = { '\\', 'u', '0', '0', 0, 0 }; + scratch[4] = hexdig[(ch >> 4) & 0xf]; + scratch[5] = hexdig[ch & 0xf]; + fbuffer_append(search->buffer, scratch, 6); break; } } From 609393b8af310381edce4bc5a742abc500432ed0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 1 May 2025 08:45:07 +0200 Subject: [PATCH 054/506] Namespace the ENABLE_SIMD define --- ext/json/ext/generator/extconf.rb | 4 ++-- ext/json/ext/generator/generator.c | 16 ++++++++-------- ext/json/ext/generator/simd.h | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index e44890e2e..60372ee55 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -18,7 +18,7 @@ return 0; } SRC - $defs.push("-DENABLE_SIMD") + $defs.push("-DJSON_ENABLE_SIMD") end end @@ -29,7 +29,7 @@ return 0; } SRC - $defs.push("-DENABLE_SIMD") + $defs.push("-DJSON_ENABLE_SIMD") end have_header('cpuid.h') diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 633fa87cc..488521dba 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -112,7 +112,7 @@ typedef struct _search_state { const char *cursor; FBuffer *buffer; -#ifdef ENABLE_SIMD +#ifdef JSON_ENABLE_SIMD const char *chunk_base; const char *chunk_end; bool has_matches; @@ -124,7 +124,7 @@ typedef struct _search_state { #else #error "Unknown SIMD Implementation." #endif /* HAVE_SIMD_NEON */ -#endif /* ENABLE_SIMD */ +#endif /* JSON_ENABLE_SIMD */ } search_state; #if (defined(__GNUC__ ) || defined(__clang__)) @@ -261,7 +261,7 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) search->cursor = (search->ptr += ch_len); } -#ifdef ENABLE_SIMD +#ifdef JSON_ENABLE_SIMD static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) { @@ -533,7 +533,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se #endif /* HAVE_SIMD_SSE2 */ -#endif /* ENABLE_SIMD */ +#endif /* JSON_ENABLE_SIMD */ static const unsigned char script_safe_escape_table[256] = { // ASCII Control Characters @@ -1298,11 +1298,11 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat search.cursor = search.ptr; search.end = search.ptr + len; -#ifdef ENABLE_SIMD +#ifdef JSON_ENABLE_SIMD search.matches_mask = 0; search.has_matches = false; search.chunk_base = NULL; -#endif /* ENABLE_SIMD */ +#endif /* JSON_ENABLE_SIMD */ switch(rb_enc_str_coderange(obj)) { case ENC_CODERANGE_7BIT: @@ -2170,7 +2170,7 @@ void Init_generator(void) switch(find_simd_implementation()) { -#ifdef ENABLE_SIMD +#ifdef JSON_ENABLE_SIMD #ifdef HAVE_SIMD_NEON case SIMD_NEON: search_escape_basic_impl = search_escape_basic_neon; @@ -2181,7 +2181,7 @@ void Init_generator(void) search_escape_basic_impl = search_escape_basic_sse2; break; #endif /* HAVE_SIMD_SSE2 */ -#endif /* ENABLE_SIMD */ +#endif /* JSON_ENABLE_SIMD */ default: search_escape_basic_impl = search_escape_basic; break; diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h index 2fbc93169..96b3d539a 100644 --- a/ext/json/ext/generator/simd.h +++ b/ext/json/ext/generator/simd.h @@ -4,7 +4,7 @@ typedef enum { SIMD_SSE2 } SIMD_Implementation; -#ifdef ENABLE_SIMD +#ifdef JSON_ENABLE_SIMD #ifdef __clang__ #if __has_builtin(__builtin_ctzll) @@ -101,7 +101,7 @@ static SIMD_Implementation find_simd_implementation(void) { #endif /* HAVE_X86INTRIN_H */ #endif /* X86_64 Support */ -#endif /* ENABLE_SIMD */ +#endif /* JSON_ENABLE_SIMD */ #ifndef FIND_SIMD_IMPLEMENTATION_DEFINED static SIMD_Implementation find_simd_implementation(void) { From 220e019e62e1f76985909b50e5d77dc496abe365 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 1 May 2025 09:25:04 +0200 Subject: [PATCH 055/506] Refactor SIMD ifdefs The extconf check if SIMD is available, but then later on the `simd.h` header has extra logic to decide if it uses it or not. e.g. for SSE2 we have: ``` if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) ``` Which exclude i686 builds. Hence if we're not entering that `#if` we can end up with `JSON_ENABLE_SIMD=1` but neither `HAVE_SIMD_NEON` nor `HAVE_SIMD_SSE2`. So instead of relying on `JSON_ENABLE_SIMD` to check if any SIMD implementation is available, this commit introduce `HAVE_SIMD`. --- ext/json/ext/generator/generator.c | 16 ++++++++-------- ext/json/ext/generator/simd.h | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 488521dba..06ab8010d 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -112,7 +112,7 @@ typedef struct _search_state { const char *cursor; FBuffer *buffer; -#ifdef JSON_ENABLE_SIMD +#ifdef HAVE_SIMD const char *chunk_base; const char *chunk_end; bool has_matches; @@ -124,7 +124,7 @@ typedef struct _search_state { #else #error "Unknown SIMD Implementation." #endif /* HAVE_SIMD_NEON */ -#endif /* JSON_ENABLE_SIMD */ +#endif /* HAVE_SIMD */ } search_state; #if (defined(__GNUC__ ) || defined(__clang__)) @@ -261,7 +261,7 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) search->cursor = (search->ptr += ch_len); } -#ifdef JSON_ENABLE_SIMD +#ifdef HAVE_SIMD static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) { @@ -533,7 +533,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se #endif /* HAVE_SIMD_SSE2 */ -#endif /* JSON_ENABLE_SIMD */ +#endif /* HAVE_SIMD */ static const unsigned char script_safe_escape_table[256] = { // ASCII Control Characters @@ -1298,11 +1298,11 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat search.cursor = search.ptr; search.end = search.ptr + len; -#ifdef JSON_ENABLE_SIMD +#ifdef HAVE_SIMD search.matches_mask = 0; search.has_matches = false; search.chunk_base = NULL; -#endif /* JSON_ENABLE_SIMD */ +#endif /* HAVE_SIMD */ switch(rb_enc_str_coderange(obj)) { case ENC_CODERANGE_7BIT: @@ -2170,7 +2170,7 @@ void Init_generator(void) switch(find_simd_implementation()) { -#ifdef JSON_ENABLE_SIMD +#ifdef HAVE_SIMD #ifdef HAVE_SIMD_NEON case SIMD_NEON: search_escape_basic_impl = search_escape_basic_neon; @@ -2181,7 +2181,7 @@ void Init_generator(void) search_escape_basic_impl = search_escape_basic_sse2; break; #endif /* HAVE_SIMD_SSE2 */ -#endif /* JSON_ENABLE_SIMD */ +#endif /* HAVE_SIMD */ default: search_escape_basic_impl = search_escape_basic; break; diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h index 96b3d539a..b12890cb0 100644 --- a/ext/json/ext/generator/simd.h +++ b/ext/json/ext/generator/simd.h @@ -56,6 +56,7 @@ static SIMD_Implementation find_simd_implementation(void) { return SIMD_NEON; } +#define HAVE_SIMD 1 #define HAVE_SIMD_NEON 1 uint8x16x4_t load_uint8x16_4(const unsigned char *table) { @@ -74,6 +75,7 @@ uint8x16x4_t load_uint8x16_4(const unsigned char *table) { #ifdef HAVE_X86INTRIN_H #include +#define HAVE_SIMD 1 #define HAVE_SIMD_SSE2 1 #ifdef HAVE_CPUID_H From 3b605d9b1eec5262386a469b7ec64206eaa10be3 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 6 May 2025 18:22:18 +0900 Subject: [PATCH 056/506] Constify static data in fpconv.c --- ext/json/ext/vendor/fpconv.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 2887c6488..854cae288 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -41,7 +41,7 @@ typedef struct Fp { int exp; } Fp; -static Fp powers_ten[] = { +static const Fp powers_ten[] = { { 18054884314459144840U, -1220 }, { 13451937075301367670U, -1193 }, { 10022474136428063862U, -1166 }, { 14934650266808366570U, -1140 }, { 11127181549972568877U, -1113 }, { 16580792590934885855U, -1087 }, @@ -123,7 +123,7 @@ static Fp find_cachedpow10(int exp, int* k) #define absv(n) ((n) < 0 ? -(n) : (n)) #define minv(a, b) ((a) < (b) ? (a) : (b)) -static uint64_t tens[] = { +static const uint64_t tens[] = { 10000000000000000000U, 1000000000000000000U, 100000000000000000U, 10000000000000000U, 1000000000000000U, 100000000000000U, 10000000000000U, 1000000000000U, 100000000000U, @@ -244,7 +244,7 @@ static int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) uint64_t part2 = upper->frac & (one.frac - 1); int idx = 0, kappa = 10; - uint64_t* divp; + const uint64_t* divp; /* 1000000000 */ for(divp = tens + 10; kappa > 0; divp++) { @@ -268,7 +268,7 @@ static int generate_digits(Fp* fp, Fp* upper, Fp* lower, char* digits, int* K) } /* 10 */ - uint64_t* unit = tens + 18; + const uint64_t* unit = tens + 18; while(true) { part2 *= 10; From 866f72a4379959e8fb5e7c9adaca2f519ffcaade Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 10 May 2025 11:31:33 +0200 Subject: [PATCH 057/506] Favor decimal notation over scientific notation for floats e.g. ``` JSON.dump(1746861937.7842371) ``` master: ``` "1.7468619377842371e+9" ``` This branch and older json versions: ``` 1746861937.7842371 ``` In the end it's shorter, and according to `canada.json` benchmark performance is the same. --- ext/json/ext/vendor/fpconv.c | 2 +- test/json/json_generator_test.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 854cae288..440759e5a 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -340,7 +340,7 @@ static int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) } /* write decimal w/o scientific notation */ - if(K < 0 && (K > -7 || exp < 4)) { + if(K < 0 && (K > -7 || exp < 10)) { int offset = ndigits - absv(K); /* fp < 1.0 -> write leading zero */ if(offset <= 0) { diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 3192b555a..55a3065ae 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -771,6 +771,14 @@ def test_json_generate_float values = [-1.0, 1.0, 0.0, 12.2, 7.5 / 3.2, 12.0, 100.0, 1000.0] expecteds = ["-1.0", "1.0", "0.0", "12.2", "2.34375", "12.0", "100.0", "1000.0"] + if RUBY_ENGINE == "jruby" + values << 1746861937.7842371 + expecteds << "1.7468619377842371E9" + else + values << 1746861937.7842371 + expecteds << "1746861937.7842371" + end + values.zip(expecteds).each do |value, expected| assert_equal expected, value.to_json end From 832b5b1a4c6d07b5feda599caf4c4edada249203 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 13:06:05 +0200 Subject: [PATCH 058/506] parser.c: refactor `raise_parse_error` to have document start --- ext/json/ext/parser/parser.c | 197 +++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 91 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index f20769a36..6a12ac051 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -337,19 +337,69 @@ static size_t strnlen(const char *s, size_t maxlen) } #endif +static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) +{ + int len = 1; + if (ch <= 0x7F) { + buf[0] = (char) ch; + } else if (ch <= 0x07FF) { + buf[0] = (char) ((ch >> 6) | 0xC0); + buf[1] = (char) ((ch & 0x3F) | 0x80); + len++; + } else if (ch <= 0xFFFF) { + buf[0] = (char) ((ch >> 12) | 0xE0); + buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); + buf[2] = (char) ((ch & 0x3F) | 0x80); + len += 2; + } else if (ch <= 0x1fffff) { + buf[0] =(char) ((ch >> 18) | 0xF0); + buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); + buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); + buf[3] =(char) ((ch & 0x3F) | 0x80); + len += 3; + } else { + buf[0] = '?'; + } + return len; +} + +typedef struct JSON_ParserStruct { + VALUE on_load_proc; + VALUE decimal_class; + ID decimal_method_id; + int max_nesting; + bool allow_nan; + bool allow_trailing_comma; + bool parsing_name; + bool symbolize_names; + bool freeze; +} JSON_ParserConfig; + +typedef struct JSON_ParserStateStruct { + VALUE stack_handle; + const char *start; + const char *cursor; + const char *end; + rvalue_stack *stack; + rvalue_cache name_cache; + int in_array; + int current_nesting; +} JSON_ParserState; + + #define PARSE_ERROR_FRAGMENT_LEN 32 #ifdef RBIMPL_ATTR_NORETURN RBIMPL_ATTR_NORETURN() #endif -static void raise_parse_error(const char *format, const char *start) +static void raise_parse_error(const char *format, JSON_ParserState *state) { unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; - size_t len = start ? strnlen(start, PARSE_ERROR_FRAGMENT_LEN) : 0; - const char *ptr = start; + const char *ptr = state->cursor; + size_t len = ptr ? strnlen(ptr, PARSE_ERROR_FRAGMENT_LEN) : 0; if (len == PARSE_ERROR_FRAGMENT_LEN) { - MEMCPY(buffer, start, char, PARSE_ERROR_FRAGMENT_LEN); + MEMCPY(buffer, ptr, char, PARSE_ERROR_FRAGMENT_LEN); while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte len--; @@ -366,6 +416,15 @@ static void raise_parse_error(const char *format, const char *start) rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr); } +#ifdef RBIMPL_ATTR_NORETURN +RBIMPL_ATTR_NORETURN() +#endif +static void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) +{ + state->cursor = at; + raise_parse_error(format, state); +} + /* unicode */ static const signed char digit_values[256] = { @@ -385,73 +444,25 @@ static const signed char digit_values[256] = { -1, -1, -1, -1, -1, -1, -1 }; -static uint32_t unescape_unicode(const unsigned char *p) +static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p) { signed char b; uint32_t result = 0; b = digit_values[p[0]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[1]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[2]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[3]]; - if (b < 0) raise_parse_error("incomplete unicode character escape sequence at '%s'", (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; return result; } -static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) -{ - int len = 1; - if (ch <= 0x7F) { - buf[0] = (char) ch; - } else if (ch <= 0x07FF) { - buf[0] = (char) ((ch >> 6) | 0xC0); - buf[1] = (char) ((ch & 0x3F) | 0x80); - len++; - } else if (ch <= 0xFFFF) { - buf[0] = (char) ((ch >> 12) | 0xE0); - buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); - buf[2] = (char) ((ch & 0x3F) | 0x80); - len += 2; - } else if (ch <= 0x1fffff) { - buf[0] =(char) ((ch >> 18) | 0xF0); - buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); - buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); - buf[3] =(char) ((ch & 0x3F) | 0x80); - len += 3; - } else { - buf[0] = '?'; - } - return len; -} - -typedef struct JSON_ParserStruct { - VALUE on_load_proc; - VALUE decimal_class; - ID decimal_method_id; - int max_nesting; - bool allow_nan; - bool allow_trailing_comma; - bool parsing_name; - bool symbolize_names; - bool freeze; -} JSON_ParserConfig; - -typedef struct JSON_ParserStateStruct { - VALUE stack_handle; - const char *cursor; - const char *end; - rvalue_stack *stack; - rvalue_cache name_cache; - int in_array; - int current_nesting; -} JSON_ParserState; - #define GET_PARSER_CONFIG \ JSON_ParserConfig *config; \ TypedData_Get_Struct(self, JSON_ParserConfig, &JSON_ParserConfig_type, config) @@ -485,8 +496,7 @@ json_eat_comments(JSON_ParserState *state) while (true) { state->cursor = memchr(state->cursor, '*', state->end - state->cursor); if (!state->cursor) { - state->cursor = state->end; - raise_parse_error("unexpected end of input, expected closing '*/'", state->cursor); + raise_parse_error_at("unexpected end of input, expected closing '*/'", state, state->end); } else { state->cursor++; if (state->cursor < state->end && *state->cursor == '/') { @@ -498,11 +508,11 @@ json_eat_comments(JSON_ParserState *state) break; } default: - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; } } else { - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); } } @@ -621,9 +631,9 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c break; case 'u': if (pe > stringEnd - 5) { - raise_parse_error("incomplete unicode character escape sequence at '%s'", p); + raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, p); } else { - uint32_t ch = unescape_unicode((unsigned char *) ++pe); + uint32_t ch = unescape_unicode(state, (unsigned char *) ++pe); pe += 3; /* To handle values above U+FFFF, we take a sequence of * \uXXXX escapes in the U+D800..U+DBFF then @@ -638,10 +648,10 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c if ((ch & 0xFC00) == 0xD800) { pe++; if (pe > stringEnd - 6) { - raise_parse_error("incomplete surrogate pair at '%s'", p); + raise_parse_error_at("incomplete surrogate pair at '%s'", state, p); } if (pe[0] == '\\' && pe[1] == 'u') { - uint32_t sur = unescape_unicode((unsigned char *) pe + 2); + uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2); ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); pe += 5; @@ -829,12 +839,12 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig state->cursor++; escaped = true; if ((unsigned char)*state->cursor < 0x20) { - raise_parse_error("invalid ASCII control character in string: %s", state->cursor); + raise_parse_error("invalid ASCII control character in string: %s", state); } break; } default: - raise_parse_error("invalid ASCII control character in string: %s", state->cursor); + raise_parse_error("invalid ASCII control character in string: %s", state); break; } } @@ -842,7 +852,7 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig state->cursor++; } - raise_parse_error("unexpected end of input, expected closing \"", state->cursor); + raise_parse_error("unexpected end of input, expected closing \"", state); return Qfalse; } @@ -850,7 +860,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { json_eat_whitespace(state); if (state->cursor >= state->end) { - raise_parse_error("unexpected end of input", state->cursor); + raise_parse_error("unexpected end of input", state); } switch (*state->cursor) { @@ -860,7 +870,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qnil); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; case 't': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { @@ -868,7 +878,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qtrue); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; case 'f': // Note: memcmp with a small power of two compile to an integer comparison @@ -877,7 +887,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qfalse); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; case 'N': // Note: memcmp with a small power of two compile to an integer comparison @@ -886,7 +896,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CNaN); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; case 'I': if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { @@ -894,7 +904,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CInfinity); } - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); break; case '-': // Note: memcmp with a small power of two compile to an integer comparison @@ -903,7 +913,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor += 9; return json_push_value(state, config, CMinusInfinity); } else { - raise_parse_error("unexpected token at '%s'", state->cursor); + raise_parse_error("unexpected token at '%s'", state); } } // Fallthrough @@ -921,11 +931,11 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) long integer_length = state->cursor - start; if (RB_UNLIKELY(start[0] == '0' && integer_length > 1)) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } else if (RB_UNLIKELY(integer_length > 2 && start[0] == '-' && start[1] == '0')) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } else if (RB_UNLIKELY(integer_length == 1 && start[0] == '-')) { - raise_parse_error("invalid number: %s", start); + raise_parse_error_at("invalid number: %s", state, start); } if ((state->cursor < state->end) && (*state->cursor == '.')) { @@ -933,7 +943,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state->cursor); + raise_parse_error("invalid number: %s", state); } while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { @@ -949,7 +959,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state->cursor); + raise_parse_error("invalid number: %s", state); } while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { @@ -1009,7 +1019,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } } - raise_parse_error("expected ',' or ']' after array value", state->cursor); + raise_parse_error("expected ',' or ']' after array value", state); } break; } @@ -1028,13 +1038,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got '%s", state->cursor); + raise_parse_error("expected object key, got '%s", state); } json_parse_string(state, config, true); json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key", state->cursor); + raise_parse_error("expected ':' after object key", state); } state->cursor++; @@ -1063,13 +1073,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got: '%s'", state->cursor); + raise_parse_error("expected object key, got: '%s'", state); } json_parse_string(state, config, true); json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key, got: '%s", state->cursor); + raise_parse_error("expected ':' after object key, got: '%s", state); } state->cursor++; @@ -1079,24 +1089,24 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } } - raise_parse_error("expected ',' or '}' after object value, got: '%s'", state->cursor); + raise_parse_error("expected ',' or '}' after object value, got: '%s'", state); } break; } default: - raise_parse_error("unexpected character: '%s'", state->cursor); + raise_parse_error("unexpected character: '%s'", state); break; } - raise_parse_error("unreacheable: '%s'", state->cursor); + raise_parse_error("unreacheable: '%s'", state); } static void json_ensure_eof(JSON_ParserState *state) { json_eat_whitespace(state); if (state->cursor != state->end) { - raise_parse_error("unexpected token at end of stream '%s'", state->cursor); + raise_parse_error("unexpected token at end of stream '%s'", state); } } @@ -1232,9 +1242,14 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) .capa = RVALUE_STACK_INITIAL_CAPA, }; + long len; + const char *start; + RSTRING_GETMEM(Vsource, start, len); + JSON_ParserState _state = { - .cursor = RSTRING_PTR(Vsource), - .end = RSTRING_END(Vsource), + .start = start, + .cursor = start, + .end = start + len, .stack = &stack, }; JSON_ParserState *state = &_state; From 30e35b9ba5944298b9ad11965a9eb95894c42f3c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 13:34:13 +0200 Subject: [PATCH 059/506] parser.c: include line and column in error messages --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 42 ++++++++++++++++++++++++------- lib/json/common.rb | 4 ++- test/json/json_ext_parser_test.rb | 12 ++++++--- test/json/json_parser_test.rb | 12 ++++----- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c2ee0daff..1f58e9dc5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Include line and column in parser errors. Both in the message and as exception attributes. * Handle non-string hash keys with broken `to_s` implementations. * `JSON.generate` now uses SSE2 (x86) or NEON (arm64) instructions when available to escape strings. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 6a12ac051..13fd85219 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -395,6 +395,23 @@ static void raise_parse_error(const char *format, JSON_ParserState *state) { unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; + const char *cursor = state->cursor; + long column = 0; + long line = 1; + + while (cursor >= state->start) { + if (*cursor-- == '\n') { + break; + } + column++; + } + + while (cursor >= state->start) { + if (*cursor-- == '\n') { + line++; + } + } + const char *ptr = state->cursor; size_t len = ptr ? strnlen(ptr, PARSE_ERROR_FRAGMENT_LEN) : 0; @@ -413,7 +430,14 @@ static void raise_parse_error(const char *format, JSON_ParserState *state) ptr = (const char *)buffer; } - rb_enc_raise(enc_utf8, rb_path2class("JSON::ParserError"), format, ptr); + VALUE msg = rb_sprintf(format, ptr); + VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column); + RB_GC_GUARD(msg); + + VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message); + rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line)); + rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column)); + rb_exc_raise(exc); } #ifdef RBIMPL_ATTR_NORETURN @@ -508,11 +532,11 @@ json_eat_comments(JSON_ParserState *state) break; } default: - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; } } else { - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); } } @@ -870,7 +894,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qnil); } - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; case 't': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { @@ -878,7 +902,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qtrue); } - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; case 'f': // Note: memcmp with a small power of two compile to an integer comparison @@ -887,7 +911,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qfalse); } - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; case 'N': // Note: memcmp with a small power of two compile to an integer comparison @@ -896,7 +920,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CNaN); } - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; case 'I': if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { @@ -904,7 +928,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CInfinity); } - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); break; case '-': // Note: memcmp with a small power of two compile to an integer comparison @@ -913,7 +937,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor += 9; return json_push_value(state, config, CMinusInfinity); } else { - raise_parse_error("unexpected token at '%s'", state); + raise_parse_error("unexpected token '%s'", state); } } // Fallthrough diff --git a/lib/json/common.rb b/lib/json/common.rb index 98bc0ac85..7627761b5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -230,7 +230,9 @@ def self.create_id class JSONError < StandardError; end # This exception is raised if a parser error occurs. - class ParserError < JSONError; end + class ParserError < JSONError + attr_reader :line, :column + end # This exception is raised if the nesting of parsed data structures is too # deep. diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index 8aa626257..739798d05 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -15,15 +15,19 @@ def test_allocate def test_error_messages ex = assert_raise(ParserError) { parse('Infinity') } - assert_equal "unexpected token at 'Infinity'", ex.message + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token 'Infinity' at line 1 column 1", ex.message + end + ex = assert_raise(ParserError) { parse('-Infinity') } unless RUBY_PLATFORM =~ /java/ - ex = assert_raise(ParserError) { parse('-Infinity') } - assert_equal "unexpected token at '-Infinity'", ex.message + assert_equal "unexpected token '-Infinity' at line 1 column 1", ex.message end ex = assert_raise(ParserError) { parse('NaN') } - assert_equal "unexpected token at 'NaN'", ex.message + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token 'NaN' at line 1 column 1", ex.message + end end if GC.respond_to?(:stress=) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 87b78fb0c..7cfdfc69d 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -638,7 +638,7 @@ def test_parse_error_message_length error = assert_raise(JSON::ParserError) do JSON.parse('{"foo": ' + ('A' * 500) + '}') end - assert_operator 60, :>, error.message.bytesize + assert_operator 80, :>, error.message.bytesize end def test_parse_error_incomplete_hash @@ -646,7 +646,7 @@ def test_parse_error_incomplete_hash JSON.parse('{"input":{"firstName":"Bob","lastName":"Mob","email":"bob@example.com"}') end if RUBY_ENGINE == "ruby" - assert_equal %(expected ',' or '}' after object value, got: ''), error.message + assert_equal %(expected ',' or '}' after object value, got: '' at line 1 column 72), error.message end end @@ -654,16 +654,16 @@ def test_parse_error_snippet omit "C ext only test" unless RUBY_ENGINE == "ruby" error = assert_raise(JSON::ParserError) { JSON.parse("あああああああああああああああああああああああ") } - assert_equal "unexpected character: 'ああああああああああ'", error.message + assert_equal "unexpected character: 'ああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("aあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'aああああああああああ'", error.message + assert_equal "unexpected character: 'aああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abあああああああああ'", error.message + assert_equal "unexpected character: 'abあああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abcあああああああああ'", error.message + assert_equal "unexpected character: 'abcあああああああああ' at line 1 column 1", error.message end def test_parse_leading_slash From 41f1f6939d95fe7a6df81b7d66e2f068bbaa10e0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 14:31:12 +0200 Subject: [PATCH 060/506] Release 2.12.0 --- CHANGES.md | 3 +++ lib/json/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1f58e9dc5..4f91abd47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +### 2025-05-12 (2.12.0) + +* Improve floating point generation to not use scientific notation as much. * Include line and column in parser errors. Both in the message and as exception attributes. * Handle non-string hash keys with broken `to_s` implementations. * `JSON.generate` now uses SSE2 (x86) or NEON (arm64) instructions when available to escape strings. diff --git a/lib/json/version.rb b/lib/json/version.rb index c8744c973..45d2b0a1f 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.11.3' + VERSION = '2.12.0' end From f3dde3cb2fb95ce683620865a265e6034f4c8f49 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 14:35:55 +0200 Subject: [PATCH 061/506] Add missing single quotes in error messages --- ext/json/ext/parser/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 13fd85219..93f2d1cf0 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1062,7 +1062,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got '%s", state); + raise_parse_error("expected object key, got '%s'", state); } json_parse_string(state, config, true); @@ -1103,7 +1103,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key, got: '%s", state); + raise_parse_error("expected ':' after object key, got: '%s'", state); } state->cursor++; From cc1daba860aba70b0e0bf7676d61991570c72cc1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 15:41:12 +0200 Subject: [PATCH 062/506] Further improve parsing errors Report EOF when applicable instead of an empty fragment. Also stop fragment extraction on first whitespace. --- ext/json/ext/parser/parser.c | 80 ++++++++++++++++++------------- test/json/json_ext_parser_test.rb | 21 ++++++-- test/json/json_parser_test.rb | 2 +- 3 files changed, 65 insertions(+), 38 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 93f2d1cf0..c5f300183 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -393,7 +393,7 @@ RBIMPL_ATTR_NORETURN() #endif static void raise_parse_error(const char *format, JSON_ParserState *state) { - unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 1]; + unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; const char *cursor = state->cursor; long column = 0; @@ -412,22 +412,34 @@ static void raise_parse_error(const char *format, JSON_ParserState *state) } } - const char *ptr = state->cursor; - size_t len = ptr ? strnlen(ptr, PARSE_ERROR_FRAGMENT_LEN) : 0; + const char *ptr = "EOF"; + if (state->cursor && state->cursor < state->end) { + ptr = state->cursor; + size_t len = 0; + while (len < PARSE_ERROR_FRAGMENT_LEN) { + char ch = ptr[len]; + if (!ch || ch == '\n' || ch == ' ' || ch == '\t' || ch == '\r') { + break; + } + len++; + } - if (len == PARSE_ERROR_FRAGMENT_LEN) { - MEMCPY(buffer, ptr, char, PARSE_ERROR_FRAGMENT_LEN); + if (len) { + buffer[0] = '\''; + MEMCPY(buffer + 1, ptr, char, len); - while (buffer[len - 1] >= 0x80 && buffer[len - 1] < 0xC0) { // Is continuation byte - len--; - } + while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte + len--; + } - if (buffer[len - 1] >= 0xC0) { // multibyte character start - len--; - } + if (buffer[len] >= 0xC0) { // multibyte character start + len--; + } - buffer[len] = '\0'; - ptr = (const char *)buffer; + buffer[len + 1] = '\''; + buffer[len + 2] = '\0'; + ptr = (const char *)buffer; + } } VALUE msg = rb_sprintf(format, ptr); @@ -473,16 +485,16 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p signed char b; uint32_t result = 0; b = digit_values[p[0]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[1]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[2]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; b = digit_values[p[3]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, (char *)p - 2); + if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); result = (result << 4) | (unsigned char)b; return result; } @@ -532,11 +544,11 @@ json_eat_comments(JSON_ParserState *state) break; } default: - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; } } else { - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); } } @@ -655,7 +667,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c break; case 'u': if (pe > stringEnd - 5) { - raise_parse_error_at("incomplete unicode character escape sequence at '%s'", state, p); + raise_parse_error_at("incomplete unicode character escape sequence at %s", state, p); } else { uint32_t ch = unescape_unicode(state, (unsigned char *) ++pe); pe += 3; @@ -672,7 +684,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c if ((ch & 0xFC00) == 0xD800) { pe++; if (pe > stringEnd - 6) { - raise_parse_error_at("incomplete surrogate pair at '%s'", state, p); + raise_parse_error_at("incomplete surrogate pair at %s", state, p); } if (pe[0] == '\\' && pe[1] == 'u') { uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2); @@ -894,7 +906,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qnil); } - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; case 't': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { @@ -902,7 +914,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qtrue); } - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; case 'f': // Note: memcmp with a small power of two compile to an integer comparison @@ -911,7 +923,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, Qfalse); } - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; case 'N': // Note: memcmp with a small power of two compile to an integer comparison @@ -920,7 +932,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CNaN); } - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; case 'I': if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { @@ -928,7 +940,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) return json_push_value(state, config, CInfinity); } - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); break; case '-': // Note: memcmp with a small power of two compile to an integer comparison @@ -937,7 +949,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor += 9; return json_push_value(state, config, CMinusInfinity); } else { - raise_parse_error("unexpected token '%s'", state); + raise_parse_error("unexpected token %s", state); } } // Fallthrough @@ -1062,7 +1074,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got '%s'", state); + raise_parse_error("expected object key, got %s", state); } json_parse_string(state, config, true); @@ -1097,13 +1109,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } if (*state->cursor != '"') { - raise_parse_error("expected object key, got: '%s'", state); + raise_parse_error("expected object key, got: %s", state); } json_parse_string(state, config, true); json_eat_whitespace(state); if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key, got: '%s'", state); + raise_parse_error("expected ':' after object key, got: %s", state); } state->cursor++; @@ -1113,24 +1125,24 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } } - raise_parse_error("expected ',' or '}' after object value, got: '%s'", state); + raise_parse_error("expected ',' or '}' after object value, got: %s", state); } break; } default: - raise_parse_error("unexpected character: '%s'", state); + raise_parse_error("unexpected character: %s", state); break; } - raise_parse_error("unreacheable: '%s'", state); + raise_parse_error("unreacheable: %s", state); } static void json_ensure_eof(JSON_ParserState *state) { json_eat_whitespace(state); if (state->cursor != state->end) { - raise_parse_error("unexpected token at end of stream '%s'", state); + raise_parse_error("unexpected token at end of stream %s", state); } } diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index 739798d05..e610f642f 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -14,20 +14,35 @@ def test_allocate end def test_error_messages - ex = assert_raise(ParserError) { parse('Infinity') } + ex = assert_raise(ParserError) { parse('Infinity something') } unless RUBY_PLATFORM =~ /java/ assert_equal "unexpected token 'Infinity' at line 1 column 1", ex.message end - ex = assert_raise(ParserError) { parse('-Infinity') } + ex = assert_raise(ParserError) { parse('foo bar') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected token 'foo' at line 1 column 1", ex.message + end + + ex = assert_raise(ParserError) { parse('-Infinity something') } unless RUBY_PLATFORM =~ /java/ assert_equal "unexpected token '-Infinity' at line 1 column 1", ex.message end - ex = assert_raise(ParserError) { parse('NaN') } + ex = assert_raise(ParserError) { parse('NaN something') } unless RUBY_PLATFORM =~ /java/ assert_equal "unexpected token 'NaN' at line 1 column 1", ex.message end + + ex = assert_raise(ParserError) { parse(' ') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "unexpected end of input at line 1 column 4", ex.message + end + + ex = assert_raise(ParserError) { parse('{ ') } + unless RUBY_PLATFORM =~ /java/ + assert_equal "expected object key, got EOF at line 1 column 5", ex.message + end end if GC.respond_to?(:stress=) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 7cfdfc69d..befc80c95 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -646,7 +646,7 @@ def test_parse_error_incomplete_hash JSON.parse('{"input":{"firstName":"Bob","lastName":"Mob","email":"bob@example.com"}') end if RUBY_ENGINE == "ruby" - assert_equal %(expected ',' or '}' after object value, got: '' at line 1 column 72), error.message + assert_equal %(expected ',' or '}' after object value, got: EOF at line 1 column 72), error.message end end From 3226913a577a0543bb59be426e47033879835b57 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 May 2025 19:27:51 +0200 Subject: [PATCH 063/506] Explicit cast down from `double` to `int` Co-Authored-By: Nobuyoshi Nakada --- ext/json/ext/vendor/fpconv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 440759e5a..1bbca2873 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -92,7 +92,7 @@ static Fp find_cachedpow10(int exp, int* k) { const double one_log_ten = 0.30102999566398114; - int approx = -(exp + npowers) * one_log_ten; + int approx = (int)(-(exp + npowers) * one_log_ten); int idx = (approx - firstpower) / steppowers; while(1) { From 8433571dcfae8587fd2ef6ed7d47ff46a8298ccf Mon Sep 17 00:00:00 2001 From: Cody Horton Date: Fri, 16 May 2025 09:58:27 -0500 Subject: [PATCH 064/506] fix for pretty_generate throwing wrong number of arguments error --- CHANGES.md | 2 ++ lib/json/common.rb | 2 +- test/json/json_generator_test.rb | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4f91abd47..9fb7627ef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix for JSON.pretty_generate to use passed state object's generate instead of state class as the required parameters aren't available. + ### 2025-05-12 (2.12.0) * Improve floating point generation to not use scientific notation as much. diff --git a/lib/json/common.rb b/lib/json/common.rb index 7627761b5..320df2a94 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -490,7 +490,7 @@ def fast_generate(obj, opts = nil) # } # def pretty_generate(obj, opts = nil) - return state.generate(obj) if State === opts + return opts.generate(obj) if State === opts options = PRETTY_GENERATE_OPTIONS diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 55a3065ae..f869e43fb 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -122,6 +122,22 @@ def test_generate_pretty assert_equal '666', pretty_generate(666) end + def test_generate_pretty_custom + state = State.new(:space_before => "", :space => "", :indent => "", :object_nl => "\n\n", :array_nl => "") + json = pretty_generate({1=>{}, 2=>['a','b'], 3=>4}, state) + assert_equal(<<~'JSON'.chomp, json) + { + + "1":{}, + + "2":["a","b"], + + "3":4 + + } + JSON + end + def test_generate_custom state = State.new(:space_before => " ", :space => " ", :indent => "", :object_nl => "\n", :array_nl => "") json = generate({1=>{2=>3,4=>[5,6]}}, state) From 0ac54a8161544e03927e7360614e0aa0f5c76fec Mon Sep 17 00:00:00 2001 From: Grant Birkinbine Date: Sat, 17 May 2025 10:30:41 -0700 Subject: [PATCH 065/506] Update json_encoding_test.rb --- test/json/json_encoding_test.rb | 184 +++++++++++++++++++++++++++++++- 1 file changed, 180 insertions(+), 4 deletions(-) diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index afffd8976..873e96fdd 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative 'test_helper' class JSONEncodingTest < Test::Unit::TestCase @@ -37,7 +38,7 @@ def test_unicode assert_equal '"\u001f"', 0x1f.chr.to_json assert_equal '" "', ' '.to_json assert_equal "\"#{0x7f.chr}\"", 0x7f.chr.to_json - utf8 = [ "© ≠ €! \01" ] + utf8 = ["© ≠ €! \01"] json = '["© ≠ €! \u0001"]' assert_equal json, utf8.to_json(ascii_only: false) assert_equal utf8, parse(json) @@ -78,10 +79,10 @@ def test_chars json = '"\u%04x"' % i i = i.chr assert_equal i, parse(json)[0] - if i == ?\b + if i == "\b" generated = generate(i) - assert '"\b"' == generated || '"\10"' == generated - elsif [?\n, ?\r, ?\t, ?\f].include?(i) + assert ['"\b"', '"\10"'].include?(generated) + elsif ["\n", "\r", "\t", "\f"].include?(i) assert_equal i.dump, generate(i) elsif i.chr < 0x20.chr assert_equal json, generate(i) @@ -92,4 +93,179 @@ def test_chars end assert_equal "\302\200", parse('"\u0080"') end + + def test_deeply_nested_structures + # Test for deeply nested arrays + nesting_level = 100 + deeply_nested = [] + current = deeply_nested + + (nesting_level - 1).times do + current << [] + current = current[0] + end + + json = generate(deeply_nested) + assert_equal deeply_nested, parse(json) + + # Test for deeply nested objects/hashes + deeply_nested_hash = {} + current_hash = deeply_nested_hash + + (nesting_level - 1).times do |i| + current_hash["key#{i}"] = {} + current_hash = current_hash["key#{i}"] + end + + json = generate(deeply_nested_hash) + assert_equal deeply_nested_hash, parse(json) + end + + def test_very_large_json_strings + # Create a large array with repeated elements + large_array = Array.new(10_000) { |i| "item#{i}" } + + json = generate(large_array) + parsed = parse(json) + + assert_equal large_array.size, parsed.size + assert_equal large_array.first, parsed.first + assert_equal large_array.last, parsed.last + + # Create a large hash + large_hash = {} + 10_000.times { |i| large_hash["key#{i}"] = "value#{i}" } + + json = generate(large_hash) + parsed = parse(json) + + assert_equal large_hash.size, parsed.size + assert_equal large_hash["key0"], parsed["key0"] + assert_equal large_hash["key9999"], parsed["key9999"] + end + + def test_invalid_utf8_sequences + # Create strings with invalid UTF-8 sequences + invalid_utf8 = "\xFF\xFF" + + # Test that generating JSON with invalid UTF-8 raises an error + # Different JSON implementations may handle this differently, + # so we'll check if any exception is raised + begin + generate(invalid_utf8) + raise "Expected an exception when generating JSON with invalid UTF8" + rescue StandardError => e + assert true + assert_match(%r{source sequence is illegal/malformed utf-8}, e.message) + end + end + + def test_surrogate_pair_handling + # Test valid surrogate pairs + assert_equal "\u{10000}", parse('"\ud800\udc00"') + assert_equal "\u{10FFFF}", parse('"\udbff\udfff"') + + # The existing test already checks for orphaned high surrogate + assert_raise(JSON::ParserError) { parse('"\ud800"') } + + # Test generating surrogate pairs + utf8_string = "\u{10437}" + generated = generate(utf8_string, ascii_only: true) + assert_match(/\\ud801\\udc37/, generated) + end + + def test_json_escaping_edge_cases + # Test escaping forward slashes + assert_equal "/", parse('"\/"') + + # Test escaping backslashes + assert_equal "\\", parse('"\\\\"') + + # Test escaping quotes + assert_equal '"', parse('"\\""') + + # Multiple escapes in sequence - different JSON parsers might handle escaped forward slashes differently + # Some parsers preserve the escaping, others don't + escaped_result = parse('"\\\\\\"\\/"') + assert_match(/\\"/, escaped_result) + assert_match(%r{/}, escaped_result) + + # Generate string with all special characters + special_chars = "\b\f\n\r\t\"\\" + escaped_json = generate(special_chars) + assert_equal special_chars, parse(escaped_json) + end + + def test_empty_objects_and_arrays + # Test empty objects with different encodings + assert_equal({}, parse('{}')) + assert_equal({}, parse('{}'.encode(Encoding::UTF_16BE))) + assert_equal({}, parse('{}'.encode(Encoding::UTF_16LE))) + assert_equal({}, parse('{}'.encode(Encoding::UTF_32BE))) + assert_equal({}, parse('{}'.encode(Encoding::UTF_32LE))) + + # Test empty arrays with different encodings + assert_equal([], parse('[]')) + assert_equal([], parse('[]'.encode(Encoding::UTF_16BE))) + assert_equal([], parse('[]'.encode(Encoding::UTF_16LE))) + assert_equal([], parse('[]'.encode(Encoding::UTF_32BE))) + assert_equal([], parse('[]'.encode(Encoding::UTF_32LE))) + + # Test generating empty objects and arrays + assert_equal '{}', generate({}) + assert_equal '[]', generate([]) + end + + def test_null_character_handling + # Test parsing null character + assert_equal "\u0000", parse('"\u0000"') + + # Test generating null character + string_with_null = "\u0000" + generated = generate(string_with_null) + assert_equal '"\u0000"', generated + + # Test null characters in middle of string + mixed_string = "before\u0000after" + generated = generate(mixed_string) + assert_equal mixed_string, parse(generated) + end + + def test_whitespace_handling + # Test parsing with various whitespace patterns + assert_equal({}, parse(' { } ')) + assert_equal({}, parse("{\r\n}")) + assert_equal([], parse(" [ \n ] ")) + assert_equal(["a", "b"], parse(" [ \n\"a\",\r\n \"b\"\n ] ")) + assert_equal({ "a" => "b" }, parse(" { \n\"a\" \r\n: \t\"b\"\n } ")) + + # Test with excessive whitespace + excessive_whitespace = " \n\r\t" * 10 + "{}" + " \n\r\t" * 10 + assert_equal({}, parse(excessive_whitespace)) + + # Mixed whitespace in keys and values + mixed_json = '{"a \n b":"c \r\n d"}' + assert_equal({ "a \n b" => "c \r\n d" }, parse(mixed_json)) + end + + def test_control_character_handling + # Test all control characters (U+0000 to U+001F) + (0..0x1F).each do |i| + # Skip already tested ones + next if [0x08, 0x0A, 0x0D, 0x0C, 0x09].include?(i) + + control_char = i.chr('UTF-8') + escaped_json = '"' + "\\u%04x" % i + '"' + assert_equal control_char, parse(escaped_json) + + # Check that the character is properly escaped when generating + assert_match(/\\u00[0-1][0-9a-f]/, generate(control_char)) + end + + # Test string with multiple control characters + control_str = "\u0001\u0002\u0003\u0004" + generated = generate(control_str) + assert_equal control_str, parse(generated) + assert_match(/\\u0001\\u0002\\u0003\\u0004/, generated) + end end From 912d17953b4a25eddea626d2682ea131e102981d Mon Sep 17 00:00:00 2001 From: GrantBirki Date: Sat, 17 May 2025 11:24:41 -0700 Subject: [PATCH 066/506] update the readme to include bootstrap + testing docs --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index d327f74a1..119327213 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,19 @@ the `pp` library's `pp` methods. ## Development +### Prerequisites + +1. Clone the repository +2. Install dependencies with `bundle install` + +### Testing + +The full test suite can be run with: + +```bash +bundle exec rake test +``` + ### Release Update the `lib/json/version.rb` file. From f5c1b8c45d5b2bd70d38aa6b32ebb033980a9bdf Mon Sep 17 00:00:00 2001 From: GrantBirki Date: Sat, 17 May 2025 11:35:55 -0700 Subject: [PATCH 067/506] use `.` over `::` for consistency --- lib/json/common.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 320df2a94..d7f0eb885 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1072,7 +1072,7 @@ def j(*objs) end objs.each do |obj| - puts JSON::generate(obj, :allow_nan => true, :max_nesting => false) + puts JSON.generate(obj, :allow_nan => true, :max_nesting => false) end nil end @@ -1087,7 +1087,7 @@ def jj(*objs) end objs.each do |obj| - puts JSON::pretty_generate(obj, :allow_nan => true, :max_nesting => false) + puts JSON.pretty_generate(obj, :allow_nan => true, :max_nesting => false) end nil end From c060943d042ac7a1f68e4b6c691a12646a467acb Mon Sep 17 00:00:00 2001 From: GrantBirki Date: Sat, 17 May 2025 11:37:39 -0700 Subject: [PATCH 068/506] remove redundant `self.` --- lib/json/common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index d7f0eb885..6393a6df5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -172,7 +172,7 @@ def generator=(generator) # :nodoc: end end self.state = generator::State - const_set :State, self.state + const_set :State, state ensure $VERBOSE = old end From 7c03ffc3e0a388ecf390f859221a047e2a21da46 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 17 May 2025 21:02:04 +0200 Subject: [PATCH 069/506] Remove some unnecessary top level constant lookups --- lib/json/ext.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 1db5ea122..5bacc5e37 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -34,12 +34,12 @@ def parse if RUBY_ENGINE == 'truffleruby' require 'json/truffle_ruby/generator' - JSON.generator = ::JSON::TruffleRuby::Generator + JSON.generator = JSON::TruffleRuby::Generator else require 'json/ext/generator' JSON.generator = Generator end end - JSON_LOADED = true unless defined?(::JSON::JSON_LOADED) + JSON_LOADED = true unless defined?(JSON::JSON_LOADED) end From d73ae93d3cf7f58a83d4aef4392f48c4de8c11a5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 23 May 2025 09:28:56 +0200 Subject: [PATCH 070/506] Fix: generate_json_float to reserve enough memory for large negative floats. Fix: https://github.com/ruby/json/issues/807 Since https://github.com/ruby/json/pull/800, `fpconv_dtoa` can actually generate up to 28 chars. --- CHANGES.md | 2 ++ ext/json/ext/generator/generator.c | 5 ++--- ext/json/ext/vendor/fpconv.c | 8 ++++---- test/json/json_generator_test.rb | 5 +++++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9fb7627ef..0eec257cb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Fix a potential crash in large negative floating point number generation. + ### Unreleased * Fix for JSON.pretty_generate to use passed state object's generate instead of state class as the required parameters aren't available. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 06ab8010d..e67351f8b 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1406,10 +1406,9 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data } /* This implementation writes directly into the buffer. We reserve - * the 24 characters that fpconv_dtoa states as its maximum, plus - * 2 more characters for the potential ".0" suffix. + * the 28 characters that fpconv_dtoa states as its maximum. */ - fbuffer_inc_capa(buffer, 26); + fbuffer_inc_capa(buffer, 28); char* d = buffer->ptr + buffer->len; int len = fpconv_dtoa(value, d); diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 1bbca2873..75efd46f1 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -432,8 +432,8 @@ static int filter_special(double fp, char* dest) * * Input: * fp -> the double to convert, dest -> destination buffer. - * The generated string will never be longer than 24 characters. - * Make sure to pass a pointer to at least 24 bytes of memory. + * The generated string will never be longer than 28 characters. + * Make sure to pass a pointer to at least 28 bytes of memory. * The emitted string will not be null terminated. * * Output: @@ -443,7 +443,7 @@ static int filter_special(double fp, char* dest) * * void print(double d) * { - * char buf[24 + 1] // plus null terminator + * char buf[28 + 1] // plus null terminator * int str_len = fpconv_dtoa(d, buf); * * buf[str_len] = '\0'; @@ -451,7 +451,7 @@ static int filter_special(double fp, char* dest) * } * */ -static int fpconv_dtoa(double d, char dest[24]) +static int fpconv_dtoa(double d, char dest[28]) { char digits[18]; diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f869e43fb..914b3f4ed 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -795,6 +795,11 @@ def test_json_generate_float expecteds << "1746861937.7842371" end + if RUBY_ENGINE == "ruby" + values << -2.2471348024634545e-08 << -2.2471348024634545e-09 << -2.2471348024634545e-10 + expecteds << "-0.000000022471348024634545" << "-0.0000000022471348024634545" << "-2.2471348024634546e-10" + end + values.zip(expecteds).each do |value, expected| assert_equal expected, value.to_json end From 8109421fb428a56877100ebed38001783abfb820 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 23 May 2025 11:27:29 +0200 Subject: [PATCH 071/506] fbuffer.c: add debug mode with bound checks. This would have caught https://github.com/ruby/json/pull/808 on CI. --- .github/workflows/ci.yml | 1 + ext/json/ext/fbuffer/fbuffer.h | 42 +++++++++++++++++++++++++++--- ext/json/ext/generator/extconf.rb | 3 ++- ext/json/ext/generator/generator.c | 6 ++--- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f82ae90..27a40bdd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: include: - { os: ubuntu-24.04-arm, ruby: 3.4 } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } + - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } - { os: macos-13, ruby: 3.4 } - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index b8a4e983d..d32371476 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -36,6 +36,12 @@ typedef unsigned char _Bool; # define MAYBE_UNUSED(x) x #endif +#ifdef RUBY_DEBUG +#ifndef JSON_DEBUG +#define JSON_DEBUG RUBY_DEBUG +#endif +#endif + enum fbuffer_type { FBUFFER_HEAP_ALLOCATED = 0, FBUFFER_STACK_ALLOCATED = 1, @@ -46,6 +52,9 @@ typedef struct FBufferStruct { unsigned long initial_length; unsigned long len; unsigned long capa; +#ifdef JSON_DEBUG + unsigned long requested; +#endif char *ptr; VALUE io; } FBuffer; @@ -74,6 +83,20 @@ static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char * fb->ptr = stack_buffer; fb->capa = stack_buffer_size; } +#ifdef JSON_DEBUG + fb->requested = 0; +#endif +} + +static inline void fbuffer_consumed(FBuffer *fb, unsigned long consumed) +{ +#ifdef JSON_DEBUG + if (consumed > fb->requested) { + rb_bug("fbuffer: Out of bound write"); + } + fb->requested = 0; +#endif + fb->len += consumed; } static void fbuffer_free(FBuffer *fb) @@ -137,6 +160,10 @@ static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested) static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested) { +#ifdef JSON_DEBUG + fb->requested = requested; +#endif + if (RB_UNLIKELY(requested > fb->capa - fb->len)) { fbuffer_do_inc_capa(fb, requested); } @@ -147,15 +174,22 @@ static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len) if (len > 0) { fbuffer_inc_capa(fb, len); MEMCPY(fb->ptr + fb->len, newstr, char, len); - fb->len += len; + fbuffer_consumed(fb, len); } } /* Appends a character into a buffer. The buffer needs to have sufficient capacity, via fbuffer_inc_capa(...). */ static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) { +#ifdef JSON_DEBUG + if (fb->requested < 1) { + rb_bug("fbuffer: unreserved write"); + } + fb->requested--; +#endif + fb->ptr[fb->len] = chr; - fb->len += 1; + fb->len++; } static void fbuffer_append_str(FBuffer *fb, VALUE str) @@ -172,7 +206,7 @@ static inline void fbuffer_append_char(FBuffer *fb, char newchr) { fbuffer_inc_capa(fb, 1); *(fb->ptr + fb->len) = newchr; - fb->len++; + fbuffer_consumed(fb, 1); } static inline char *fbuffer_cursor(FBuffer *fb) @@ -182,7 +216,7 @@ static inline char *fbuffer_cursor(FBuffer *fb) static inline void fbuffer_advance_to(FBuffer *fb, char *end) { - fb->len = end - fb->ptr; + fbuffer_consumed(fb, (end - fb->ptr) - fb->len); } /* diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 60372ee55..26a8c2ba4 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -4,8 +4,9 @@ # The pure-Ruby generator is faster on TruffleRuby, so skip compiling the generator extension File.write('Makefile', dummy_makefile("").join) else - append_cflags("-std=c99") + append_cflags("-std=c99 -O0") $defs << "-DJSON_GENERATOR" + $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] if enable_config('generator-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index e67351f8b..f7690a23e 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -404,7 +404,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) if (!mask) { // Nothing to escape, ensure search_flush doesn't do anything by setting // search->cursor to search->ptr. - search->buffer->len += remaining; + fbuffer_consumed(search->buffer, remaining); search->ptr = search->end; search->cursor = search->end; return 0; @@ -511,7 +511,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se if (needs_escape_mask == 0) { // Nothing to escape, ensure search_flush doesn't do anything by setting // search->cursor to search->ptr. - search->buffer->len += remaining; + fbuffer_consumed(search->buffer, remaining); search->ptr = search->end; search->cursor = search->end; return 0; @@ -1415,7 +1415,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data /* fpconv_dtoa converts a float to its shortest string representation, * but it adds a ".0" if this is a plain integer. */ - buffer->len += len; + fbuffer_consumed(buffer, len); } static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj) From 8603a57a919eb8718955f37f33eb64d8bc9b7492 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 23 May 2025 11:36:41 +0200 Subject: [PATCH 072/506] Release 2.12.1 --- CHANGES.md | 5 +++-- lib/json/version.rb | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0eec257cb..4fd34a59b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,10 @@ # Changes -* Fix a potential crash in large negative floating point number generation. - ### Unreleased +### 2025-05-23 (2.12.1) + +* Fix a potential crash in large negative floating point number generation. * Fix for JSON.pretty_generate to use passed state object's generate instead of state class as the required parameters aren't available. ### 2025-05-12 (2.12.0) diff --git a/lib/json/version.rb b/lib/json/version.rb index 45d2b0a1f..e4c6a8945 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.12.0' + VERSION = '2.12.1' end From a29cb77d5234c504f34e326ed6eb826997ffdd05 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 23 May 2025 17:07:39 +0200 Subject: [PATCH 073/506] Release 2.12.2 --- CHANGES.md | 4 ++++ ext/json/ext/generator/extconf.rb | 2 +- lib/json/version.rb | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4fd34a59b..b02ee1975 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ### Unreleased +### 2025-05-23 (2.12.2) + +* Fix compiler optimization level. + ### 2025-05-23 (2.12.1) * Fix a potential crash in large negative floating point number generation. diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 26a8c2ba4..f58574a6c 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -4,7 +4,7 @@ # The pure-Ruby generator is faster on TruffleRuby, so skip compiling the generator extension File.write('Makefile', dummy_makefile("").join) else - append_cflags("-std=c99 -O0") + append_cflags("-std=c99") $defs << "-DJSON_GENERATOR" $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] diff --git a/lib/json/version.rb b/lib/json/version.rb index e4c6a8945..15ebd12f5 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.12.1' + VERSION = '2.12.2' end From 90f00b1ce1252d1d7873119c53b3b5560375db4d Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Tue, 27 May 2025 03:58:04 +0900 Subject: [PATCH 074/506] `Ractor::Port` * Added `Ractor::Port` * `Ractor::Port#receive` (support multi-threads) * `Rcator::Port#close` * `Ractor::Port#closed?` * Added some methods * `Ractor#join` * `Ractor#value` * `Ractor#monitor` * `Ractor#unmonitor` * Removed some methods * `Ractor#take` * `Ractor.yield` * Change the spec * `Racotr.select` You can wait for multiple sequences of messages with `Ractor::Port`. ```ruby ports = 3.times.map{ Ractor::Port.new } ports.map.with_index do |port, ri| Ractor.new port,ri do |port, ri| 3.times{|i| port << "r#{ri}-#{i}"} end end p ports.each{|port| pp 3.times.map{port.receive}} ``` In this example, we use 3 ports, and 3 Ractors send messages to them respectively. We can receive a series of messages from each port. You can use `Ractor#value` to get the last value of a Ractor's block: ```ruby result = Ractor.new do heavy_task() end.value ``` You can wait for the termination of a Ractor with `Ractor#join` like this: ```ruby Ractor.new do some_task() end.join ``` `#value` and `#join` are similar to `Thread#value` and `Thread#join`. To implement `#join`, `Ractor#monitor` (and `Ractor#unmonitor`) is introduced. This commit changes `Ractor.select()` method. It now only accepts ports or Ractors, and returns when a port receives a message or a Ractor terminates. We removes `Ractor.yield` and `Ractor#take` because: * `Ractor::Port` supports most of similar use cases in a simpler manner. * Removing them significantly simplifies the code. We also change the internal thread scheduler code (thread_pthread.c): * During barrier synchronization, we keep the `ractor_sched` lock to avoid deadlocks. This lock is released by `rb_ractor_sched_barrier_end()` which is called at the end of operations that require the barrier. * fix potential deadlock issues by checking interrupts just before setting UBF. https://bugs.ruby-lang.org/issues/21262 --- test/json/ractor_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index f857c9a8b..ced901bc5 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -25,7 +25,7 @@ def test_generate end expected_json = JSON.parse('{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},' + '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}') - actual_json = r.take + actual_json = r.value if expected_json == actual_json exit 0 From d42b36963d71d0eb4355e49699dd5d9a003c9ffc Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 3 Jun 2025 08:41:53 +0200 Subject: [PATCH 075/506] Update `JSONInRactorTest` to handle Ruby 3.5 Ractors. --- test/json/ractor_test.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index ced901bc5..dda34c64c 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -8,6 +8,16 @@ end class JSONInRactorTest < Test::Unit::TestCase + unless Ractor.method_defined?(:value) + module RactorBackport + refine Ractor do + alias_method :value, :take + end + end + + using RactorBackport + end + def test_generate pid = fork do r = Ractor.new do From 6fc2c4b6ab5347c8a2924bf50771a824658b0344 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 17 Jun 2025 17:01:34 +0900 Subject: [PATCH 076/506] Fix a typo ruby/ruby#13636 Co-Authored-By: Tim Smith --- lib/json/common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 6393a6df5..486ec62a5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -268,7 +268,7 @@ def detailed_message(...) # to string interpolation. # # Note: no validation is performed on the provided string. It is the - # responsability of the caller to ensure the string contains valid JSON. + # responsibility of the caller to ensure the string contains valid JSON. Fragment = Struct.new(:json) do def initialize(json) unless string = String.try_convert(json) From eed753ffdeeda504b5889ecbe9f3e00b436bb49c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Jun 2025 16:22:06 +0100 Subject: [PATCH 077/506] Add missing parser options documentation --- lib/json.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/json.rb b/lib/json.rb index dfd9b7dfc..264301e05 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -143,8 +143,23 @@ # ruby = JSON.parse(source, {allow_nan: true}) # ruby # => [NaN, Infinity, -Infinity] # +# --- +# +# Option +allow_trailing_comma+ (boolean) specifies whether to allow +# trailing commas in objects and arrays; +# defaults to +false+. +# +# With the default, +false+: +# JSON.parse('[1,]') # unexpected character: ']' at line 1 column 4 (JSON::ParserError) +# +# When enabled: +# JSON.parse('[1,]', allow_trailing_comma: true) # => [1] +# # ====== Output Options # +# Option +freeze+ (boolean) specifies whether the returned objects will be frozen; +# defaults to +false+. +# # Option +symbolize_names+ (boolean) specifies whether returned \Hash keys # should be Symbols; # defaults to +false+ (use Strings). From 06f00a42e8841cb768bf78514cf49b0d8cd63c27 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Jun 2025 16:11:33 +0100 Subject: [PATCH 078/506] Deprecate duplicate keys in object There are few legitimate use cases for duplicate keys, and can in some case be exploited. Rather to always silently accept them, we should emit a warning, and in the future require to explictly allow them. --- CHANGES.md | 3 + ext/json/ext/parser/parser.c | 72 +++++++++-- java/src/json/ext/OptionsReader.java | 7 ++ java/src/json/ext/ParserConfig.java | 175 ++++++++++++++++----------- java/src/json/ext/ParserConfig.rl | 29 ++++- lib/json.rb | 18 +++ test/json/json_parser_test.rb | 9 ++ 7 files changed, 225 insertions(+), 88 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b02ee1975..2208bd7db 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +* Add bnew `allow_duplicate_key` parsing options. By default a warning is now emitted when a duplicated key is encountered. + In `json 3.0` an error will be raised. + ### 2025-05-23 (2.12.2) * Fix compiler optimization level. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c5f300183..627971eb5 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -35,7 +35,7 @@ static ID i_chr, i_aset, i_aref, i_leftshift, i_new, i_try_convert, i_uminus, i_encode; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze, - sym_decimal_class, sym_on_load; + sym_decimal_class, sym_on_load, sym_allow_duplicate_key; static int binary_encindex; static int utf8_encindex; @@ -363,10 +363,17 @@ static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) return len; } +enum duplicate_key_action { + JSON_DEPRECATED = 0, + JSON_IGNORE, + JSON_RAISE, +}; + typedef struct JSON_ParserStruct { VALUE on_load_proc; VALUE decimal_class; ID decimal_method_id; + enum duplicate_key_action on_duplicate_key; int max_nesting; bool allow_nan; bool allow_trailing_comma; @@ -386,15 +393,8 @@ typedef struct JSON_ParserStateStruct { int current_nesting; } JSON_ParserState; - -#define PARSE_ERROR_FRAGMENT_LEN 32 -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_parse_error(const char *format, JSON_ParserState *state) +static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out) { - unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; - const char *cursor = state->cursor; long column = 0; long line = 1; @@ -411,6 +411,27 @@ static void raise_parse_error(const char *format, JSON_ParserState *state) line++; } } + *line_out = line; + *column_out = column; +} + +static void emit_parse_warning(const char *message, JSON_ParserState *state) +{ + long line, column; + cursor_position(state, &line, &column); + + rb_warn("%s at line %ld column %ld", message, line, column); +} + +#define PARSE_ERROR_FRAGMENT_LEN 32 +#ifdef RBIMPL_ATTR_NORETURN +RBIMPL_ATTR_NORETURN() +#endif +static void raise_parse_error(const char *format, JSON_ParserState *state) +{ + unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; + long line, column; + cursor_position(state, &line, &column); const char *ptr = "EOF"; if (state->cursor && state->cursor < state->end) { @@ -807,11 +828,25 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig return array; } -static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, long count) +static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count) { - VALUE object = rb_hash_new_capa(count); + size_t entries_count = count / 2; + VALUE object = rb_hash_new_capa(entries_count); rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object); + if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) { + switch (config->on_duplicate_key) { + case JSON_IGNORE: + break; + case JSON_DEPRECATED: + emit_parse_warning("detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", state); + break; + case JSON_RAISE: + raise_parse_error("duplicate key", state); + break; + } + } + rvalue_stack_pop(state->stack, count); if (config->freeze) { @@ -1060,6 +1095,8 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; } case '{': { + const char *object_start_cursor = state->cursor; + state->cursor++; json_eat_whitespace(state); long stack_head = state->stack->head; @@ -1094,8 +1131,15 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if (*state->cursor == '}') { state->cursor++; state->current_nesting--; - long count = state->stack->head - stack_head; - return json_push_value(state, config, json_decode_object(state, config, count)); + size_t count = state->stack->head - stack_head; + + // Temporary rewind cursor in case an error is raised + const char *final_cursor = state->cursor; + state->cursor = object_start_cursor; + VALUE object = json_decode_object(state, config, count); + state->cursor = final_cursor; + + return json_push_value(state, config, object); } if (*state->cursor == ',') { @@ -1184,6 +1228,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } + else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { @@ -1400,6 +1445,7 @@ void Init_parser(void) sym_freeze = ID2SYM(rb_intern("freeze")); sym_on_load = ID2SYM(rb_intern("on_load")); sym_decimal_class = ID2SYM(rb_intern("decimal_class")); + sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); i_chr = rb_intern("chr"); i_aset = rb_intern("[]="); diff --git a/java/src/json/ext/OptionsReader.java b/java/src/json/ext/OptionsReader.java index c372eb1a8..829e36b31 100644 --- a/java/src/json/ext/OptionsReader.java +++ b/java/src/json/ext/OptionsReader.java @@ -53,6 +53,13 @@ IRubyObject get(String key) { return opts == null ? null : opts.fastARef(runtime.newSymbol(key)); } + boolean hasKey(String key) { + if (opts == null) { + return false; + } + return opts.hasKey(runtime.newSymbol(key)); + } + boolean getBool(String key, boolean defaultValue) { IRubyObject value = get(key); return value == null ? defaultValue : value.isTrue(); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 692b3dfaa..ccfc558e5 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -54,6 +54,8 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowDuplicateKey; + private boolean deprecateDuplicateKey; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; @@ -177,6 +179,14 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.allowNaN = opts.getBool("allow_nan", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); + if (opts.hasKey("allow_duplicate_key")) { + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.deprecateDuplicateKey = false; + } else { + this.allowDuplicateKey = false; + this.deprecateDuplicateKey = true; + } + this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); @@ -280,19 +290,23 @@ private ParserSession(ParserConfig config, RubyString source, ThreadContext cont this.decoder = new StringDecoder(); } - private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { + private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { RubyString msg = context.runtime.newString("unexpected token at '") .cat(data, absStart, Math.min(absEnd - absStart, 32)) .cat((byte)'\''); return newException(context, Utils.M_PARSER_ERROR, msg); } + private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { + return parsingError(context, "unexpected token at '", absStart, absEnd); + } + -// line 314 "ParserConfig.rl" +// line 328 "ParserConfig.rl" -// line 296 "ParserConfig.java" +// line 310 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -406,7 +420,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 420 "ParserConfig.rl" +// line 434 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -414,14 +428,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 418 "ParserConfig.java" +// line 432 "ParserConfig.java" { cs = JSON_value_start; } -// line 427 "ParserConfig.rl" +// line 441 "ParserConfig.rl" -// line 425 "ParserConfig.java" +// line 439 "ParserConfig.java" { int _klen; int _trans = 0; @@ -447,13 +461,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 405 "ParserConfig.rl" +// line 419 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 457 "ParserConfig.java" +// line 471 "ParserConfig.java" } } @@ -516,25 +530,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 322 "ParserConfig.rl" +// line 336 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 325 "ParserConfig.rl" +// line 339 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 328 "ParserConfig.rl" +// line 342 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 331 "ParserConfig.rl" +// line 345 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -544,7 +558,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 338 "ParserConfig.rl" +// line 352 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -554,7 +568,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 345 "ParserConfig.rl" +// line 359 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -583,7 +597,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 371 "ParserConfig.rl" +// line 385 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -596,7 +610,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 381 "ParserConfig.rl" +// line 395 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -611,7 +625,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 393 "ParserConfig.rl" +// line 407 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -625,7 +639,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 629 "ParserConfig.java" +// line 643 "ParserConfig.java" } } } @@ -645,7 +659,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 428 "ParserConfig.rl" +// line 442 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -658,7 +672,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 662 "ParserConfig.java" +// line 676 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -757,7 +771,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 450 "ParserConfig.rl" +// line 464 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -774,15 +788,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 778 "ParserConfig.java" +// line 792 "ParserConfig.java" { cs = JSON_integer_start; } -// line 466 "ParserConfig.rl" +// line 480 "ParserConfig.rl" int memo = p; -// line 786 "ParserConfig.java" +// line 800 "ParserConfig.java" { int _klen; int _trans = 0; @@ -863,13 +877,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 444 "ParserConfig.rl" +// line 458 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 873 "ParserConfig.java" +// line 887 "ParserConfig.java" } } } @@ -889,7 +903,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 468 "ParserConfig.rl" +// line 482 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -909,7 +923,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 913 "ParserConfig.java" +// line 927 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1011,7 +1025,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 501 "ParserConfig.rl" +// line 515 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1030,15 +1044,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1034 "ParserConfig.java" +// line 1048 "ParserConfig.java" { cs = JSON_float_start; } -// line 519 "ParserConfig.rl" +// line 533 "ParserConfig.rl" int memo = p; -// line 1042 "ParserConfig.java" +// line 1056 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1119,13 +1133,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 492 "ParserConfig.rl" +// line 506 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1129 "ParserConfig.java" +// line 1143 "ParserConfig.java" } } } @@ -1145,7 +1159,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 521 "ParserConfig.rl" +// line 535 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1155,7 +1169,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1159 "ParserConfig.java" +// line 1173 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1257,7 +1271,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 560 "ParserConfig.rl" +// line 574 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1265,15 +1279,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1269 "ParserConfig.java" +// line 1283 "ParserConfig.java" { cs = JSON_string_start; } -// line 567 "ParserConfig.rl" +// line 581 "ParserConfig.rl" int memo = p; -// line 1277 "ParserConfig.java" +// line 1291 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1354,7 +1368,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 535 "ParserConfig.rl" +// line 549 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1369,13 +1383,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 548 "ParserConfig.rl" +// line 562 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1379 "ParserConfig.java" +// line 1393 "ParserConfig.java" } } } @@ -1395,7 +1409,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 569 "ParserConfig.rl" +// line 583 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1416,7 +1430,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1420 "ParserConfig.java" +// line 1434 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1583,7 +1597,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 623 "ParserConfig.rl" +// line 637 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1597,14 +1611,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); -// line 1601 "ParserConfig.java" +// line 1615 "ParserConfig.java" { cs = JSON_array_start; } -// line 636 "ParserConfig.rl" +// line 650 "ParserConfig.rl" -// line 1608 "ParserConfig.java" +// line 1622 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1647,7 +1661,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 594 "ParserConfig.rl" +// line 608 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1717,7 +1731,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 596 "ParserConfig.rl" +// line 610 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1730,13 +1744,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 607 "ParserConfig.rl" +// line 621 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1740 "ParserConfig.java" +// line 1754 "ParserConfig.java" } } } @@ -1756,7 +1770,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 637 "ParserConfig.rl" +// line 651 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1766,7 +1780,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1770 "ParserConfig.java" +// line 1784 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1943,7 +1957,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 694 "ParserConfig.rl" +// line 721 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -1960,14 +1974,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyHash.newHash(context.runtime); -// line 1964 "ParserConfig.java" +// line 1978 "ParserConfig.java" { cs = JSON_object_start; } -// line 710 "ParserConfig.rl" +// line 737 "ParserConfig.rl" -// line 1971 "ParserConfig.java" +// line 1985 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2010,7 +2024,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 651 "ParserConfig.rl" +// line 665 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2080,7 +2094,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 653 "ParserConfig.rl" +// line 667 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2093,7 +2107,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 664 "ParserConfig.rl" +// line 678 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2106,18 +2120,31 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } else { lastName = name; } + + if (!config.allowDuplicateKey) { + if (((RubyHash)result).hasKey(lastName)) { + if (config.deprecateDuplicateKey) { + context.runtime.getWarnings().warning( + "detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" + ); + } else { + throw parsingError(context, "duplicate key", p, pe); + } + } + } + {p = (( res.p))-1;} } } break; case 2: -// line 680 "ParserConfig.rl" +// line 707 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2121 "ParserConfig.java" +// line 2148 "ParserConfig.java" } } } @@ -2137,7 +2164,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 711 "ParserConfig.rl" +// line 738 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2148,7 +2175,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2152 "ParserConfig.java" +// line 2179 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2251,7 +2278,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 740 "ParserConfig.rl" +// line 767 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2261,16 +2288,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2265 "ParserConfig.java" +// line 2292 "ParserConfig.java" { cs = JSON_start; } -// line 749 "ParserConfig.rl" +// line 776 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2274 "ParserConfig.java" +// line 2301 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2351,7 +2378,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 726 "ParserConfig.rl" +// line 753 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2363,7 +2390,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2367 "ParserConfig.java" +// line 2394 "ParserConfig.java" } } } @@ -2383,7 +2410,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 752 "ParserConfig.rl" +// line 779 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index ce0d9438e..4bc5d93bd 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -52,6 +52,8 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowDuplicateKey; + private boolean deprecateDuplicateKey; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; @@ -175,6 +177,14 @@ public class ParserConfig extends RubyObject { this.allowNaN = opts.getBool("allow_nan", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); + if (opts.hasKey("allow_duplicate_key")) { + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.deprecateDuplicateKey = false; + } else { + this.allowDuplicateKey = false; + this.deprecateDuplicateKey = true; + } + this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); @@ -278,13 +288,17 @@ public class ParserConfig extends RubyObject { this.decoder = new StringDecoder(); } - private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { + private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { RubyString msg = context.runtime.newString("unexpected token at '") .cat(data, absStart, Math.min(absEnd - absStart, 32)) .cat((byte)'\''); return newException(context, Utils.M_PARSER_ERROR, msg); } + private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { + return parsingError(context, "unexpected token at '", absStart, absEnd); + } + %%{ machine JSON_common; @@ -673,6 +687,19 @@ public class ParserConfig extends RubyObject { } else { lastName = name; } + + if (!config.allowDuplicateKey) { + if (((RubyHash)result).hasKey(lastName)) { + if (config.deprecateDuplicateKey) { + context.runtime.getWarnings().warning( + "detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" + ); + } else { + throw parsingError(context, "duplicate key", p, pe); + } + } + } + fexec res.p; } } diff --git a/lib/json.rb b/lib/json.rb index 264301e05..735f23806 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -127,6 +127,24 @@ # # --- # +# Option +allow_duplicate_key+ specifies whether duplicate keys in objects +# should be ignored or cause an error to be raised: +# +# When not specified: +# # The last value is used and a deprecation warning emitted. +# JSON.parse('{"a": 1, "a":2}') => {"a" => 2} +# # waring: detected duplicate keys in JSON object. +# # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` +# +# When set to `+true+` +# # The last value is used. +# JSON.parse('{"a": 1, "a":2}') => {"a" => 2} +# +# When set to `+false+`, the future default: +# JSON.parse('{"a": 1, "a":2}') => duplicate key at line 1 column 1 (JSON::ParserError) +# +# --- +# # Option +allow_nan+ (boolean) specifies whether to allow # NaN, Infinity, and MinusInfinity in +source+; # defaults to +false+. diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index befc80c95..739a4cf63 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -331,6 +331,15 @@ def test_parse_big_integers assert_equal orig, parse(json5) end + def test_parse_duplicate_key + expected = {"a" => 2} + assert_equal expected, parse('{"a": 1, "a": 2}', allow_duplicate_key: true) + assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false) } + assert_deprecated_warning(/duplicate keys/) do + assert_equal expected, parse('{"a": 1, "a": 2}') + end + end + def test_some_wrong_inputs assert_raise(ParserError) { parse('[] bla') } assert_raise(ParserError) { parse('[] 1') } From 6c411625222a7c348fcef0bfa9e1d561d591e5e6 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 24 Jun 2025 18:06:23 +0900 Subject: [PATCH 079/506] Remove trailing spaces [ci skip] --- ext/json/ext/generator/generator.c | 48 +++++++++++++++--------------- ext/json/ext/generator/simd.h | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index f7690a23e..43a7f5f64 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -333,7 +333,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) if (search->matches_mask > 0) { return neon_next_match(search); } else { - // neon_next_match will only advance search->ptr up to the last matching character. + // neon_next_match will only advance search->ptr up to the last matching character. // Skip over any characters in the last chunk that occur after the last match. search->has_matches = false; search->ptr = search->chunk_end; @@ -342,40 +342,40 @@ static inline unsigned char search_escape_basic_neon(search_state *search) /* * The code below implements an SIMD-based algorithm to determine if N bytes at a time - * need to be escaped. - * + * need to be escaped. + * * Assume the ptr = "Te\sting!" (the double quotes are included in the string) - * + * * The explanation will be limited to the first 8 bytes of the string for simplicity. However * the vector insructions may work on larger vectors. - * + * * First, we load three constants 'lower_bound', 'backslash' and 'dblquote" in vector registers. - * - * lower_bound: [20 20 20 20 20 20 20 20] - * backslash: [5C 5C 5C 5C 5C 5C 5C 5C] - * dblquote: [22 22 22 22 22 22 22 22] - * - * Next we load the first chunk of the ptr: + * + * lower_bound: [20 20 20 20 20 20 20 20] + * backslash: [5C 5C 5C 5C 5C 5C 5C 5C] + * dblquote: [22 22 22 22 22 22 22 22] + * + * Next we load the first chunk of the ptr: * [22 54 65 5C 73 74 69 6E] (" T e \ s t i n) - * + * * First we check if any byte in chunk is less than 32 (0x20). This returns the following vector * as no bytes are less than 32 (0x20): * [0 0 0 0 0 0 0 0] - * + * * Next, we check if any byte in chunk is equal to a backslash: * [0 0 0 FF 0 0 0 0] - * + * * Finally we check if any byte in chunk is equal to a double quote: - * [FF 0 0 0 0 0 0 0] - * + * [FF 0 0 0 0 0 0 0] + * * Now we have three vectors where each byte indicates if the corresponding byte in chunk * needs to be escaped. We combine these vectors with a series of logical OR instructions. * This is the needs_escape vector and it is equal to: - * [FF 0 0 FF 0 0 0 0] - * + * [FF 0 0 FF 0 0 0 0] + * * Next we compute the bitwise AND between each byte and 0x1 and compute the horizontal sum of * the values in the vector. This computes how many bytes need to be escaped within this chunk. - * + * * Finally we compute a mask that indicates which bytes need to be escaped. If the mask is 0 then, * no bytes need to be escaped and we can continue to the next chunk. If the mask is not 0 then we * have at least one byte that needs to be escaped. @@ -394,7 +394,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) return neon_next_match(search); } - // There are fewer than 16 bytes left. + // There are fewer than 16 bytes left. unsigned long remaining = (search->end - search->ptr); if (remaining >= SIMD_MINIMUM_THRESHOLD) { char *s = copy_remaining_bytes(search, sizeof(uint8x16_t), remaining); @@ -402,7 +402,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) uint64_t mask = neon_rules_update(s); if (!mask) { - // Nothing to escape, ensure search_flush doesn't do anything by setting + // Nothing to escape, ensure search_flush doesn't do anything by setting // search->cursor to search->ptr. fbuffer_consumed(search->buffer, remaining); search->ptr = search->end; @@ -476,7 +476,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se if (search->matches_mask > 0) { return sse2_next_match(search); } else { - // sse2_next_match will only advance search->ptr up to the last matching character. + // sse2_next_match will only advance search->ptr up to the last matching character. // Skip over any characters in the last chunk that occur after the last match. search->has_matches = false; if (RB_UNLIKELY(search->chunk_base + sizeof(__m128i) >= search->end)) { @@ -501,7 +501,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se return sse2_next_match(search); } - // There are fewer than 16 bytes left. + // There are fewer than 16 bytes left. unsigned long remaining = (search->end - search->ptr); if (remaining >= SIMD_MINIMUM_THRESHOLD) { char *s = copy_remaining_bytes(search, sizeof(__m128i), remaining); @@ -509,7 +509,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se int needs_escape_mask = sse2_update(s); if (needs_escape_mask == 0) { - // Nothing to escape, ensure search_flush doesn't do anything by setting + // Nothing to escape, ensure search_flush doesn't do anything by setting // search->cursor to search->ptr. fbuffer_consumed(search->buffer, remaining); search->ptr = search->end; diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h index b12890cb0..329c0387f 100644 --- a/ext/json/ext/generator/simd.h +++ b/ext/json/ext/generator/simd.h @@ -87,7 +87,7 @@ uint8x16x4_t load_uint8x16_4(const unsigned char *table) { static SIMD_Implementation find_simd_implementation(void) { #if defined(__GNUC__ ) || defined(__clang__) -#ifdef __GNUC__ +#ifdef __GNUC__ __builtin_cpu_init(); #endif /* __GNUC__ */ From 51054bfff9a4bf1f3ad8f4aca705c2e32fa98ad2 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 12 Jun 2025 20:58:15 -0500 Subject: [PATCH 080/506] Optimize 'json_parse_string' using SIMD. --- ext/json/ext/generator/depend | 2 +- ext/json/ext/generator/generator.c | 64 +--------- ext/json/ext/generator/simd.h | 112 ----------------- ext/json/ext/parser/depend | 1 + ext/json/ext/parser/extconf.rb | 28 +++++ ext/json/ext/parser/parser.c | 83 ++++++++++--- ext/json/ext/simd/simd.h | 189 +++++++++++++++++++++++++++++ test/json/json_parser_test.rb | 84 +++++++++++++ 8 files changed, 373 insertions(+), 190 deletions(-) delete mode 100644 ext/json/ext/generator/simd.h create mode 100644 ext/json/ext/simd/simd.h diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 21bdc420f..14241d7f0 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1,4 +1,4 @@ generator.o: generator.c $(srcdir)/../fbuffer/fbuffer.h generator.o: generator.c $(srcdir)/../vendor/fpconv.c generator.o: generator.c $(srcdir)/../vendor/jeaiii-ltoa.h -generator.o: generator.c $(srcdir)/simd.h +generator.o: generator.c $(srcdir)/../simd/simd.h diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 43a7f5f64..01e8badc9 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -5,7 +5,7 @@ #include #include -#include "simd.h" +#include "../simd/simd.h" /* ruby api and some helpers */ @@ -304,28 +304,6 @@ static inline FORCE_INLINE unsigned char neon_next_match(search_state *search) return 1; } -// See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon -static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches) -{ - const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4); - const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); - return mask & 0x8888888888888888ull; -} - -static inline FORCE_INLINE uint64_t neon_rules_update(const char *ptr) -{ - uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); - - // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 - // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ - const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33)); - - uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\')); - uint8x16_t needs_escape = vorrq_u8(too_low_or_dbl_quote, has_backslash); - - return neon_match_mask(needs_escape); -} - static inline unsigned char search_escape_basic_neon(search_state *search) { if (RB_UNLIKELY(search->has_matches)) { @@ -380,14 +358,8 @@ static inline unsigned char search_escape_basic_neon(search_state *search) * no bytes need to be escaped and we can continue to the next chunk. If the mask is not 0 then we * have at least one byte that needs to be escaped. */ - while (search->ptr + sizeof(uint8x16_t) <= search->end) { - uint64_t mask = neon_rules_update(search->ptr); - if (!mask) { - search->ptr += sizeof(uint8x16_t); - continue; - } - search->matches_mask = mask; + if (string_scan_simd_neon(&search->ptr, search->end, &search->matches_mask)) { search->has_matches = true; search->chunk_base = search->ptr; search->chunk_end = search->ptr + sizeof(uint8x16_t); @@ -399,7 +371,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) if (remaining >= SIMD_MINIMUM_THRESHOLD) { char *s = copy_remaining_bytes(search, sizeof(uint8x16_t), remaining); - uint64_t mask = neon_rules_update(s); + uint64_t mask = compute_chunk_mask_neon(s); if (!mask) { // Nothing to escape, ensure search_flush doesn't do anything by setting @@ -428,11 +400,6 @@ static inline unsigned char search_escape_basic_neon(search_state *search) #ifdef HAVE_SIMD_SSE2 -#define _mm_cmpge_epu8(a, b) _mm_cmpeq_epi8(_mm_max_epu8(a, b), a) -#define _mm_cmple_epu8(a, b) _mm_cmpge_epu8(b, a) -#define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) -#define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) - static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search) { int mask = search->matches_mask; @@ -457,18 +424,6 @@ static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search) #define TARGET_SSE2 #endif -static inline TARGET_SSE2 FORCE_INLINE int sse2_update(const char *ptr) -{ - __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); - - // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 - // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ - __m128i too_low_or_dbl_quote = _mm_cmplt_epu8(_mm_xor_si128(chunk, _mm_set1_epi8(2)), _mm_set1_epi8(33)); - __m128i has_backslash = _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\')); - __m128i needs_escape = _mm_or_si128(too_low_or_dbl_quote, has_backslash); - return _mm_movemask_epi8(needs_escape); -} - static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(search_state *search) { if (RB_UNLIKELY(search->has_matches)) { @@ -487,17 +442,10 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se } } - while (search->ptr + sizeof(__m128i) <= search->end) { - int needs_escape_mask = sse2_update(search->ptr); - - if (needs_escape_mask == 0) { - search->ptr += sizeof(__m128i); - continue; - } - + if (string_scan_simd_sse2(&search->ptr, search->end, &search->matches_mask)) { search->has_matches = true; - search->matches_mask = needs_escape_mask; search->chunk_base = search->ptr; + search->chunk_end = search->ptr + sizeof(__m128i); return sse2_next_match(search); } @@ -506,7 +454,7 @@ static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(se if (remaining >= SIMD_MINIMUM_THRESHOLD) { char *s = copy_remaining_bytes(search, sizeof(__m128i), remaining); - int needs_escape_mask = sse2_update(s); + int needs_escape_mask = compute_chunk_mask_sse2(s); if (needs_escape_mask == 0) { // Nothing to escape, ensure search_flush doesn't do anything by setting diff --git a/ext/json/ext/generator/simd.h b/ext/json/ext/generator/simd.h deleted file mode 100644 index 329c0387f..000000000 --- a/ext/json/ext/generator/simd.h +++ /dev/null @@ -1,112 +0,0 @@ -typedef enum { - SIMD_NONE, - SIMD_NEON, - SIMD_SSE2 -} SIMD_Implementation; - -#ifdef JSON_ENABLE_SIMD - -#ifdef __clang__ - #if __has_builtin(__builtin_ctzll) - #define HAVE_BUILTIN_CTZLL 1 - #else - #define HAVE_BUILTIN_CTZLL 0 - #endif -#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) - #define HAVE_BUILTIN_CTZLL 1 -#else - #define HAVE_BUILTIN_CTZLL 0 -#endif - -static inline uint32_t trailing_zeros64(uint64_t input) { -#if HAVE_BUILTIN_CTZLL - return __builtin_ctzll(input); -#else - uint32_t trailing_zeros = 0; - uint64_t temp = input; - while ((temp & 1) == 0 && temp > 0) { - trailing_zeros++; - temp >>= 1; - } - return trailing_zeros; -#endif -} - -static inline int trailing_zeros(int input) { - #if HAVE_BUILTIN_CTZLL - return __builtin_ctz(input); - #else - int trailing_zeros = 0; - int temp = input; - while ((temp & 1) == 0 && temp > 0) { - trailing_zeros++; - temp >>= 1; - } - return trailing_zeros; - #endif -} - -#define SIMD_MINIMUM_THRESHOLD 6 - -#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) -#include - -#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 -static SIMD_Implementation find_simd_implementation(void) { - return SIMD_NEON; -} - -#define HAVE_SIMD 1 -#define HAVE_SIMD_NEON 1 - -uint8x16x4_t load_uint8x16_4(const unsigned char *table) { - uint8x16x4_t tab; - tab.val[0] = vld1q_u8(table); - tab.val[1] = vld1q_u8(table+16); - tab.val[2] = vld1q_u8(table+32); - tab.val[3] = vld1q_u8(table+48); - return tab; -} - -#endif /* ARM Neon Support.*/ - -#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) - -#ifdef HAVE_X86INTRIN_H -#include - -#define HAVE_SIMD 1 -#define HAVE_SIMD_SSE2 1 - -#ifdef HAVE_CPUID_H -#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 - -#include -#endif /* HAVE_CPUID_H */ - -static SIMD_Implementation find_simd_implementation(void) { - -#if defined(__GNUC__ ) || defined(__clang__) -#ifdef __GNUC__ - __builtin_cpu_init(); -#endif /* __GNUC__ */ - - // TODO Revisit. I think the SSE version now only uses SSE2 instructions. - if (__builtin_cpu_supports("sse2")) { - return SIMD_SSE2; - } -#endif /* __GNUC__ || __clang__*/ - - return SIMD_NONE; -} - -#endif /* HAVE_X86INTRIN_H */ -#endif /* X86_64 Support */ - -#endif /* JSON_ENABLE_SIMD */ - -#ifndef FIND_SIMD_IMPLEMENTATION_DEFINED -static SIMD_Implementation find_simd_implementation(void) { - return SIMD_NONE; -} -#endif diff --git a/ext/json/ext/parser/depend b/ext/json/ext/parser/depend index c051a244f..a1926b7d6 100644 --- a/ext/json/ext/parser/depend +++ b/ext/json/ext/parser/depend @@ -1 +1,2 @@ parser.o: parser.c $(srcdir)/../fbuffer/fbuffer.h +parser.o: parser.c $(srcdir)/../simd/simd.h diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 09c963778..eca4d8c10 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -8,4 +8,32 @@ append_cflags("-std=c99") +if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) + if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ + # Try to compile a small program using NEON instructions + if have_header('arm_neon.h') + have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') + #include + int main() { + uint8x16_t test = vdupq_n_u8(32); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") + end + end + + if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') + #include + int main() { + __m128i test = _mm_set1_epi8(32); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") + end + + have_header('cpuid.h') + end + create_makefile 'json/ext/parser' diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 627971eb5..d77969482 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -20,6 +20,8 @@ typedef unsigned char _Bool; #endif #endif +#include "../simd/simd.h" + #ifndef RB_UNLIKELY #define RB_UNLIKELY(expr) expr #endif @@ -879,7 +881,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig * return value; } -static const bool string_scan[256] = { +static const bool string_scan_table[256] = { // ASCII Control Characters 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -892,32 +894,71 @@ static const bool string_scan[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; +#if (defined(__GNUC__ ) || defined(__clang__)) +#define FORCE_INLINE __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + +#ifdef HAVE_SIMD +static SIMD_Implementation simd_impl = SIMD_NONE; +#endif /* HAVE_SIMD */ + +static inline bool FORCE_INLINE string_scan(JSON_ParserState *state) +{ +#ifdef HAVE_SIMD +#if defined(HAVE_SIMD_NEON) + + uint64_t mask = 0; + if (string_scan_simd_neon(&state->cursor, state->end, &mask)) { + state->cursor += trailing_zeros64(mask) >> 2; + return 1; + } + +#elif defined(HAVE_SIMD_SSE2) + if (simd_impl == SIMD_SSE2) { + int mask = 0; + if (string_scan_simd_sse2(&state->cursor, state->end, &mask)) { + state->cursor += trailing_zeros(mask); + return 1; + } + } +#endif /* HAVE_SIMD_NEON or HAVE_SIMD_SSE2 */ +#endif /* HAVE_SIMD */ + + while (state->cursor < state->end) { + if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) { + return 1; + } + *state->cursor++; + } + return 0; +} + static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) { state->cursor++; const char *start = state->cursor; bool escaped = false; - while (state->cursor < state->end) { - if (RB_UNLIKELY(string_scan[(unsigned char)*state->cursor])) { - switch (*state->cursor) { - case '"': { - VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name); - state->cursor++; - return json_push_value(state, config, string); - } - case '\\': { - state->cursor++; - escaped = true; - if ((unsigned char)*state->cursor < 0x20) { - raise_parse_error("invalid ASCII control character in string: %s", state); - } - break; - } - default: + while (RB_UNLIKELY(string_scan(state))) { + switch (*state->cursor) { + case '"': { + VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name); + state->cursor++; + return json_push_value(state, config, string); + } + case '\\': { + state->cursor++; + escaped = true; + if ((unsigned char)*state->cursor < 0x20) { raise_parse_error("invalid ASCII control character in string: %s", state); - break; + } + break; } + default: + raise_parse_error("invalid ASCII control character in string: %s", state); + break; } state->cursor++; @@ -1459,4 +1500,8 @@ void Init_parser(void) binary_encindex = rb_ascii8bit_encindex(); utf8_encindex = rb_utf8_encindex(); enc_utf8 = rb_utf8_encoding(); + +#ifdef HAVE_SIMD + simd_impl = find_simd_implementation(); +#endif } diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h new file mode 100644 index 000000000..ed2a6d467 --- /dev/null +++ b/ext/json/ext/simd/simd.h @@ -0,0 +1,189 @@ +typedef enum { + SIMD_NONE, + SIMD_NEON, + SIMD_SSE2 +} SIMD_Implementation; + +#ifdef JSON_ENABLE_SIMD + +#ifdef __clang__ + #if __has_builtin(__builtin_ctzll) + #define HAVE_BUILTIN_CTZLL 1 + #else + #define HAVE_BUILTIN_CTZLL 0 + #endif +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define HAVE_BUILTIN_CTZLL 1 +#else + #define HAVE_BUILTIN_CTZLL 0 +#endif + +static inline uint32_t trailing_zeros64(uint64_t input) { +#if HAVE_BUILTIN_CTZLL + return __builtin_ctzll(input); +#else + uint32_t trailing_zeros = 0; + uint64_t temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; +#endif +} + +static inline int trailing_zeros(int input) { + #if HAVE_BUILTIN_CTZLL + return __builtin_ctz(input); + #else + int trailing_zeros = 0; + int temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; + #endif +} + +#if (defined(__GNUC__ ) || defined(__clang__)) +#define FORCE_INLINE __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + + +#define SIMD_MINIMUM_THRESHOLD 6 + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) +#include + +#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 +static SIMD_Implementation find_simd_implementation(void) { + return SIMD_NEON; +} + +#define HAVE_SIMD 1 +#define HAVE_SIMD_NEON 1 + +// See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon +static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches) +{ + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4); + const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); + return mask & 0x8888888888888888ull; +} + +static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr) +{ + uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); + + // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 + // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ + const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33)); + + uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\')); + uint8x16_t needs_escape = vorrq_u8(too_low_or_dbl_quote, has_backslash); + return neon_match_mask(needs_escape); +} + +static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) +{ + while(*ptr + sizeof(uint8x16_t) <= end) { + uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); + if (chunk_mask) { + *mask = chunk_mask; + return 1; + } + *ptr += sizeof(uint8x16_t); + } + return 0; +} + +uint8x16x4_t load_uint8x16_4(const unsigned char *table) { + uint8x16x4_t tab; + tab.val[0] = vld1q_u8(table); + tab.val[1] = vld1q_u8(table+16); + tab.val[2] = vld1q_u8(table+32); + tab.val[3] = vld1q_u8(table+48); + return tab; +} + +#endif /* ARM Neon Support.*/ + +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) + +#ifdef HAVE_X86INTRIN_H +#include + +#define HAVE_SIMD 1 +#define HAVE_SIMD_SSE2 1 + +#ifdef HAVE_CPUID_H +#define FIND_SIMD_IMPLEMENTATION_DEFINED 1 + +#if defined(__clang__) || defined(__GNUC__) +#define TARGET_SSE2 __attribute__((target("sse2"))) +#else +#define TARGET_SSE2 +#endif + +#define _mm_cmpge_epu8(a, b) _mm_cmpeq_epi8(_mm_max_epu8(a, b), a) +#define _mm_cmple_epu8(a, b) _mm_cmpge_epu8(b, a) +#define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) +#define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) + +static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *ptr) +{ + __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); + // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 + // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ + __m128i too_low_or_dbl_quote = _mm_cmplt_epu8(_mm_xor_si128(chunk, _mm_set1_epi8(2)), _mm_set1_epi8(33)); + __m128i has_backslash = _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\')); + __m128i needs_escape = _mm_or_si128(too_low_or_dbl_quote, has_backslash); + return _mm_movemask_epi8(needs_escape); +} + +static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) +{ + while (*ptr + sizeof(__m128i) <= end) { + int chunk_mask = compute_chunk_mask_sse2(*ptr); + if (chunk_mask) { + *mask = chunk_mask; + return 1; + } + *ptr += sizeof(__m128i); + } + + return 0; +} + +#include +#endif /* HAVE_CPUID_H */ + +static SIMD_Implementation find_simd_implementation(void) { + +#if defined(__GNUC__ ) || defined(__clang__) +#ifdef __GNUC__ + __builtin_cpu_init(); +#endif /* __GNUC__ */ + + // TODO Revisit. I think the SSE version now only uses SSE2 instructions. + if (__builtin_cpu_supports("sse2")) { + return SIMD_SSE2; + } +#endif /* __GNUC__ || __clang__*/ + + return SIMD_NONE; +} + +#endif /* HAVE_X86INTRIN_H */ +#endif /* X86_64 Support */ + +#endif /* JSON_ENABLE_SIMD */ + +#ifndef FIND_SIMD_IMPLEMENTATION_DEFINED +static SIMD_Implementation find_simd_implementation(void) { + return SIMD_NONE; +} +#endif diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 739a4cf63..106492e1c 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -469,6 +469,90 @@ def test_backslash json = '["\/"]' data = [ '/' ] assert_equal data, parse(json) + + data = ['"""""""""""""""""""""""""'] + json = '["\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""]' + assert_equal data, parse(json) + + data = '["This is a "test" of the emergency broadcast system."]' + json = "\"[\\\"This is a \\\"test\\\" of the emergency broadcast system.\\\"]\"" + assert_equal data, parse(json) + + data = '\tThis is a test of the emergency broadcast system.' + json = "\"\\\\tThis is a test of the emergency broadcast system.\"" + assert_equal data, parse(json) + + data = 'This\tis a test of the emergency broadcast system.' + json = "\"This\\\\tis a test of the emergency broadcast system.\"" + assert_equal data, parse(json) + + data = 'This is\ta test of the emergency broadcast system.' + json = "\"This is\\\\ta test of the emergency broadcast system.\"" + assert_equal data, parse(json) + + data = 'This is a test of the emergency broadcast\tsystem.' + json = "\"This is a test of the emergency broadcast\\\\tsystem.\"" + assert_equal data, parse(json) + + data = 'This is a test of the emergency broadcast\tsystem.\n' + json = "\"This is a test of the emergency broadcast\\\\tsystem.\\\\n\"" + assert_equal data, parse(json) + + data = '"' * 15 + json = "\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\"" + assert_equal data, parse(json) + + data = "\"\"\"\"\"\"\"\"\"\"\"\"\"\"a" + json = "\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"\\\"a\"" + assert_equal data, parse(json) + + data = "\u0001\u0001\u0001\u0001" + json = "\"\\u0001\\u0001\\u0001\\u0001\"" + assert_equal data, parse(json) + + data = "\u0001a\u0001a\u0001a\u0001a" + json = "\"\\u0001a\\u0001a\\u0001a\\u0001a\"" + assert_equal data, parse(json) + + data = "\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\"" + assert_equal data, parse(json) + + data = "\u0001aa\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\\u0001aa\"" + assert_equal data, parse(json) + + data = "\u0001aa\u0001aa\u0001aa\u0001aa\u0001aa\u0001aa" + json = "\"\\u0001aa\\u0001aa\\u0001aa\\u0001aa\\u0001aa\\u0001aa\"" + assert_equal data, parse(json) + + data = "\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002\u0001a\u0002" + json = "\"\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\\u0001a\\u0002\"" + assert_equal data, parse(json) + + data = "ab\u0002c" + json = "\"ab\\u0002c\"" + assert_equal data, parse(json) + + data = "ab\u0002cab\u0002cab\u0002cab\u0002c" + json = "\"ab\\u0002cab\\u0002cab\\u0002cab\\u0002c\"" + assert_equal data, parse(json) + + data = "ab\u0002cab\u0002cab\u0002cab\u0002cab\u0002cab\u0002c" + json = "\"ab\\u0002cab\\u0002cab\\u0002cab\\u0002cab\\u0002cab\\u0002c\"" + assert_equal data, parse(json) + + data = "\n\t\f\b\n\t\f\b\n\t\f\b\n\t\f" + json = "\"\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\"" + assert_equal data, parse(json) + + data = "\n\t\f\b\n\t\f\b\n\t\f\b\n\t\f\b" + json = "\"\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\"" + assert_equal data, parse(json) + + data = "a\n\t\f\b\n\t\f\b\n\t\f\b\n\t" + json = "\"a\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\\f\\b\\n\\t\"" + assert_equal data, parse(json) end class SubArray < Array From 3ae3eeb9d357311c22ee904cda7a8cc3a26f7cc6 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 29 Jun 2025 12:04:34 +0200 Subject: [PATCH 081/506] Reduce duplication in extconf.rb --- ext/json/ext/generator/extconf.rb | 26 +------------------------- ext/json/ext/parser/extconf.rb | 28 ++-------------------------- ext/json/ext/simd/conf.rb | 25 +++++++++++++++++++++++++ json.gemspec | 2 +- 4 files changed, 29 insertions(+), 52 deletions(-) create mode 100644 ext/json/ext/simd/conf.rb diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index f58574a6c..aaf02c77d 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -9,31 +9,7 @@ $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] if enable_config('generator-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) - if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ - # Try to compile a small program using NEON instructions - if have_header('arm_neon.h') - have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') - #include - int main() { - uint8x16_t test = vdupq_n_u8(32); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end - end - - if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') - #include - int main() { - __m128i test = _mm_set1_epi8(32); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end - - have_header('cpuid.h') + require_relative "../simd/conf.rb" end create_makefile 'json/ext/generator' diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index eca4d8c10..0b62fd613 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -9,31 +9,7 @@ append_cflags("-std=c99") if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) - if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ - # Try to compile a small program using NEON instructions - if have_header('arm_neon.h') - have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') - #include - int main() { - uint8x16_t test = vdupq_n_u8(32); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end - end - - if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') - #include - int main() { - __m128i test = _mm_set1_epi8(32); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end - - have_header('cpuid.h') - end + require_relative "../simd/conf.rb" +end create_makefile 'json/ext/parser' diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb new file mode 100644 index 000000000..6393cf789 --- /dev/null +++ b/ext/json/ext/simd/conf.rb @@ -0,0 +1,25 @@ +if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ + # Try to compile a small program using NEON instructions + if have_header('arm_neon.h') + have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') + #include + int main() { + uint8x16_t test = vdupq_n_u8(32); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") + end +end + +if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') + #include + int main() { + __m128i test = _mm_set1_epi8(32); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") +end + +have_header('cpuid.h') diff --git a/json.gemspec b/json.gemspec index 943c78aab..07426363a 100644 --- a/json.gemspec +++ b/json.gemspec @@ -52,7 +52,7 @@ spec = Gem::Specification.new do |s| s.files += Dir["lib/json/ext/**/*.jar"] else s.extensions = Dir["ext/json/**/extconf.rb"] - s.files += Dir["ext/json/**/*.{c,h}"] + s.files += Dir["ext/json/**/*.{c,h,rb}"] end end From 68ee9cf18871d7c71747b6d5538bf50330db9864 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:17:33 +0900 Subject: [PATCH 082/506] Remove trailing spaces [ci skip] --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d77969482..9bf247039 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -908,7 +908,7 @@ static inline bool FORCE_INLINE string_scan(JSON_ParserState *state) { #ifdef HAVE_SIMD #if defined(HAVE_SIMD_NEON) - + uint64_t mask = 0; if (string_scan_simd_neon(&state->cursor, state->end, &mask)) { state->cursor += trailing_zeros64(mask) >> 2; From 8e775320b755bd2d0f428f5bf36019fb9cb3df74 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:17:57 +0900 Subject: [PATCH 083/506] Use `load` simd/conf.rb When both extconf.rb of generator and parser are run in one process, the second `require_relative` does nothing. --- ext/json/ext/generator/extconf.rb | 2 +- ext/json/ext/parser/extconf.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index aaf02c77d..fb9afd07f 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -9,7 +9,7 @@ $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] if enable_config('generator-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) - require_relative "../simd/conf.rb" + load __dir__ + "/../simd/conf.rb" end create_makefile 'json/ext/generator' diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 0b62fd613..84049a7fe 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -9,7 +9,7 @@ append_cflags("-std=c99") if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) - require_relative "../simd/conf.rb" + load __dir__ + "/../simd/conf.rb" end create_makefile 'json/ext/parser' From 58dc0aa938245e58adcf52be6aed7a223d1d09e2 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:37:20 +0900 Subject: [PATCH 084/506] Refactor simd/conf.rb - compiler warnings Suppress warnings for old style function definition and unused variable. --- ext/json/ext/simd/conf.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index 6393cf789..ee56718be 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -3,8 +3,9 @@ if have_header('arm_neon.h') have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') #include - int main() { + int main(int argc, char **argv) { uint8x16_t test = vdupq_n_u8(32); + if (argc > 100000) printf("%p", &test); return 0; } SRC @@ -14,8 +15,9 @@ if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') #include - int main() { + int main(int argc, char **argv) { __m128i test = _mm_set1_epi8(32); + if (argc > 100000) printf("%p", &test); return 0; } SRC From 2211e30a59a31cabd6c8642164314a3d37bb0f1e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:38:56 +0900 Subject: [PATCH 085/506] Refactor simd/conf.rb - balance Align code for arm and x86_64 in parallel. --- ext/json/ext/simd/conf.rb | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index ee56718be..deaac412d 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -1,4 +1,5 @@ -if RbConfig::CONFIG['host_cpu'] =~ /^(arm.*|aarch64.*)/ +case RbConfig::CONFIG['host_cpu'] +when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions if have_header('arm_neon.h') have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') @@ -8,20 +9,20 @@ if (argc > 100000) printf("%p", &test); return 0; } - SRC - $defs.push("-DJSON_ENABLE_SIMD") + SRC + $defs.push("-DJSON_ENABLE_SIMD") end -end - -if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') - #include - int main(int argc, char **argv) { - __m128i test = _mm_set1_epi8(32); - if (argc > 100000) printf("%p", &test); - return 0; - } - SRC +when /^(x86_64|x64)/ + if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') + #include + int main(int argc, char **argv) { + __m128i test = _mm_set1_epi8(32); + if (argc > 100000) printf("%p", &test); + return 0; + } + SRC $defs.push("-DJSON_ENABLE_SIMD") + end end have_header('cpuid.h') From fdbb6062c2608565f2901bbd4f0c82dce92fea84 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:43:13 +0900 Subject: [PATCH 086/506] Refactor simd/conf.rb - conditions to enable See the results of `have_type` and `try_compile` in addition to `have_header` for NEON as well as x86_64. The former results were just ignored, and `HAVE_TYPE_` macros are unused too. --- ext/json/ext/simd/conf.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index deaac412d..3b3cf68af 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -1,8 +1,8 @@ case RbConfig::CONFIG['host_cpu'] when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions - if have_header('arm_neon.h') - have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') + if have_header('arm_neon.h') && + have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') #include int main(int argc, char **argv) { uint8x16_t test = vdupq_n_u8(32); @@ -13,7 +13,8 @@ $defs.push("-DJSON_ENABLE_SIMD") end when /^(x86_64|x64)/ - if have_header('x86intrin.h') && have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') + if have_header('x86intrin.h') && + have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') #include int main(int argc, char **argv) { __m128i test = _mm_set1_epi8(32); From b08e1ca2c104e400756807787225c37c9a9823e8 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 21:53:27 +0900 Subject: [PATCH 087/506] Refactor simd/conf.rb - unnecessary `have_type` Remove `have_type` calls because the next `try_compile` calls check those types. --- ext/json/ext/simd/conf.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index 3b3cf68af..fa5b97801 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -2,7 +2,7 @@ when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions if have_header('arm_neon.h') && - have_type('uint8x16_t', headers=['arm_neon.h']) && try_compile(<<~'SRC') + try_compile(<<~'SRC') #include int main(int argc, char **argv) { uint8x16_t test = vdupq_n_u8(32); @@ -14,7 +14,7 @@ end when /^(x86_64|x64)/ if have_header('x86intrin.h') && - have_type('__m128i', headers=['x86intrin.h']) && try_compile(<<~'SRC') + try_compile(<<~'SRC') #include int main(int argc, char **argv) { __m128i test = _mm_set1_epi8(32); From 1a768d91796cefffa6e273b7362e462a1cbc4c7a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 30 Jun 2025 22:34:14 +0900 Subject: [PATCH 088/506] Refactor simd/conf.rb - duplicate code Integrate duplicate code by extracting headers, types and initialization code. --- ext/json/ext/simd/conf.rb | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index fa5b97801..8e7d8ee26 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -1,29 +1,20 @@ case RbConfig::CONFIG['host_cpu'] when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions - if have_header('arm_neon.h') && - try_compile(<<~'SRC') - #include - int main(int argc, char **argv) { - uint8x16_t test = vdupq_n_u8(32); - if (argc > 100000) printf("%p", &test); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end + header, type, init = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)' when /^(x86_64|x64)/ - if have_header('x86intrin.h') && - try_compile(<<~'SRC') - #include - int main(int argc, char **argv) { - __m128i test = _mm_set1_epi8(32); - if (argc > 100000) printf("%p", &test); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") - end + header, type, init = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)' +end +if header + have_header(header) && try_compile(<<~SRC) + #{cpp_include(header)} + int main(int argc, char **argv) { + #{type} test = #{init}; + if (argc > 100000) printf("%p", &test); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") end have_header('cpuid.h') From 94ed471814c39037b87bb1edb49350fdf9e023d1 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 30 Jun 2025 08:07:17 -0700 Subject: [PATCH 089/506] Suppress -Wunused-function --- ext/json/ext/simd/simd.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index ed2a6d467..d11e4df3f 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -59,7 +59,7 @@ static inline int trailing_zeros(int input) { #include #define FIND_SIMD_IMPLEMENTATION_DEFINED 1 -static SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) { return SIMD_NEON; } @@ -161,7 +161,7 @@ static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **pt #include #endif /* HAVE_CPUID_H */ -static SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) { #if defined(__GNUC__ ) || defined(__clang__) #ifdef __GNUC__ @@ -183,7 +183,7 @@ static SIMD_Implementation find_simd_implementation(void) { #endif /* JSON_ENABLE_SIMD */ #ifndef FIND_SIMD_IMPLEMENTATION_DEFINED -static SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) { return SIMD_NONE; } #endif From 94591ae63b15dc4e16523444589aba1b14f4aea2 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 1 Jul 2025 10:00:31 +0900 Subject: [PATCH 090/506] Use Dir.glob and base keyword arg for the installer of Ruby package --- json.gemspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/json.gemspec b/json.gemspec index 07426363a..557573102 100644 --- a/json.gemspec +++ b/json.gemspec @@ -44,8 +44,7 @@ spec = Gem::Specification.new do |s| "LEGAL", "README.md", "json.gemspec", - *Dir["lib/**/*.rb"], - ] + ] + Dir.glob("lib/**/*.rb", base: File.expand_path("..", __FILE__)) if java_ext s.platform = 'java' From d3317b9f82fe01a692e5d7237a75e73fe83744bd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 1 Jul 2025 08:48:00 +0200 Subject: [PATCH 091/506] Stop calling `__builtin_cpu_init` It's only needed if using GCC `ifunc` mecanism, which we don't. --- ext/json/ext/simd/simd.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index d11e4df3f..e0cf4754a 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -162,17 +162,10 @@ static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **pt #endif /* HAVE_CPUID_H */ static inline SIMD_Implementation find_simd_implementation(void) { - -#if defined(__GNUC__ ) || defined(__clang__) -#ifdef __GNUC__ - __builtin_cpu_init(); -#endif /* __GNUC__ */ - // TODO Revisit. I think the SSE version now only uses SSE2 instructions. if (__builtin_cpu_supports("sse2")) { return SIMD_SSE2; } -#endif /* __GNUC__ || __clang__*/ return SIMD_NONE; } From 95fb084027a85e847b252461a67ed8adb8a395fa Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 1 Jul 2025 17:38:25 +0900 Subject: [PATCH 092/506] Run `have_func` with the header providing the declarations --- ext/json/ext/parser/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 84049a7fe..de5d5758b 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'mkmf' -have_func("rb_enc_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 +have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby have_func("strnlen", "string.h") # Missing on Solaris 10 From a497c71960bb90eed90db10e28528edb9539cd2e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 5 Jul 2025 09:10:41 +0200 Subject: [PATCH 093/506] Improve consistency of code style --- ext/json/ext/generator/generator.c | 19 +++++++++++-------- ext/json/ext/parser/parser.c | 2 +- ext/json/ext/simd/simd.h | 17 +++++++++++------ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 01e8badc9..33b1bf349 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -586,7 +586,8 @@ static inline unsigned char search_ascii_only_escape(search_state *search, const return 0; } -static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_len) { +static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_len) +{ const unsigned char ch = (unsigned char)*search->ptr; switch (ch_len) { case 1: { @@ -616,7 +617,7 @@ static inline void full_escape_UTF8_char(search_state *search, unsigned char ch_ uint32_t wchar = 0; - switch(ch_len) { + switch (ch_len) { case 2: wchar = ch & 0x1F; break; @@ -776,7 +777,8 @@ static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self) * _state_ is a JSON::State object, that can also be used to configure the * produced JSON string output further. */ -static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) { +static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) +{ rb_check_arity(argc, 0, 1); VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); return cState_partial_generate(Vstate, self, generate_json_array, Qfalse); @@ -838,7 +840,8 @@ static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self) * * Extends _modul_ with the String::Extend module. */ -static VALUE mString_included_s(VALUE self, VALUE modul) { +static VALUE mString_included_s(VALUE self, VALUE modul) +{ VALUE result = rb_funcall(modul, i_extend, 1, mString_Extend); rb_call_super(1, &modul); return result; @@ -1083,7 +1086,7 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) } VALUE key_to_s; - switch(rb_type(key)) { + switch (rb_type(key)) { case T_STRING: if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) { key_to_s = key; @@ -1167,7 +1170,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data fbuffer_append_char(buffer, '['); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); - for(i = 0; i < RARRAY_LEN(obj); i++) { + for (i = 0; i < RARRAY_LEN(obj); i++) { if (i > 0) { fbuffer_append_char(buffer, ','); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); @@ -1252,7 +1255,7 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat search.chunk_base = NULL; #endif /* HAVE_SIMD */ - switch(rb_enc_str_coderange(obj)) { + switch (rb_enc_str_coderange(obj)) { case ENC_CODERANGE_7BIT: case ENC_CODERANGE_VALID: if (RB_UNLIKELY(data->state->ascii_only)) { @@ -2116,7 +2119,7 @@ void Init_generator(void) rb_require("json/ext/generator/state"); - switch(find_simd_implementation()) { + switch (find_simd_implementation()) { #ifdef HAVE_SIMD #ifdef HAVE_SIMD_NEON case SIMD_NEON: diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 9bf247039..01b6e6293 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -540,7 +540,7 @@ static void json_eat_comments(JSON_ParserState *state) { if (state->cursor + 1 < state->end) { - switch(state->cursor[1]) { + switch (state->cursor[1]) { case '/': { state->cursor = memchr(state->cursor, '\n', state->end - state->cursor); if (!state->cursor) { diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index e0cf4754a..f8503d139 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -18,7 +18,8 @@ typedef enum { #define HAVE_BUILTIN_CTZLL 0 #endif -static inline uint32_t trailing_zeros64(uint64_t input) { +static inline uint32_t trailing_zeros64(uint64_t input) +{ #if HAVE_BUILTIN_CTZLL return __builtin_ctzll(input); #else @@ -32,7 +33,8 @@ static inline uint32_t trailing_zeros64(uint64_t input) { #endif } -static inline int trailing_zeros(int input) { +static inline int trailing_zeros(int input) +{ #if HAVE_BUILTIN_CTZLL return __builtin_ctz(input); #else @@ -59,7 +61,8 @@ static inline int trailing_zeros(int input) { #include #define FIND_SIMD_IMPLEMENTATION_DEFINED 1 -static inline SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) +{ return SIMD_NEON; } @@ -89,7 +92,7 @@ static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr) static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) { - while(*ptr + sizeof(uint8x16_t) <= end) { + while (*ptr + sizeof(uint8x16_t) <= end) { uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); if (chunk_mask) { *mask = chunk_mask; @@ -161,7 +164,8 @@ static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **pt #include #endif /* HAVE_CPUID_H */ -static inline SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) +{ // TODO Revisit. I think the SSE version now only uses SSE2 instructions. if (__builtin_cpu_supports("sse2")) { return SIMD_SSE2; @@ -176,7 +180,8 @@ static inline SIMD_Implementation find_simd_implementation(void) { #endif /* JSON_ENABLE_SIMD */ #ifndef FIND_SIMD_IMPLEMENTATION_DEFINED -static inline SIMD_Implementation find_simd_implementation(void) { +static inline SIMD_Implementation find_simd_implementation(void) +{ return SIMD_NONE; } #endif From b362c088ecd12145ac9a8cc6732144da0c565515 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 9 Jul 2025 20:38:56 -0500 Subject: [PATCH 094/506] Use the platform native path.separator when building using JRuby. --- Rakefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index 5fc7fa6dc..72ce13740 100644 --- a/Rakefile +++ b/Rakefile @@ -48,6 +48,8 @@ desc "Generate parser with ragel" task :ragel => [JAVA_PARSER_SRC] if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'java' + path_sep = java.lang.System.getProperty("path.separator") ENV['JAVA_HOME'] ||= [ '/usr/local/java/jdk', '/usr/lib/jvm/java-6-openjdk', @@ -55,7 +57,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' ].find { |c| File.directory?(c) } if ENV['JAVA_HOME'] warn " *** JAVA_HOME is set to #{ENV['JAVA_HOME'].inspect}" - ENV['PATH'] = ENV['PATH'].split(/:/).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * ':' + ENV['PATH'] = ENV['PATH'].split(/path_sep/).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * path_sep warn " *** java binaries are assumed to be in #{java_path.inspect}" else warn " *** JAVA_HOME was not set or could not be guessed!" @@ -65,7 +67,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' JRUBY_JAR = File.join(CONFIG["libdir"], "jruby.jar") if File.exist?(JRUBY_JAR) JAVA_SOURCES.each do |src| - classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * ':' + classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_sep obj = src.sub(/\.java\Z/, '.class') file obj => src do sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src From 90680fa85ab8f00733c25632c488b8fd6bd95fdb Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 9 Jul 2025 20:48:34 -0500 Subject: [PATCH 095/506] Use File::PATH_SEPARATOR instead of java.lang.System.getProperty. --- Rakefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Rakefile b/Rakefile index 72ce13740..1117391b8 100644 --- a/Rakefile +++ b/Rakefile @@ -48,8 +48,7 @@ desc "Generate parser with ragel" task :ragel => [JAVA_PARSER_SRC] if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' - require 'java' - path_sep = java.lang.System.getProperty("path.separator") + path_separator = File::PATH_SEPARATOR ENV['JAVA_HOME'] ||= [ '/usr/local/java/jdk', '/usr/lib/jvm/java-6-openjdk', @@ -57,7 +56,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' ].find { |c| File.directory?(c) } if ENV['JAVA_HOME'] warn " *** JAVA_HOME is set to #{ENV['JAVA_HOME'].inspect}" - ENV['PATH'] = ENV['PATH'].split(/path_sep/).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * path_sep + ENV['PATH'] = ENV['PATH'].split(/path_separator/).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * path_separator warn " *** java binaries are assumed to be in #{java_path.inspect}" else warn " *** JAVA_HOME was not set or could not be guessed!" @@ -67,7 +66,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' JRUBY_JAR = File.join(CONFIG["libdir"], "jruby.jar") if File.exist?(JRUBY_JAR) JAVA_SOURCES.each do |src| - classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_sep + classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator obj = src.sub(/\.java\Z/, '.class') file obj => src do sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src From 37e689094a0152e5c3c774a0ec16f2ab70548336 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 10 Jul 2025 14:06:16 +0200 Subject: [PATCH 096/506] Fix mistake in jruby Rakefile --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 1117391b8..52e178ed1 100644 --- a/Rakefile +++ b/Rakefile @@ -56,7 +56,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' ].find { |c| File.directory?(c) } if ENV['JAVA_HOME'] warn " *** JAVA_HOME is set to #{ENV['JAVA_HOME'].inspect}" - ENV['PATH'] = ENV['PATH'].split(/path_separator/).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * path_separator + ENV['PATH'] = ENV['PATH'].split(path_separator).unshift(java_path = "#{ENV['JAVA_HOME']}/bin") * path_separator warn " *** java binaries are assumed to be in #{java_path.inspect}" else warn " *** JAVA_HOME was not set or could not be guessed!" From 8d08494deebc406f69cfaa726c1b27f31bf6bb04 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 17 Jul 2025 10:26:38 +0200 Subject: [PATCH 097/506] Release 2.13.0 --- CHANGES.md | 5 ++++- lib/json/version.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2208bd7db..562bdb6f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,11 @@ ### Unreleased -* Add bnew `allow_duplicate_key` parsing options. By default a warning is now emitted when a duplicated key is encountered. +### 2025-05-23 (2.13.0) + +* Add new `allow_duplicate_key` parsing options. By default a warning is now emitted when a duplicated key is encountered. In `json 3.0` an error will be raised. +* Optimize parsing further using SIMD to scan strings. ### 2025-05-23 (2.12.2) diff --git a/lib/json/version.rb b/lib/json/version.rb index 15ebd12f5..7aeaea43a 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.12.2' + VERSION = '2.13.0' end From c24342d801f4b5f9ab5ca37bdc2449361d3635b5 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Wed, 23 Jul 2025 10:27:08 -0700 Subject: [PATCH 098/506] Fix missing write barrier on Generator State Found by wbcheck WBCHECK ERROR: Missed write barrier detected! Parent object: 0x7b7b8487c450 (wb_protected: true) rb_obj_info_dump: 0x00007b7b8487c450 JSON/Generator/State/JSON::Ext::Generator::State JSON/Generator/State Reference counts - snapshot: 1, writebarrier: 0, current: 6, missed: 5 Missing reference to: 0x7b7b82f35a10 rb_obj_info_dump: 0x00007b7b82f35a10 T_STRING/String len: 1, capa: 15 "1" Missing reference to: 0x7b7b82f35e90 rb_obj_info_dump: 0x00007b7b82f35e90 T_STRING/String len: 1, capa: 15 "2" Missing reference to: 0x7b7b83629e50 rb_obj_info_dump: 0x00007b7b83629e50 T_STRING/String len: 1, capa: 15 "3" Missing reference to: 0x7b7b83b62190 rb_obj_info_dump: 0x00007b7b83b62190 T_STRING/String len: 1, capa: 15 "4" Missing reference to: 0x7b7b83629490 rb_obj_info_dump: 0x00007b7b83629490 T_STRING/String len: 1, capa: 15 "5" --- ext/json/ext/generator/generator.c | 45 ++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 083388537..9f56cf7f5 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1609,15 +1609,30 @@ static VALUE cState_buffer_initial_length_set(VALUE self, VALUE buffer_initial_l return Qnil; } +struct configure_state_data { + JSON_Generator_State *state; + VALUE vstate; // Ruby object that owns the state, or Qfalse if stack-allocated +}; + +static inline void state_write_value(struct configure_state_data *data, VALUE *field, VALUE value) +{ + if (RTEST(data->vstate)) { + RB_OBJ_WRITE(data->vstate, field, value); + } else { + *field = value; + } +} + static int configure_state_i(VALUE key, VALUE val, VALUE _arg) { - JSON_Generator_State *state = (JSON_Generator_State *)_arg; + struct configure_state_data *data = (struct configure_state_data *)_arg; + JSON_Generator_State *state = data->state; - if (key == sym_indent) { state->indent = string_config(val); } - else if (key == sym_space) { state->space = string_config(val); } - else if (key == sym_space_before) { state->space_before = string_config(val); } - else if (key == sym_object_nl) { state->object_nl = string_config(val); } - else if (key == sym_array_nl) { state->array_nl = string_config(val); } + if (key == sym_indent) { state_write_value(data, &state->indent, string_config(val)); } + else if (key == sym_space) { state_write_value(data, &state->space, string_config(val)); } + else if (key == sym_space_before) { state_write_value(data, &state->space_before, string_config(val)); } + else if (key == sym_object_nl) { state_write_value(data, &state->object_nl, string_config(val)); } + else if (key == sym_array_nl) { state_write_value(data, &state->array_nl, string_config(val)); } else if (key == sym_max_nesting) { state->max_nesting = long_config(val); } else if (key == sym_allow_nan) { state->allow_nan = RTEST(val); } else if (key == sym_ascii_only) { state->ascii_only = RTEST(val); } @@ -1626,11 +1641,14 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_script_safe) { state->script_safe = RTEST(val); } else if (key == sym_escape_slash) { state->script_safe = RTEST(val); } else if (key == sym_strict) { state->strict = RTEST(val); } - else if (key == sym_as_json) { state->as_json = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; } + else if (key == sym_as_json) { + VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; + state_write_value(data, &state->as_json, proc); + } return ST_CONTINUE; } -static void configure_state(JSON_Generator_State *state, VALUE config) +static void configure_state(JSON_Generator_State *state, VALUE vstate, VALUE config) { if (!RTEST(config)) return; @@ -1638,15 +1656,20 @@ static void configure_state(JSON_Generator_State *state, VALUE config) if (!RHASH_SIZE(config)) return; + struct configure_state_data data = { + .state = state, + .vstate = vstate + }; + // We assume in most cases few keys are set so it's faster to go over // the provided keys than to check all possible keys. - rb_hash_foreach(config, configure_state_i, (VALUE)state); + rb_hash_foreach(config, configure_state_i, (VALUE)&data); } static VALUE cState_configure(VALUE self, VALUE opts) { GET_STATE(self); - configure_state(state, opts); + configure_state(state, self, opts); return self; } @@ -1654,7 +1677,7 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) { JSON_Generator_State state = {0}; state_init(&state); - configure_state(&state, opts); + configure_state(&state, Qfalse, opts); char stack_buffer[FBUFFER_STACK_SIZE]; FBuffer buffer = { From da878435dc44c2c4b5a9890e5557170dc310347d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Jul 2025 13:24:55 +0200 Subject: [PATCH 099/506] Don't assume `__builtin_cpu_supports` exists Fix: https://github.com/ruby/json/issues/827 On very old compilers it might not exist, at that point might as well skip SIMD entirely. --- CHANGES.md | 2 ++ ext/json/ext/simd/conf.rb | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 562bdb6f2..697de4e7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ # Changes +* Fix support for older compilers without `__builtin_cpu_supports`. + ### Unreleased ### 2025-05-23 (2.13.0) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index 8e7d8ee26..76f774bc9 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -1,20 +1,24 @@ case RbConfig::CONFIG['host_cpu'] when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions - header, type, init = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)' + header, type, init, extra = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)', nil when /^(x86_64|x64)/ - header, type, init = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)' + header, type, init, extra = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)', 'if (__builtin_cpu_supports("sse2")) { printf("OK"); }' end if header - have_header(header) && try_compile(<<~SRC) - #{cpp_include(header)} - int main(int argc, char **argv) { - #{type} test = #{init}; - if (argc > 100000) printf("%p", &test); - return 0; - } - SRC - $defs.push("-DJSON_ENABLE_SIMD") + if have_header(header) && try_compile(<<~SRC, '-Werror=implicit-function-declaration') + #{cpp_include(header)} + int main(int argc, char **argv) { + #{type} test = #{init}; + #{extra} + if (argc > 100000) printf("%p", &test); + return 0; + } + SRC + $defs.push("-DJSON_ENABLE_SIMD") + else + puts "Disable SIMD" + end end have_header('cpuid.h') From cfe9337eda64a1bd7edad53053b0079c33868d9e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 24 Jul 2025 14:05:58 +0200 Subject: [PATCH 100/506] Release 2.13.1 --- CHANGES.md | 8 +++++--- lib/json/version.rb | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 697de4e7f..58e3f60d2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,12 @@ # Changes -* Fix support for older compilers without `__builtin_cpu_supports`. - ### Unreleased -### 2025-05-23 (2.13.0) +### 2025-07-24 (2.13.1) + +* Fix support for older compilers without `__builtin_cpu_supports`. + +### 2025-07-17 (2.13.0) * Add new `allow_duplicate_key` parsing options. By default a warning is now emitted when a duplicated key is encountered. In `json 3.0` an error will be raised. diff --git a/lib/json/version.rb b/lib/json/version.rb index 7aeaea43a..2aef3d7f6 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.13.0' + VERSION = '2.13.1' end From 020693b17a84116487cf1382d386cca5d3cac770 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 27 Jul 2025 14:58:53 +0900 Subject: [PATCH 101/506] Functions defined in headers should be `static inline` If `load_uint8x16_4` has an external linkage, it is defined in both `generator` and `parser` extension libraries. This duplicate symbol causes a linker error when `--with-static-linked-ext` is given, on some platforms. --- ext/json/ext/simd/simd.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index f8503d139..e424bda9a 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -103,7 +103,8 @@ static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const cha return 0; } -uint8x16x4_t load_uint8x16_4(const unsigned char *table) { +static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) +{ uint8x16x4_t tab; tab.val[0] = vld1q_u8(table); tab.val[1] = vld1q_u8(table+16); From 1988a3ae4c5675889dc4523b517298b6383caadd Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 27 Jul 2025 15:34:28 +0900 Subject: [PATCH 102/506] Keep indentation consistent across functions --- ext/json/ext/simd/simd.h | 90 ++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index e424bda9a..3abbdb020 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -7,45 +7,45 @@ typedef enum { #ifdef JSON_ENABLE_SIMD #ifdef __clang__ - #if __has_builtin(__builtin_ctzll) - #define HAVE_BUILTIN_CTZLL 1 - #else - #define HAVE_BUILTIN_CTZLL 0 - #endif +# if __has_builtin(__builtin_ctzll) +# define HAVE_BUILTIN_CTZLL 1 +# else +# define HAVE_BUILTIN_CTZLL 0 +# endif #elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) - #define HAVE_BUILTIN_CTZLL 1 +# define HAVE_BUILTIN_CTZLL 1 #else - #define HAVE_BUILTIN_CTZLL 0 +# define HAVE_BUILTIN_CTZLL 0 #endif static inline uint32_t trailing_zeros64(uint64_t input) { #if HAVE_BUILTIN_CTZLL - return __builtin_ctzll(input); + return __builtin_ctzll(input); #else - uint32_t trailing_zeros = 0; - uint64_t temp = input; - while ((temp & 1) == 0 && temp > 0) { - trailing_zeros++; - temp >>= 1; - } - return trailing_zeros; + uint32_t trailing_zeros = 0; + uint64_t temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; #endif } static inline int trailing_zeros(int input) { - #if HAVE_BUILTIN_CTZLL +#if HAVE_BUILTIN_CTZLL return __builtin_ctz(input); - #else +#else int trailing_zeros = 0; int temp = input; while ((temp & 1) == 0 && temp > 0) { - trailing_zeros++; - temp >>= 1; + trailing_zeros++; + temp >>= 1; } return trailing_zeros; - #endif +#endif } #if (defined(__GNUC__ ) || defined(__clang__)) @@ -79,38 +79,38 @@ static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches) static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr) { - uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); + uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); - // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 - // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ - const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33)); + // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 + // https://lemire.me/blog/2025/04/13/detect-control-characters-quotes-and-backslashes-efficiently-using-swar/ + const uint8x16_t too_low_or_dbl_quote = vcltq_u8(veorq_u8(chunk, vdupq_n_u8(2)), vdupq_n_u8(33)); - uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\')); - uint8x16_t needs_escape = vorrq_u8(too_low_or_dbl_quote, has_backslash); - return neon_match_mask(needs_escape); + uint8x16_t has_backslash = vceqq_u8(chunk, vdupq_n_u8('\\')); + uint8x16_t needs_escape = vorrq_u8(too_low_or_dbl_quote, has_backslash); + return neon_match_mask(needs_escape); } static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) { while (*ptr + sizeof(uint8x16_t) <= end) { - uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); - if (chunk_mask) { - *mask = chunk_mask; - return 1; - } - *ptr += sizeof(uint8x16_t); + uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); + if (chunk_mask) { + *mask = chunk_mask; + return 1; + } + *ptr += sizeof(uint8x16_t); } return 0; } static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) { - uint8x16x4_t tab; - tab.val[0] = vld1q_u8(table); - tab.val[1] = vld1q_u8(table+16); - tab.val[2] = vld1q_u8(table+32); - tab.val[3] = vld1q_u8(table+48); - return tab; + uint8x16x4_t tab; + tab.val[0] = vld1q_u8(table); + tab.val[1] = vld1q_u8(table+16); + tab.val[2] = vld1q_u8(table+32); + tab.val[3] = vld1q_u8(table+48); + return tab; } #endif /* ARM Neon Support.*/ @@ -151,12 +151,12 @@ static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *p static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) { while (*ptr + sizeof(__m128i) <= end) { - int chunk_mask = compute_chunk_mask_sse2(*ptr); - if (chunk_mask) { - *mask = chunk_mask; - return 1; - } - *ptr += sizeof(__m128i); + int chunk_mask = compute_chunk_mask_sse2(*ptr); + if (chunk_mask) { + *mask = chunk_mask; + return 1; + } + *ptr += sizeof(__m128i); } return 0; From e3de4cc59c4c2ad7a44895a2206094bc54b17826 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 27 Jul 2025 13:13:11 +0200 Subject: [PATCH 103/506] Improve duplicate key warning and errors to include the key name Followup: https://github.com/ruby/json/pull/818 --- CHANGES.md | 2 ++ ext/json/ext/parser/parser.c | 50 +++++++++++++++++++++++++++-- java/src/json/ext/ParserConfig.java | 4 +-- java/src/json/ext/ParserConfig.rl | 4 +-- test/json/json_parser_test.rb | 21 +++++++++++- 5 files changed, 73 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 58e3f60d2..53a18b900 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Improve duplicate key warning and errors to include the key name. + ### 2025-07-24 (2.13.1) * Fix support for older compilers without `__builtin_cpu_supports`. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 01b6e6293..8d72c18ac 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -426,6 +426,7 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state) } #define PARSE_ERROR_FRAGMENT_LEN 32 + #ifdef RBIMPL_ATTR_NORETURN RBIMPL_ATTR_NORETURN() #endif @@ -830,21 +831,64 @@ static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig return array; } +static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs) +{ + VALUE set = rb_hash_new_capa(count / 2); + for (size_t index = 0; index < count; index += 2) { + size_t before = RHASH_SIZE(set); + VALUE key = pairs[index]; + rb_hash_aset(set, key, Qtrue); + if (RHASH_SIZE(set) == before) { + if (RB_SYMBOL_P(key)) { + return rb_sym2str(key); + } + return key; + } + } + return Qfalse; +} + +static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key) +{ + VALUE message = rb_sprintf( + "detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", + rb_inspect(duplicate_key) + ); + + emit_parse_warning(RSTRING_PTR(message), state); + RB_GC_GUARD(message); +} + +#ifdef RBIMPL_ATTR_NORETURN +RBIMPL_ATTR_NORETURN() +#endif +static void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key) +{ + VALUE message = rb_sprintf( + "duplicate key %"PRIsVALUE, + rb_inspect(duplicate_key) + ); + + raise_parse_error(RSTRING_PTR(message), state); + RB_GC_GUARD(message); +} + static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count) { size_t entries_count = count / 2; VALUE object = rb_hash_new_capa(entries_count); - rb_hash_bulk_insert(count, rvalue_stack_peek(state->stack, count), object); + const VALUE *pairs = rvalue_stack_peek(state->stack, count); + rb_hash_bulk_insert(count, pairs, object); if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) { switch (config->on_duplicate_key) { case JSON_IGNORE: break; case JSON_DEPRECATED: - emit_parse_warning("detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", state); + emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); break; case JSON_RAISE: - raise_parse_error("duplicate key", state); + raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); break; } } diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index ccfc558e5..2bd03babc 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -2125,10 +2125,10 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) if (((RubyHash)result).hasKey(lastName)) { if (config.deprecateDuplicateKey) { context.runtime.getWarnings().warning( - "detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" + "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" ); } else { - throw parsingError(context, "duplicate key", p, pe); + throw parsingError(context, "duplicate key" + name.inspect(), p, pe); } } } diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 4bc5d93bd..533b13227 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -692,10 +692,10 @@ public class ParserConfig extends RubyObject { if (((RubyHash)result).hasKey(lastName)) { if (config.deprecateDuplicateKey) { context.runtime.getWarnings().warning( - "detected duplicate keys in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" + "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" ); } else { - throw parsingError(context, "duplicate key", p, pe); + throw parsingError(context, "duplicate key" + name.inspect(), p, pe); } } } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 106492e1c..be13c5ed3 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -333,11 +333,30 @@ def test_parse_big_integers def test_parse_duplicate_key expected = {"a" => 2} + expected_sym = {a: 2} + assert_equal expected, parse('{"a": 1, "a": 2}', allow_duplicate_key: true) assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false) } - assert_deprecated_warning(/duplicate keys/) do + assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false, symbolize_names: true) } + + assert_deprecated_warning(/duplicate key "a"/) do assert_equal expected, parse('{"a": 1, "a": 2}') end + assert_deprecated_warning(/duplicate key "a"/) do + assert_equal expected_sym, parse('{"a": 1, "a": 2}', symbolize_names: true) + end + + unless RUBY_ENGINE == 'jruby' + assert_raise(ParserError) do + fake_key = Object.new + JSON.load('{"a": 1, "a": 2}', -> (obj) { obj == "a" ? fake_key : obj }, allow_duplicate_key: false) + end + + assert_deprecated_warning(/duplicate key # (obj) { obj == "a" ? fake_key : obj }) + end + end end def test_some_wrong_inputs From cd5155738792a0ace44b0c366eb8a46166b45932 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 27 Jul 2025 14:31:40 +0200 Subject: [PATCH 104/506] Fix duplicated key warning location Followup: https://github.com/ruby/json/pull/818 Now the warning should point at the `JSON.parse` caller, and not inside the json gem itself. --- ext/json/ext/parser/parser.c | 3 ++- lib/json/common.rb | 35 ++++++++++++++++++----------------- test/json/json_parser_test.rb | 6 ++++++ 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 8d72c18ac..ab9d6c205 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -422,7 +422,8 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state) long line, column; cursor_position(state, &line, &column); - rb_warn("%s at line %ld column %ld", message, line, column); + VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column); + rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning); } #define PARSE_ERROR_FRAGMENT_LEN 32 diff --git a/lib/json/common.rb b/lib/json/common.rb index 486ec62a5..69e436f19 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -48,7 +48,7 @@ def array_class_proc(array_class, on_load) end end - # TODO: exctract :create_additions support to another gem for version 3.0 + # TODO: extract :create_additions support to another gem for version 3.0 def create_additions_proc(opts) if opts[:symbolize_names] raise ArgumentError, "options :symbolize_names and :create_additions cannot be used in conjunction" @@ -87,31 +87,32 @@ def create_additions_proc(opts) opts end - GEM_ROOT = File.expand_path("../../../", __FILE__) + "/" def create_additions_warning - message = "JSON.load implicit support for `create_additions: true` is deprecated " \ + JSON.deprecation_warning "JSON.load implicit support for `create_additions: true` is deprecated " \ "and will be removed in 3.0, use JSON.unsafe_load or explicitly " \ "pass `create_additions: true`" + end + end + end - uplevel = 4 - caller_locations(uplevel, 10).each do |frame| - if frame.path.nil? || frame.path.start_with?(GEM_ROOT) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") - uplevel += 1 - else - break - end - end - - if RUBY_VERSION >= "3.0" - warn(message, uplevel: uplevel - 1, category: :deprecated) + class << self + def deprecation_warning(message, uplevel = 4) # :nodoc: + gem_root = File.expand_path("../../../", __FILE__) + "/" + caller_locations(uplevel, 10).each do |frame| + if frame.path.nil? || frame.path.start_with?(gem_root) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") + uplevel += 1 else - warn(message, uplevel: uplevel - 1) + break end end + + if RUBY_VERSION >= "3.0" + warn(message, uplevel: uplevel - 1, category: :deprecated) + else + warn(message, uplevel: uplevel - 1) + end end - end - class << self # :call-seq: # JSON[object] -> new_array or new_string # diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index be13c5ed3..6455d2971 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -346,6 +346,12 @@ def test_parse_duplicate_key assert_equal expected_sym, parse('{"a": 1, "a": 2}', symbolize_names: true) end + if RUBY_ENGINE == 'RUBY_ENGINE' + assert_deprecated_warning(/#{File.basename(__FILE__)}\:#{__LINE__ + 1}/) do + assert_equal expected, parse('{"a": 1, "a": 2}') + end + end + unless RUBY_ENGINE == 'jruby' assert_raise(ParserError) do fake_key = Object.new From 132049bde235d3f401fa4ee3a99adf751dbeb476 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 27 Jul 2025 16:19:50 +0200 Subject: [PATCH 105/506] Improve deprecation warning location detection --- lib/json/common.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 69e436f19..9a878cead 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -96,7 +96,7 @@ def create_additions_warning end class << self - def deprecation_warning(message, uplevel = 4) # :nodoc: + def deprecation_warning(message, uplevel = 3) # :nodoc: gem_root = File.expand_path("../../../", __FILE__) + "/" caller_locations(uplevel, 10).each do |frame| if frame.path.nil? || frame.path.start_with?(gem_root) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") @@ -107,9 +107,9 @@ def deprecation_warning(message, uplevel = 4) # :nodoc: end if RUBY_VERSION >= "3.0" - warn(message, uplevel: uplevel - 1, category: :deprecated) + warn(message, uplevel: uplevel, category: :deprecated) else - warn(message, uplevel: uplevel - 1) + warn(message, uplevel: uplevel) end end From 9e3efbfa2223ad30c39bbeb6e549a90bab04a0c9 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 28 Jul 2025 09:02:29 +0200 Subject: [PATCH 106/506] Release 2.13.2 --- CHANGES.md | 4 +++- lib/json/version.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 53a18b900..13b7e279f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,9 @@ ### Unreleased -* Improve duplicate key warning and errors to include the key name. +### 2025-07-28 (2.13.2) + +* Improve duplicate key warning and errors to include the key name and point to the right caller. ### 2025-07-24 (2.13.1) diff --git a/lib/json/version.rb b/lib/json/version.rb index 2aef3d7f6..f9ac3e17a 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.13.1' + VERSION = '2.13.2' end From d3dc415132a11ebf97d27c2acd392fea0f1e2463 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 3 Aug 2025 10:33:30 +0200 Subject: [PATCH 107/506] Only define `String.json_create` & al when `json/add` is required All the `json/add` related methods for string were always defined unconditionally from the generators. It's preferable to only define them if `json/add` is actually used. --- ext/json/ext/generator/generator.c | 65 +---------------- java/src/json/ext/GeneratorMethods.java | 92 ------------------------- lib/json/add/core.rb | 1 + lib/json/add/string.rb | 35 ++++++++++ lib/json/truffle_ruby/generator.rb | 33 --------- 5 files changed, 37 insertions(+), 189 deletions(-) create mode 100644 lib/json/add/string.rb diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d60fbba41..5248a9e26 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -31,7 +31,7 @@ typedef struct JSON_Generator_StateStruct { #define RB_UNLIKELY(cond) (cond) #endif -static VALUE mJSON, cState, cFragment, mString_Extend, eGeneratorError, eNestingError, Encoding_UTF_8; +static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8; static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, @@ -835,18 +835,6 @@ static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self) return cState_partial_generate(Vstate, self, generate_json_float, Qfalse); } -/* - * call-seq: String.included(modul) - * - * Extends _modul_ with the String::Extend module. - */ -static VALUE mString_included_s(VALUE self, VALUE modul) -{ - VALUE result = rb_funcall(modul, i_extend, 1, mString_Extend); - rb_call_super(1, &modul); - return result; -} - /* * call-seq: to_json(*) * @@ -861,51 +849,6 @@ static VALUE mString_to_json(int argc, VALUE *argv, VALUE self) return cState_partial_generate(Vstate, self, generate_json_string, Qfalse); } -/* - * call-seq: to_json_raw_object() - * - * This method creates a raw object hash, that can be nested into - * other data structures and will be generated as a raw string. This - * method should be used, if you want to convert raw strings to JSON - * instead of UTF-8 strings, e. g. binary data. - */ -static VALUE mString_to_json_raw_object(VALUE self) -{ - VALUE ary; - VALUE result = rb_hash_new(); - rb_hash_aset(result, rb_funcall(mJSON, i_create_id, 0), rb_class_name(rb_obj_class(self))); - ary = rb_funcall(self, i_unpack, 1, rb_str_new2("C*")); - rb_hash_aset(result, rb_utf8_str_new_lit("raw"), ary); - return result; -} - -/* - * call-seq: to_json_raw(*args) - * - * This method creates a JSON text from the result of a call to - * to_json_raw_object of this String. - */ -static VALUE mString_to_json_raw(int argc, VALUE *argv, VALUE self) -{ - VALUE obj = mString_to_json_raw_object(self); - Check_Type(obj, T_HASH); - return mHash_to_json(argc, argv, obj); -} - -/* - * call-seq: json_create(o) - * - * Raw Strings are JSON Objects (the raw bytes are stored in an array for the - * key "raw"). The Ruby String can be created by this module method. - */ -static VALUE mString_Extend_json_create(VALUE self, VALUE o) -{ - VALUE ary; - Check_Type(o, T_HASH); - ary = rb_hash_aref(o, rb_str_new2("raw")); - return rb_funcall(ary, i_pack, 1, rb_str_new2("C*")); -} - /* * call-seq: to_json(*) * @@ -2091,13 +2034,7 @@ void Init_generator(void) rb_define_method(mFloat, "to_json", mFloat_to_json, -1); VALUE mString = rb_define_module_under(mGeneratorMethods, "String"); - rb_define_singleton_method(mString, "included", mString_included_s, 1); rb_define_method(mString, "to_json", mString_to_json, -1); - rb_define_method(mString, "to_json_raw", mString_to_json_raw, -1); - rb_define_method(mString, "to_json_raw_object", mString_to_json_raw_object, 0); - - mString_Extend = rb_define_module_under(mString, "Extend"); - rb_define_method(mString_Extend, "json_create", mString_Extend_json_create, 1); VALUE mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass"); rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1); diff --git a/java/src/json/ext/GeneratorMethods.java b/java/src/json/ext/GeneratorMethods.java index eb31b9efc..3019ae091 100644 --- a/java/src/json/ext/GeneratorMethods.java +++ b/java/src/json/ext/GeneratorMethods.java @@ -43,9 +43,6 @@ static void populate(RuntimeInfo info, RubyModule module) { defineMethods(module, "Object", RbObject.class); defineMethods(module, "String", RbString.class); defineMethods(module, "TrueClass", RbTrue.class); - - RubyModule stringExtend = module.defineModuleUnder("String").defineModuleUnder("Extend"); - stringExtend.defineAnnotatedMethods(StringExtend.class); } /** @@ -118,95 +115,6 @@ public static IRubyObject to_json(ThreadContext context, IRubyObject vSelf) { public static IRubyObject to_json(ThreadContext context, IRubyObject vSelf, IRubyObject arg0) { return Generator.generateJson(context, (RubyString)vSelf, Generator.STRING_HANDLER, arg0); } - - /** - * {@link RubyString String}#to_json_raw(*) - * - *

This method creates a JSON text from the result of a call to - * {@link #to_json_raw_object} of this String. - */ - @JRubyMethod - public static IRubyObject to_json_raw(ThreadContext context, IRubyObject vSelf) { - RubyHash obj = toJsonRawObject(context, Utils.ensureString(vSelf)); - return Generator.generateJson(context, obj, Generator.HASH_HANDLER); - } - - @JRubyMethod - public static IRubyObject to_json_raw(ThreadContext context, IRubyObject vSelf, IRubyObject arg0) { - RubyHash obj = toJsonRawObject(context, Utils.ensureString(vSelf)); - return Generator.generateJson(context, obj, Generator.HASH_HANDLER, arg0); - } - - /** - * {@link RubyString String}#to_json_raw_object(*) - * - *

This method creates a raw object Hash, that can be nested into - * other data structures and will be unparsed as a raw string. This - * method should be used if you want to convert raw strings to JSON - * instead of UTF-8 strings, e.g. binary data. - */ - @JRubyMethod - public static IRubyObject to_json_raw_object(ThreadContext context, IRubyObject vSelf) { - return toJsonRawObject(context, Utils.ensureString(vSelf)); - } - - private static RubyHash toJsonRawObject(ThreadContext context, - RubyString self) { - Ruby runtime = context.runtime; - RubyHash result = RubyHash.newHash(runtime); - - IRubyObject createId = RuntimeInfo.forRuntime(runtime) - .jsonModule.get().callMethod(context, "create_id"); - result.op_aset(context, createId, self.getMetaClass().to_s()); - - ByteList bl = self.getByteList(); - byte[] uBytes = bl.unsafeBytes(); - RubyArray array = runtime.newArray(bl.length()); - for (int i = bl.begin(), t = bl.begin() + bl.length(); i < t; i++) { - array.store(i, runtime.newFixnum(uBytes[i] & 0xff)); - } - - result.op_aset(context, runtime.newString("raw"), array); - return result; - } - - @JRubyMethod(module=true) - public static IRubyObject included(ThreadContext context, IRubyObject extendModule, IRubyObject module) { - RuntimeInfo info = RuntimeInfo.forRuntime(context.runtime); - return module.callMethod(context, "extend", ((RubyModule) extendModule).getConstant("Extend")); - } - } - - public static class StringExtend { - /** - * {@link RubyString String}#json_create(o) - * - *

Raw Strings are JSON Objects (the raw bytes are stored in an - * array for the key "raw"). The Ruby String can be created by this - * module method. - */ - @JRubyMethod - public static IRubyObject json_create(ThreadContext context, - IRubyObject vSelf, IRubyObject vHash) { - Ruby runtime = context.runtime; - RubyHash o = vHash.convertToHash(); - IRubyObject rawData = o.fastARef(runtime.newString("raw")); - if (rawData == null) { - throw runtime.newArgumentError("\"raw\" value not defined " - + "for encoded String"); - } - RubyArray ary = Utils.ensureArray(rawData); - byte[] bytes = new byte[ary.getLength()]; - for (int i = 0, t = ary.getLength(); i < t; i++) { - IRubyObject element = ary.eltInternal(i); - if (element instanceof RubyFixnum) { - bytes[i] = (byte)RubyNumeric.fix2long(element); - } else { - throw runtime.newTypeError(element, runtime.getFixnum()); - } - } - return runtime.newString(new ByteList(bytes, false)); - } } public static class RbTrue { diff --git a/lib/json/add/core.rb b/lib/json/add/core.rb index 485f097ff..61ff45421 100644 --- a/lib/json/add/core.rb +++ b/lib/json/add/core.rb @@ -7,6 +7,7 @@ require 'json/add/exception' require 'json/add/range' require 'json/add/regexp' +require 'json/add/string' require 'json/add/struct' require 'json/add/symbol' require 'json/add/time' diff --git a/lib/json/add/string.rb b/lib/json/add/string.rb new file mode 100644 index 000000000..46f07967c --- /dev/null +++ b/lib/json/add/string.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED + require 'json' +end + +class String + # call-seq: json_create(o) + # + # Raw Strings are JSON Objects (the raw bytes are stored in an array for the + # key "raw"). The Ruby String can be created by this class method. + def self.json_create(object) + object["raw"].pack("C*") + end + + # call-seq: to_json_raw_object() + # + # This method creates a raw object hash, that can be nested into + # other data structures and will be generated as a raw string. This + # method should be used, if you want to convert raw strings to JSON + # instead of UTF-8 strings, e. g. binary data. + def to_json_raw_object + { + JSON.create_id => self.class.name, + "raw" => unpack("C*"), + } + end + + # call-seq: to_json_raw(*args) + # + # This method creates a JSON text from the result of a call to + # to_json_raw_object of this String. + def to_json_raw(...) + to_json_raw_object.to_json(...) + end +end \ No newline at end of file diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index c814106de..aba20068a 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -635,39 +635,6 @@ def to_json(state = nil, *args) rescue Encoding::UndefinedConversionError => error raise ::JSON::GeneratorError.new(error.message, self) end - - # Module that holds the extending methods if, the String module is - # included. - module Extend - # Raw Strings are JSON Objects (the raw bytes are stored in an - # array for the key "raw"). The Ruby String can be created by this - # module method. - def json_create(o) - o['raw'].pack('C*') - end - end - - # Extends _modul_ with the String::Extend module. - def self.included(modul) - modul.extend Extend - end - - # This method creates a raw object hash, that can be nested into - # other data structures and will be unparsed as a raw string. This - # method should be used, if you want to convert raw strings to JSON - # instead of UTF-8 strings, e. g. binary data. - def to_json_raw_object - { - JSON.create_id => self.class.name, - 'raw' => self.unpack('C*'), - } - end - - # This method creates a JSON text from the result of - # a call to to_json_raw_object of this String. - def to_json_raw(*args) - to_json_raw_object.to_json(*args) - end end module TrueClass From 2d2e0d403dbefad6aa69bcad9035d86a2e3a83e2 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 7 Aug 2025 16:30:48 +0900 Subject: [PATCH 108/506] Remove trailing spaces [ci skip] --- lib/json/add/string.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/json/add/string.rb b/lib/json/add/string.rb index 46f07967c..64b1c217f 100644 --- a/lib/json/add/string.rb +++ b/lib/json/add/string.rb @@ -5,7 +5,7 @@ class String # call-seq: json_create(o) - # + # # Raw Strings are JSON Objects (the raw bytes are stored in an array for the # key "raw"). The Ruby String can be created by this class method. def self.json_create(object) @@ -13,7 +13,7 @@ def self.json_create(object) end # call-seq: to_json_raw_object() - # + # # This method creates a raw object hash, that can be nested into # other data structures and will be generated as a raw string. This # method should be used, if you want to convert raw strings to JSON @@ -26,7 +26,7 @@ def to_json_raw_object end # call-seq: to_json_raw(*args) - # + # # This method creates a JSON text from the result of a call to # to_json_raw_object of this String. def to_json_raw(...) From 72e231f929efe94660e031ab779dd0cb8b33a516 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 8 Aug 2025 19:48:29 +0900 Subject: [PATCH 109/506] Append newline at EOF [ci skip] --- lib/json/add/string.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/add/string.rb b/lib/json/add/string.rb index 64b1c217f..9c3bde27f 100644 --- a/lib/json/add/string.rb +++ b/lib/json/add/string.rb @@ -32,4 +32,4 @@ def to_json_raw_object def to_json_raw(...) to_json_raw_object.to_json(...) end -end \ No newline at end of file +end From 4686b2ed9246c9e4ae037420cdf52b1dd7bad379 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:07:35 +0000 Subject: [PATCH 110/506] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27a40bdd7..6c2aafaac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - { os: windows-latest, ruby: jruby-head } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 @@ -69,7 +69,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 From 6bca9962f31e82cb10cc5522c01422eb1aa40211 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Wed, 13 Aug 2025 13:52:40 -0700 Subject: [PATCH 111/506] Fix typos --- CHANGES.md | 16 +++--- benchmark/encoder.rb | 4 +- ext/json/ext/generator/generator.c | 4 +- ext/json/ext/parser/parser.c | 2 +- java/src/json/ext/ParserConfig.java | 84 ++++++++++++++--------------- lib/json.rb | 2 +- lib/json/common.rb | 2 +- test/json/ractor_test.rb | 2 +- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 13b7e279f..eb6982ad5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,7 +44,7 @@ ### 2025-04-24 (2.11.1) * Add back `JSON.restore`, `JSON.unparse`, `JSON.fast_unparse` and `JSON.pretty_unparse`. - These were deprecated 16 years ago, but never emited warnings, only undocumented, so are + These were deprecated 16 years ago, but never emitted warnings, only undocumented, so are still used by a few gems. ### 2025-04-24 (2.11.0) @@ -71,7 +71,7 @@ ### 2025-03-12 (2.10.2) * Fix a potential crash in the C extension parser. -* Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` unadvertently changed it. +* Raise a ParserError on all incomplete unicode escape sequence. This was the behavior until `2.10.0` inadvertently changed it. * Ensure document snippets that are included in parser errors don't include truncated multibyte characters. * Ensure parser error snippets are valid UTF-8. * Fix `JSON::GeneratorError#detailed_message` on Ruby < 3.2 @@ -102,7 +102,7 @@ ### 2024-11-14 (2.8.2) -* `JSON.load_file` explictly read the file as UTF-8. +* `JSON.load_file` explicitly read the file as UTF-8. ### 2024-11-06 (2.8.1) @@ -110,7 +110,7 @@ ### 2024-11-06 (2.8.0) -* Emit a deprecation warning when `JSON.load` create custom types without the `create_additions` option being explictly enabled. +* Emit a deprecation warning when `JSON.load` create custom types without the `create_additions` option being explicitly enabled. * Prefer to use `JSON.unsafe_load(string)` or `JSON.load(string, create_additions: true)`. * Emit a deprecation warning when serializing valid UTF-8 strings encoded in `ASCII_8BIT` aka `BINARY`. * Bump required Ruby version to 2.7. @@ -118,7 +118,7 @@ pre-existing support for comments, make it suitable to parse `jsonc` documents. * Many performance improvements to `JSON.parse` and `JSON.load`, up to `1.7x` faster on real world documents. * Some minor performance improvements to `JSON.dump` and `JSON.generate`. -* `JSON.pretty_generate` no longer include newline inside empty object and arrays. +* `JSON.pretty_generate` no longer includes newlines inside empty object and arrays. ### 2024-11-04 (2.7.6) @@ -135,13 +135,13 @@ * Workaround a bug in 3.4.8 and older https://github.com/rubygems/rubygems/pull/6490. This bug would cause some gems with native extension to fail during compilation. * Workaround different versions of `json` and `json_pure` being loaded (not officially supported). -* Make `json_pure` Ractor compatible. +* Make `json_pure` Ractor compatible. ### 2024-10-24 (2.7.3) * Numerous performance optimizations in `JSON.generate` and `JSON.dump` (up to 2 times faster). -* Limit the size of ParserError exception messages, only include up to 32 bytes of the unparseable source. -* Fix json-pure's `Object#to_json` to accept non state arguments +* Limit the size of ParserError exception messages, only include up to 32 bytes of the unparsable source. +* Fix json-pure's `Object#to_json` to accept non-state arguments. * Fix multiline comment support in `json-pure`. * Fix `JSON.parse` to no longer mutate the argument encoding when passed an ASCII-8BIT string. * Fix `String#to_json` to raise on invalid encoding in `json-pure`. diff --git a/benchmark/encoder.rb b/benchmark/encoder.rb index f0a05dbd2..edda4e053 100644 --- a/benchmark/encoder.rb +++ b/benchmark/encoder.rb @@ -66,7 +66,7 @@ def benchmark_encoding(benchmark_name, ruby_obj, check_expected: true, except: [ # NB: Notes are based on ruby 3.3.4 (2024-07-09 revision be1089c8ec) +YJIT [arm64-darwin23] -# On the first two micro benchmarks, the limitting factor is the fixed cost of initializing the +# On the first two micro benchmarks, the limiting factor is the fixed cost of initializing the # generator state. Since `JSON.generate` now lazily allocate the `State` object we're now ~10-20% faster # than `Oj.dump`. benchmark_encoding "small mixed", [1, "string", { a: 1, b: 2 }, [3, 4, 5]] @@ -87,7 +87,7 @@ def benchmark_encoding(benchmark_name, ruby_obj, check_expected: true, except: [ # This benchmark spent the overwhelming majority of its time in `ruby_dtoa`. We rely on Ruby's implementation # which uses a relatively old version of dtoa.c from David M. Gay. # Oj in `compat` mode is ~10% slower than `json`, but in its default mode is noticeably faster here because -# it limits the precision of floats, breaking roundtriping. That's not something we should emulate. +# it limits the precision of floats, breaking roundtripping. That's not something we should emulate. # # Since a few years there are now much faster float to string implementations such as Ryu, Dragonbox, etc, # but all these are implemented in C++11 or newer, making it hard if not impossible to include them. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 5248a9e26..9c6ed9304 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -137,8 +137,8 @@ static inline FORCE_INLINE void search_flush(search_state *search) { // Do not remove this conditional without profiling, specifically escape-heavy text. // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush). - // For back-to-back characters that need to be escaped, specifcally for the SIMD code paths, this method - // will be called just before calling escape_UTF8_char_basic. There will be no characers to append for the + // For back-to-back characters that need to be escaped, specifically for the SIMD code paths, this method + // will be called just before calling escape_UTF8_char_basic. There will be no characters to append for the // consecutive characters that need to be escaped. While the fbuffer_append is a no-op if // nothing needs to be flushed, we can save a few memory references with this conditional. if (search->ptr > search->cursor) { diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index ab9d6c205..1e6ee753f 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1265,7 +1265,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; } - raise_parse_error("unreacheable: %s", state); + raise_parse_error("unreachable: %s", state); } static void json_ensure_eof(JSON_ParserState *state) diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 2bd03babc..4a44f9be8 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -301,11 +301,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int return parsingError(context, "unexpected token at '", absStart, absEnd); } - + // line 328 "ParserConfig.rl" - + // line 310 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { @@ -427,14 +427,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject result = null; - + // line 432 "ParserConfig.java" { cs = JSON_value_start; } // line 441 "ParserConfig.rl" - + // line 439 "ParserConfig.java" { int _klen; @@ -671,7 +671,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } - + // line 676 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { @@ -733,7 +733,7 @@ private static byte[] init__JSON_integer_index_offsets_0() private static final byte _JSON_integer_index_offsets[] = init__JSON_integer_index_offsets_0(); -private static byte[] init__JSON_integer_indicies_0() +private static byte[] init__JSON_integer_indices_0() { return new byte [] { 0, 2, 3, 1, 2, 3, 1, 1, 4, 1, 3, 4, @@ -741,7 +741,7 @@ private static byte[] init__JSON_integer_indicies_0() }; } -private static final byte _JSON_integer_indicies[] = init__JSON_integer_indicies_0(); +private static final byte _JSON_integer_indices[] = init__JSON_integer_indices_0(); private static byte[] init__JSON_integer_trans_targs_0() @@ -787,7 +787,7 @@ void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { int parseIntegerInternal(int p, int pe) { int cs; - + // line 792 "ParserConfig.java" { cs = JSON_integer_start; @@ -795,7 +795,7 @@ int parseIntegerInternal(int p, int pe) { // line 480 "ParserConfig.rl" int memo = p; - + // line 800 "ParserConfig.java" { int _klen; @@ -866,7 +866,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_integer_indicies[_trans]; + _trans = _JSON_integer_indices[_trans]; cs = _JSON_integer_trans_targs[_trans]; if ( _JSON_integer_trans_actions[_trans] != 0 ) { @@ -922,7 +922,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { return ConvertBytes.byteListToInum(runtime, num, 10, true); } - + // line 927 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { @@ -986,7 +986,7 @@ private static byte[] init__JSON_float_index_offsets_0() private static final byte _JSON_float_index_offsets[] = init__JSON_float_index_offsets_0(); -private static byte[] init__JSON_float_indicies_0() +private static byte[] init__JSON_float_indices_0() { return new byte [] { 0, 2, 3, 1, 2, 3, 1, 4, 5, 5, 1, 6, @@ -995,7 +995,7 @@ private static byte[] init__JSON_float_indicies_0() }; } -private static final byte _JSON_float_indicies[] = init__JSON_float_indicies_0(); +private static final byte _JSON_float_indices[] = init__JSON_float_indices_0(); private static byte[] init__JSON_float_trans_targs_0() @@ -1043,7 +1043,7 @@ void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { int parseFloatInternal(int p, int pe) { int cs; - + // line 1048 "ParserConfig.java" { cs = JSON_float_start; @@ -1051,7 +1051,7 @@ int parseFloatInternal(int p, int pe) { // line 533 "ParserConfig.rl" int memo = p; - + // line 1056 "ParserConfig.java" { int _klen; @@ -1122,7 +1122,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_float_indicies[_trans]; + _trans = _JSON_float_indices[_trans]; cs = _JSON_float_trans_targs[_trans]; if ( _JSON_float_trans_actions[_trans] != 0 ) { @@ -1168,7 +1168,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) return p; } - + // line 1173 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { @@ -1232,7 +1232,7 @@ private static byte[] init__JSON_string_index_offsets_0() private static final byte _JSON_string_index_offsets[] = init__JSON_string_index_offsets_0(); -private static byte[] init__JSON_string_indicies_0() +private static byte[] init__JSON_string_indices_0() { return new byte [] { 0, 1, 2, 3, 1, 0, 4, 1, 0, 5, 5, 5, @@ -1241,7 +1241,7 @@ private static byte[] init__JSON_string_indicies_0() }; } -private static final byte _JSON_string_indicies[] = init__JSON_string_indicies_0(); +private static final byte _JSON_string_indices[] = init__JSON_string_indices_0(); private static byte[] init__JSON_string_trans_targs_0() @@ -1278,7 +1278,7 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject result = null; - + // line 1283 "ParserConfig.java" { cs = JSON_string_start; @@ -1286,7 +1286,7 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { // line 581 "ParserConfig.rl" int memo = p; - + // line 1291 "ParserConfig.java" { int _klen; @@ -1357,7 +1357,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_string_indicies[_trans]; + _trans = _JSON_string_indices[_trans]; cs = _JSON_string_trans_targs[_trans]; if ( _JSON_string_trans_actions[_trans] != 0 ) { @@ -1429,7 +1429,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } } - + // line 1434 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { @@ -1549,7 +1549,7 @@ private static short[] init__JSON_array_index_offsets_0() private static final short _JSON_array_index_offsets[] = init__JSON_array_index_offsets_0(); -private static byte[] init__JSON_array_indicies_0() +private static byte[] init__JSON_array_indices_0() { return new byte [] { 0, 1, 0, 0, 2, 2, 3, 2, 2, 2, 4, 2, @@ -1565,7 +1565,7 @@ private static byte[] init__JSON_array_indicies_0() }; } -private static final byte _JSON_array_indicies[] = init__JSON_array_indicies_0(); +private static final byte _JSON_array_indices[] = init__JSON_array_indices_0(); private static byte[] init__JSON_array_trans_targs_0() @@ -1610,14 +1610,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); - + // line 1615 "ParserConfig.java" { cs = JSON_array_start; } // line 650 "ParserConfig.rl" - + // line 1622 "ParserConfig.java" { int _klen; @@ -1660,7 +1660,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) switch ( _JSON_array_cond_spaces[_JSON_array_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { case 0: { _widec = 65536 + (data[p] - 0); - if ( + if ( // line 608 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; @@ -1720,7 +1720,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_array_indicies[_trans]; + _trans = _JSON_array_indices[_trans]; cs = _JSON_array_trans_targs[_trans]; if ( _JSON_array_trans_actions[_trans] != 0 ) { @@ -1779,7 +1779,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } } - + // line 1784 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { @@ -1906,7 +1906,7 @@ private static short[] init__JSON_object_index_offsets_0() private static final short _JSON_object_index_offsets[] = init__JSON_object_index_offsets_0(); -private static byte[] init__JSON_object_indicies_0() +private static byte[] init__JSON_object_indices_0() { return new byte [] { 0, 1, 0, 0, 2, 3, 4, 0, 1, 5, 5, 6, @@ -1923,7 +1923,7 @@ private static byte[] init__JSON_object_indicies_0() }; } -private static final byte _JSON_object_indicies[] = init__JSON_object_indicies_0(); +private static final byte _JSON_object_indices[] = init__JSON_object_indices_0(); private static byte[] init__JSON_object_trans_targs_0() @@ -1973,14 +1973,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { // allocator test at OptionsReader#getClass IRubyObject result = RubyHash.newHash(context.runtime); - + // line 1978 "ParserConfig.java" { cs = JSON_object_start; } // line 737 "ParserConfig.rl" - + // line 1985 "ParserConfig.java" { int _klen; @@ -2023,7 +2023,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) switch ( _JSON_object_cond_spaces[_JSON_object_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { case 0: { _widec = 65536 + (data[p] - 0); - if ( + if ( // line 665 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; @@ -2083,7 +2083,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_object_indicies[_trans]; + _trans = _JSON_object_indices[_trans]; cs = _JSON_object_trans_targs[_trans]; if ( _JSON_object_trans_actions[_trans] != 0 ) { @@ -2174,7 +2174,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) res.update(config.onLoad(context, result), p + 1); } - + // line 2179 "ParserConfig.java" private static byte[] init__JSON_actions_0() { @@ -2238,7 +2238,7 @@ private static byte[] init__JSON_index_offsets_0() private static final byte _JSON_index_offsets[] = init__JSON_index_offsets_0(); -private static byte[] init__JSON_indicies_0() +private static byte[] init__JSON_indices_0() { return new byte [] { 0, 0, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, @@ -2248,7 +2248,7 @@ private static byte[] init__JSON_indicies_0() }; } -private static final byte _JSON_indicies[] = init__JSON_indicies_0(); +private static final byte _JSON_indices[] = init__JSON_indices_0(); private static byte[] init__JSON_trans_targs_0() @@ -2287,7 +2287,7 @@ public IRubyObject parseImplementation(ThreadContext context) { IRubyObject result = null; ParserResult res = new ParserResult(); - + // line 2292 "ParserConfig.java" { cs = JSON_start; @@ -2296,7 +2296,7 @@ public IRubyObject parseImplementation(ThreadContext context) { // line 776 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); - + // line 2301 "ParserConfig.java" { int _klen; @@ -2367,7 +2367,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_indicies[_trans]; + _trans = _JSON_indices[_trans]; cs = _JSON_trans_targs[_trans]; if ( _JSON_trans_actions[_trans] != 0 ) { diff --git a/lib/json.rb b/lib/json.rb index 735f23806..0ebff2f94 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -133,7 +133,7 @@ # When not specified: # # The last value is used and a deprecation warning emitted. # JSON.parse('{"a": 1, "a":2}') => {"a" => 2} -# # waring: detected duplicate keys in JSON object. +# # warning: detected duplicate keys in JSON object. # # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` # # When set to `+true+` diff --git a/lib/json/common.rb b/lib/json/common.rb index 9a878cead..e99d152a8 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1002,7 +1002,7 @@ class Coder # See {Parsing Options}[#module-JSON-label-Parsing+Options], and {Generating Options}[#module-JSON-label-Generating+Options]. # # For generation, the strict: true option is always set. When a Ruby object with no native \JSON counterpart is - # encoutered, the block provided to the initialize method is invoked, and must return a Ruby object that has a native + # encountered, the block provided to the initialize method is invoked, and must return a Ruby object that has a native # \JSON counterpart: # # module MyApp diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index dda34c64c..0ebdb0e91 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -42,7 +42,7 @@ def test_generate else puts "Expected:" puts expected_json - puts "Acutual:" + puts "Actual:" puts actual_json puts exit 1 From 8510ea5c1a317aeedb6e42f4cea5de13986e8430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 22 Aug 2025 11:54:18 +0200 Subject: [PATCH 112/506] Test behavior of parsing a too big Float --- test/json/json_parser_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 6455d2971..3c60fec87 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -128,6 +128,7 @@ def test_parse_numbers assert_equal(1.0/0, parse('Infinity', :allow_nan => true)) assert_raise(ParserError) { parse('-Infinity') } assert_equal(-1.0/0, parse('-Infinity', :allow_nan => true)) + capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) } end def test_parse_bigdecimals From e881e55e8321ce157ff53204d8fc768e1f01a90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 22 Aug 2025 11:56:36 +0200 Subject: [PATCH 113/506] Remove too big Float from fixture to avoid warning --- test/json/fixtures/pass1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/json/fixtures/pass1.json b/test/json/fixtures/pass1.json index 7828fcc13..fa9058b13 100644 --- a/test/json/fixtures/pass1.json +++ b/test/json/fixtures/pass1.json @@ -12,7 +12,7 @@ "real": -9876.543210, "e": 0.123456789e-12, "E": 1.234567890E+34, - "": 23456789012E666, + "": 23456789012E66, "zero": 0, "one": 1, "space": " ", From 8118532ca9b01caa97c9c223fd9a48137bd4880c Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Fri, 22 Aug 2025 15:00:43 +0200 Subject: [PATCH 114/506] Remove remaining `quirks_mode` from java code Most is gone since https://github.com/ruby/json/commit/7d2ad6d6556da03300a5aeadeeacaec563435773 Also add a mention to the 2.0.0 changelog that this was the version it was removed. I had to look through commits for some context behind it. --- CHANGES.md | 1 + java/src/json/ext/GeneratorState.java | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index eb6982ad5..0895827c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -286,6 +286,7 @@ ## 2015-09-11 (2.0.0) * Now complies to newest JSON RFC 7159. * Implements compatibility to ruby 2.4 integer unification. + * Removed support for `quirks_mode` option. * Drops support for old rubies whose life has ended, that is rubies < 2.0. Also see https://www.ruby-lang.org/en/news/2014/07/01/eol-for-1-8-7-and-1-9-2/ * There were still some mentions of dual GPL licensing in the source, but JSON diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 9c15699fb..142d388dc 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -82,12 +82,6 @@ public class GeneratorState extends RubyObject { */ private boolean asciiOnly = DEFAULT_ASCII_ONLY; static final boolean DEFAULT_ASCII_ONLY = false; - /** - * If set to true all JSON values generated might not be - * RFC-conform JSON documents. - */ - private boolean quirksMode = DEFAULT_QUIRKS_MODE; - static final boolean DEFAULT_QUIRKS_MODE = false; /** * If set to true the forward slash will be escaped in * json output. @@ -218,7 +212,6 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { this.maxNesting = orig.maxNesting; this.allowNaN = orig.allowNaN; this.asciiOnly = orig.asciiOnly; - this.quirksMode = orig.quirksMode; this.scriptSafe = orig.scriptSafe; this.strict = orig.strict; this.bufferInitialLength = orig.bufferInitialLength; From e77f610b21f8b391c6509bf0932e79ac30baff25 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 23 Aug 2025 16:05:07 +0200 Subject: [PATCH 115/506] Silence ractor experimental warnings --- test/json/ractor_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index 0ebdb0e91..53e1099ce 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -20,6 +20,7 @@ module RactorBackport def test_generate pid = fork do + Warning[:experimental] = false r = Ractor.new do json = JSON.generate({ 'a' => 2, From fdac27bd42fca7e29ff35bb45f63a65ef2ea0998 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 23 Aug 2025 16:06:38 +0200 Subject: [PATCH 116/506] Fix `JSON.generate` `strict: true` mode to also restrict hash keys --- CHANGES.md | 2 ++ ext/json/ext/generator/generator.c | 3 +++ java/src/json/ext/Generator.java | 3 +++ lib/json/truffle_ruby/generator.rb | 4 ++++ test/json/json_generator_test.rb | 12 ++++++++++++ 5 files changed, 24 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0895827c0..8b94c614e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix `JSON.generate` `strict: true` mode to also restrict hash keys. + ### 2025-07-28 (2.13.2) * Improve duplicate key warning and errors to include the key name and point to the right caller. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9c6ed9304..701bb41b8 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1041,6 +1041,9 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) key_to_s = rb_sym2str(key); break; default: + if (data->state->strict) { + raise_generator_error(key, "%"PRIsVALUE" not allowed in JSON", rb_funcall(key, i_to_s, 0)); + } key_to_s = rb_convert_type(key, T_STRING, "String", "to_s"); break; } diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 852509205..19b615883 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -544,6 +544,9 @@ private static void processEntry(ThreadContext context, Session session, OutputS } else if (keyClass == runtime.getSymbol()) { keyStr = ((RubySymbol) key).id2name(context); } else { + if (session.getState(context).strict()) { + throw Utils.buildGeneratorError(context, key, key + " not allowed in JSON").toThrowable(); + } keyStr = TypeConverter.convertToType(key, runtime.getString(), "to_s"); } diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index aba20068a..937cad68b 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -476,6 +476,10 @@ def json_transform(state) result << delim unless first result << state.indent * depth if indent + if state.strict? && !(Symbol === key || String === key) + raise GeneratorError.new("#{key.class} not allowed in JSON", value) + end + key_str = key.to_s if key_str.is_a?(String) key_json = key_str.to_json(state) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 914b3f4ed..963350ea4 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -404,6 +404,18 @@ def test_json_generate_unsupported_types assert_raise JSON::GeneratorError do generate(Object.new, strict: true) end + + assert_raise JSON::GeneratorError do + generate([Object.new], strict: true) + end + + assert_raise JSON::GeneratorError do + generate({ "key" => Object.new }, strict: true) + end + + assert_raise JSON::GeneratorError do + generate({ Object.new => "value" }, strict: true) + end end def test_nesting From 4a986968908645b3fe98e6a0980a754809f227f8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 23 Aug 2025 16:58:54 +0200 Subject: [PATCH 117/506] Fix `JSON::Coder` to cast non-string keys. --- CHANGES.md | 1 + ext/json/ext/generator/generator.c | 11 ++++++- java/src/json/ext/Generator.java | 46 +++++++++++++++++++++--------- lib/json/truffle_ruby/generator.rb | 8 +++++- test/json/json_coder_test.rb | 12 ++++++++ 5 files changed, 62 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8b94c614e..670b2d3ef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ### Unreleased * Fix `JSON.generate` `strict: true` mode to also restrict hash keys. +* Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. ### 2025-07-28 (2.13.2) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 701bb41b8..135b3176c 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1029,6 +1029,9 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) } VALUE key_to_s; + bool as_json_called = false; + + start: switch (rb_type(key)) { case T_STRING: if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) { @@ -1042,7 +1045,13 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) break; default: if (data->state->strict) { - raise_generator_error(key, "%"PRIsVALUE" not allowed in JSON", rb_funcall(key, i_to_s, 0)); + if (RTEST(data->state->as_json) && !as_json_called) { + key = rb_proc_call_with_block(data->state->as_json, 1, &key, Qnil); + as_json_called = true; + goto start; + } else { + raise_generator_error(key, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(key)); + } } key_to_s = rb_convert_type(key, T_STRING, "String", "to_s"); break; diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 19b615883..31f175cd7 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -520,6 +520,23 @@ static void generateHash(ThreadContext context, Session session, RubyHash object buffer.write('}'); } + private static IRubyObject castKey(ThreadContext context, IRubyObject key) { + RubyClass keyClass = key.getType(); + Ruby runtime = context.runtime; + + if (key instanceof RubyString) { + if (keyClass == runtime.getString()) { + return key; + } else { + return key.callMethod(context, "to_s"); + } + } else if (keyClass == runtime.getSymbol()) { + return ((RubySymbol) key).id2name(context); + } else { + return null; + } + } + private static void processEntry(ThreadContext context, Session session, OutputStream buffer, RubyHash.RubyHashEntry entry, boolean firstPair, ByteList objectNl, byte[] indent, ByteList spaceBefore, ByteList space) { IRubyObject key = (IRubyObject) entry.getKey(); IRubyObject value = (IRubyObject) entry.getValue(); @@ -533,21 +550,22 @@ private static void processEntry(ThreadContext context, Session session, OutputS Ruby runtime = context.runtime; - IRubyObject keyStr; - RubyClass keyClass = key.getType(); - if (key instanceof RubyString) { - if (keyClass == runtime.getString()) { - keyStr = key; - } else { - keyStr = key.callMethod(context, "to_s"); + IRubyObject keyStr = castKey(context, key); + if (keyStr == null || !(keyStr instanceof RubyString)) { + GeneratorState state = session.getState(context); + if (state.strict()) { + if (state.getAsJSON() != null) { + key = state.getAsJSON().call(context, key); + keyStr = castKey(context, key); + } + + if (keyStr == null) { + throw Utils.buildGeneratorError(context, key, key.getType().name(context) + " not allowed as object key in JSON").toThrowable(); + } } - } else if (keyClass == runtime.getSymbol()) { - keyStr = ((RubySymbol) key).id2name(context); - } else { - if (session.getState(context).strict()) { - throw Utils.buildGeneratorError(context, key, key + " not allowed in JSON").toThrowable(); + else { + keyStr = TypeConverter.convertToType(key, runtime.getString(), "to_s"); } - keyStr = TypeConverter.convertToType(key, runtime.getString(), "to_s"); } if (keyStr.getMetaClass() == runtime.getString()) { @@ -673,7 +691,7 @@ void generate(ThreadContext context, Session session, IRubyObject object, Output static RubyString generateGenericNew(ThreadContext context, Session session, IRubyObject object) { GeneratorState state = session.getState(context); if (state.strict()) { - if (state.getAsJSON() != null ) { + if (state.getAsJSON() != null) { IRubyObject value = state.getAsJSON().call(context, object); Handler handler = getHandlerFor(context.runtime, value); if (handler == GENERIC_HANDLER) { diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 937cad68b..6facf0364 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -477,7 +477,13 @@ def json_transform(state) result << state.indent * depth if indent if state.strict? && !(Symbol === key || String === key) - raise GeneratorError.new("#{key.class} not allowed in JSON", value) + if state.as_json + key = state.as_json.call(key) + end + + unless Symbol === key || String === key + raise GeneratorError.new("#{key.class} not allowed as object key in JSON", value) + end end key_str = key.to_s diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 986118191..fc4aba296 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -18,6 +18,18 @@ def test_json_coder_with_proc_with_unsupported_value assert_raise(JSON::GeneratorError) { coder.dump([Object.new]) } end + def test_json_coder_hash_key + obj = Object.new + coder = JSON::Coder.new(&:to_s) + assert_equal %({#{obj.to_s.inspect}:1}), coder.dump({ obj => 1 }) + + coder = JSON::Coder.new { 42 } + error = assert_raise JSON::GeneratorError do + coder.dump({ obj => 1 }) + end + assert_equal "Integer not allowed as object key in JSON", error.message + end + def test_json_coder_options coder = JSON::Coder.new(array_nl: "\n") do |object| 42 From 12656777dc60c83a10dbfd9b94dbc0faa16f6c1e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 23 Aug 2025 21:04:59 +0200 Subject: [PATCH 118/506] Extract `fbuffer_append_str_repeat` function --- ext/json/ext/fbuffer/fbuffer.h | 15 +++++++++++++-- ext/json/ext/generator/generator.c | 21 +++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index d32371476..dc40dec7f 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -197,11 +197,22 @@ static void fbuffer_append_str(FBuffer *fb, VALUE str) const char *newstr = StringValuePtr(str); unsigned long len = RSTRING_LEN(str); - RB_GC_GUARD(str); - fbuffer_append(fb, newstr, len); } +static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) +{ + unsigned long len = RSTRING_LEN(str); + + size_t total = repeat * len; + fbuffer_inc_capa(fb, total); + + while (repeat) { + fbuffer_append_str(fb, str); + repeat--; + } +} + static inline void fbuffer_append_char(FBuffer *fb, char newchr) { fbuffer_inc_capa(fb, 1); diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 135b3176c..52dcd24f0 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1016,16 +1016,13 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) JSON_Generator_State *state = data->state; long depth = state->depth; - int j; if (arg->iter > 0) fbuffer_append_char(buffer, ','); if (RB_UNLIKELY(data->state->object_nl)) { fbuffer_append_str(buffer, data->state->object_nl); } if (RB_UNLIKELY(data->state->indent)) { - for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, data->state->indent); - } + fbuffer_append_str_repeat(buffer, data->state->indent, depth); } VALUE key_to_s; @@ -1083,7 +1080,6 @@ static inline long increase_depth(struct generate_json_data *data) static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { - int j; long depth = increase_depth(data); if (RHASH_SIZE(obj) == 0) { @@ -1104,9 +1100,7 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat if (RB_UNLIKELY(data->state->object_nl)) { fbuffer_append_str(buffer, data->state->object_nl); if (RB_UNLIKELY(data->state->indent)) { - for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, data->state->indent); - } + fbuffer_append_str_repeat(buffer, data->state->indent, depth); } } fbuffer_append_char(buffer, '}'); @@ -1114,7 +1108,6 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat static void generate_json_array(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { - int i, j; long depth = increase_depth(data); if (RARRAY_LEN(obj) == 0) { @@ -1125,15 +1118,13 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data fbuffer_append_char(buffer, '['); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); - for (i = 0; i < RARRAY_LEN(obj); i++) { + for (int i = 0; i < RARRAY_LEN(obj); i++) { if (i > 0) { fbuffer_append_char(buffer, ','); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); } if (RB_UNLIKELY(data->state->indent)) { - for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, data->state->indent); - } + fbuffer_append_str_repeat(buffer, data->state->indent, depth); } generate_json(buffer, data, RARRAY_AREF(obj, i)); } @@ -1141,9 +1132,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data if (RB_UNLIKELY(data->state->array_nl)) { fbuffer_append_str(buffer, data->state->array_nl); if (RB_UNLIKELY(data->state->indent)) { - for (j = 0; j < depth; j++) { - fbuffer_append_str(buffer, data->state->indent); - } + fbuffer_append_str_repeat(buffer, data->state->indent, depth); } } fbuffer_append_char(buffer, ']'); From 2d63648c0a456402c4ed41c51dde38c22d1da128 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 24 Aug 2025 15:22:31 +0200 Subject: [PATCH 119/506] parser.c: Remove useless dereference --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1e6ee753f..496a76920 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -975,7 +975,7 @@ static inline bool FORCE_INLINE string_scan(JSON_ParserState *state) if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) { return 1; } - *state->cursor++; + state->cursor++; } return 0; } From 8454149d034ab7e8d1c176bfb0cddd3842088981 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 23 Aug 2025 19:57:14 +0200 Subject: [PATCH 120/506] JSON.generate: warn or raise on duplicated key Because both strings and symbols keys are serialized the same, it always has been possible to generate documents with duplicated keys: ```ruby >> puts JSON.generate({ foo: 1, "foo" => 2 }) {"foo":1,"foo":2} ``` This is pretty much always a mistake and can cause various issues because it's not guaranteed how various JSON parsers will handle this. Until now I didn't think it was possible to catch such case without tanking performance, hence why I only made the parser more strict. But I finally found a way to check for duplicated keys cheaply enough. --- CHANGES.md | 11 ++++ ext/json/ext/fbuffer/fbuffer.h | 8 +++ ext/json/ext/generator/generator.c | 73 ++++++++++++++++++++++++--- ext/json/ext/parser/parser.c | 2 +- java/src/json/ext/Generator.java | 66 ++++++++++++++++++++++++ java/src/json/ext/GeneratorState.java | 39 ++++++++++++++ lib/json/common.rb | 19 +++++++ lib/json/ext/generator/state.rb | 5 ++ lib/json/truffle_ruby/generator.rb | 37 +++++++++++++- test/json/json_generator_test.rb | 38 ++++++++++++++ 10 files changed, 289 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 670b2d3ef..779dd3ef7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,17 @@ ### Unreleased +* Add new `allow_duplicate_key` generator options. By default a warning is now emitted when a duplicated key is encountered. + In `json 3.0` an error will be raised. + ```ruby + >> Warning[:deprecated] = true + >> puts JSON.generate({ foo: 1, "foo" => 2 }) + (irb):2: warning: detected duplicate key "foo" in {foo: 1, "foo" => 2}. + This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` + {"foo":1,"foo":2} + >> JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: false) + detected duplicate key "foo" in {foo: 1, "foo" => 2} (JSON::GeneratorError) + ``` * Fix `JSON.generate` `strict: true` mode to also restrict hash keys. * Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index dc40dec7f..1f6461b0b 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -24,6 +24,14 @@ typedef unsigned char _Bool; #endif #endif +#ifndef NOINLINE +#if defined(__has_attribute) && __has_attribute(noinline) +#define NOINLINE() __attribute__((noinline)) +#else +#define NOINLINE() +#endif +#endif + #ifndef RB_UNLIKELY #define RB_UNLIKELY(expr) expr #endif diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 52dcd24f0..d810927c5 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -9,6 +9,12 @@ /* ruby api and some helpers */ +enum duplicate_key_action { + JSON_DEPRECATED = 0, + JSON_IGNORE, + JSON_RAISE, +}; + typedef struct JSON_Generator_StateStruct { VALUE indent; VALUE space; @@ -21,6 +27,8 @@ typedef struct JSON_Generator_StateStruct { long depth; long buffer_initial_length; + enum duplicate_key_action on_duplicate_key; + bool allow_nan; bool ascii_only; bool script_safe; @@ -34,7 +42,7 @@ typedef struct JSON_Generator_StateStruct { static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8; static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode; -static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, +static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json; @@ -987,8 +995,11 @@ static inline VALUE vstate_get(struct generate_json_data *data) } struct hash_foreach_arg { + VALUE hash; struct generate_json_data *data; - int iter; + int first_key_type; + bool first; + bool mixed_keys_encountered; }; static VALUE @@ -1006,6 +1017,22 @@ convert_string_subclass(VALUE key) return key_to_s; } +NOINLINE() +static void +json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) +{ + if (arg->mixed_keys_encountered) { + return; + } + arg->mixed_keys_encountered = true; + + JSON_Generator_State *state = arg->data->state; + if (state->on_duplicate_key != JSON_IGNORE) { + VALUE do_raise = state->on_duplicate_key == JSON_RAISE ? Qtrue : Qfalse; + rb_funcall(mJSON, rb_intern("on_mixed_keys_hash"), 2, arg->hash, do_raise); + } +} + static int json_object_i(VALUE key, VALUE val, VALUE _arg) { @@ -1016,8 +1043,16 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) JSON_Generator_State *state = data->state; long depth = state->depth; + int key_type = rb_type(key); + + if (arg->first) { + arg->first = false; + arg->first_key_type = key_type; + } + else { + fbuffer_append_char(buffer, ','); + } - if (arg->iter > 0) fbuffer_append_char(buffer, ','); if (RB_UNLIKELY(data->state->object_nl)) { fbuffer_append_str(buffer, data->state->object_nl); } @@ -1029,8 +1064,12 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) bool as_json_called = false; start: - switch (rb_type(key)) { + switch (key_type) { case T_STRING: + if (RB_UNLIKELY(arg->first_key_type != T_STRING)) { + json_inspect_hash_with_mixed_keys(arg); + } + if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) { key_to_s = key; } else { @@ -1038,12 +1077,17 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) } break; case T_SYMBOL: + if (RB_UNLIKELY(arg->first_key_type != T_SYMBOL)) { + json_inspect_hash_with_mixed_keys(arg); + } + key_to_s = rb_sym2str(key); break; default: if (data->state->strict) { if (RTEST(data->state->as_json) && !as_json_called) { key = rb_proc_call_with_block(data->state->as_json, 1, &key, Qnil); + key_type = rb_type(key); as_json_called = true; goto start; } else { @@ -1064,7 +1108,6 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, data->state->space); generate_json(buffer, data, val); - arg->iter++; return ST_CONTINUE; } @@ -1091,8 +1134,9 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat fbuffer_append_char(buffer, '{'); struct hash_foreach_arg arg = { + .hash = obj, .data = data, - .iter = 0, + .first = true, }; rb_hash_foreach(obj, json_object_i, (VALUE)&arg); @@ -1794,6 +1838,19 @@ static VALUE cState_ascii_only_set(VALUE self, VALUE enable) return Qnil; } +static VALUE cState_allow_duplicate_key_p(VALUE self) +{ + GET_STATE(self); + switch (state->on_duplicate_key) { + case JSON_IGNORE: + return Qtrue; + case JSON_DEPRECATED: + return Qnil; + case JSON_RAISE: + return Qfalse; + } +} + /* * call-seq: depth * @@ -1883,6 +1940,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_script_safe) { state->script_safe = RTEST(val); } else if (key == sym_escape_slash) { state->script_safe = RTEST(val); } else if (key == sym_strict) { state->strict = RTEST(val); } + else if (key == sym_allow_duplicate_key) { state->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_as_json) { VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; state_write_value(data, &state->as_json, proc); @@ -2008,6 +2066,8 @@ void Init_generator(void) rb_define_method(cState, "generate", cState_generate, -1); rb_define_alias(cState, "generate_new", "generate"); // :nodoc: + rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); + rb_define_singleton_method(cState, "generate", cState_m_generate, 3); VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods"); @@ -2072,6 +2132,7 @@ void Init_generator(void) sym_escape_slash = ID2SYM(rb_intern("escape_slash")); sym_strict = ID2SYM(rb_intern("strict")); sym_as_json = ID2SYM(rb_intern("as_json")); + sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); usascii_encindex = rb_usascii_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1e6ee753f..00b71eb61 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1314,7 +1314,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } - else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } + else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 31f175cd7..94436cc4a 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -488,6 +488,66 @@ void generate(ThreadContext context, final Session session, RubyHash object, fin } } + private static class HashKeyTracker { + enum KeyType { + UNKNOWN, + STRING, + SYMBOL, + }; + + private KeyType keyType; + private boolean done; + private RubyHash hash; + + private HashKeyTracker(RubyHash hash) { + this.hash = hash; + this.done = false; + this.keyType = KeyType.UNKNOWN; + } + + public void trackFirst(ThreadContext context, Session session, IRubyObject key) { + if (key instanceof RubyString) { + this.keyType = KeyType.STRING; + } else if (key.getType() == context.runtime.getSymbol()) { + this.keyType = KeyType.SYMBOL; + } else { + this.done = true; + report(context, session); + } + } + + public void track(ThreadContext context, Session session, IRubyObject key) { + if (!done) { + if (keyType == KeyType.STRING) { + if (!(key instanceof RubyString)) { + this.report(context, session); + } + } else { + if (!(key.getType() == context.runtime.getSymbol())) { + this.report(context, session); + } + } + } + } + + private void report(ThreadContext context, Session session) { + this.done = true; + + final RuntimeInfo info = session.getInfo(context); + final GeneratorState state = session.getState(context); + + if (!state.getAllowDuplicateKey()) { + if (state.getDeprecateDuplicateKey()) { + IRubyObject args[] = new IRubyObject[]{hash, context.getRuntime().getFalse()}; + info.jsonModule.get().callMethod(context, "on_mixed_keys_hash", args); + } else { + IRubyObject args[] = new IRubyObject[]{hash, context.getRuntime().getTrue()}; + info.jsonModule.get().callMethod(context, "on_mixed_keys_hash", args); + } + } + } + } + static void generateHash(ThreadContext context, Session session, RubyHash object, OutputStream buffer) throws IOException { final GeneratorState state = session.getState(context); final int depth = state.increaseDepth(context); @@ -508,7 +568,13 @@ static void generateHash(ThreadContext context, Session session, RubyHash object buffer.write(objectNLBytes); boolean firstPair = true; + HashKeyTracker tracker = new HashKeyTracker(object); for (RubyHash.RubyHashEntry entry : (Set) object.directEntrySet()) { + if (firstPair) { + tracker.trackFirst(context, session, (IRubyObject)entry.getKey()); + } else { + tracker.track(context, session, (IRubyObject)entry.getKey()); + } processEntry(context, session, buffer, entry, firstPair, objectNl, indent, spaceBefore, space); firstPair = false; } diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 142d388dc..f9e8234e6 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -34,6 +34,9 @@ * @author mernen */ public class GeneratorState extends RubyObject { + private boolean allowDuplicateKey = false; + private boolean deprecateDuplicateKey = true; + /** * The indenting unit string. Will be repeated several times for larger * indenting levels. @@ -216,6 +219,10 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { this.strict = orig.strict; this.bufferInitialLength = orig.bufferInitialLength; this.depth = orig.depth; + + this.allowDuplicateKey = orig.allowDuplicateKey; + this.deprecateDuplicateKey = orig.deprecateDuplicateKey; + return this; } @@ -479,6 +486,24 @@ private ByteList prepareByteList(ThreadContext context, IRubyObject value) { return str.getByteList().dup(); } + @JRubyMethod(name="allow_duplicate_key?", visibility=Visibility.PRIVATE) + public IRubyObject allow_duplicate_key_p(ThreadContext context) { + if (allowDuplicateKey) { + return context.runtime.getTrue(); + } else if (deprecateDuplicateKey) { + return context.runtime.getNil(); + } + return context.runtime.getFalse(); + } + + public boolean getAllowDuplicateKey() { + return allowDuplicateKey; + } + + public boolean getDeprecateDuplicateKey() { + return deprecateDuplicateKey; + } + /** * State#configure(opts) * @@ -520,6 +545,10 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { depth = opts.getInt("depth", 0); + if (opts.hasKey("allow_duplicate_key")) { + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.deprecateDuplicateKey = false; + } return this; } @@ -548,6 +577,16 @@ public RubyHash to_h(ThreadContext context) { result.op_aset(context, runtime.newSymbol("strict"), strict_get(context)); result.op_aset(context, runtime.newSymbol("depth"), depth_get(context)); result.op_aset(context, runtime.newSymbol("buffer_initial_length"), buffer_initial_length_get(context)); + + if (this.allowDuplicateKey) { + if (!this.deprecateDuplicateKey) { + result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), runtime.getTrue()); + } + } + else { + result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), runtime.getFalse()); + } + for (String name: getInstanceVariableNameList()) { result.op_aset(context, runtime.newSymbol(name.substring(1)), getInstanceVariables().getInstanceVariable(name)); } diff --git a/lib/json/common.rb b/lib/json/common.rb index e99d152a8..7222a9d2d 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -186,6 +186,25 @@ def generator=(generator) # :nodoc: private + # Called from the extension when a hash has both string and symbol keys + def on_mixed_keys_hash(hash, do_raise) + set = {} + hash.each_key do |key| + key_str = key.to_s + + if set[key_str] + message = "detected duplicate key #{key_str.inspect} in #{hash.inspect}" + if do_raise + raise GeneratorError, message + else + deprecation_warning("#{message}.\nThis will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`") + end + else + set[key_str] = true + end + end + end + def deprecated_singleton_attr_accessor(*attrs) args = RUBY_VERSION >= "3.0" ? ", category: :deprecated" : "" attrs.each do |attr| diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index d40c3b5ec..d2af1631a 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -68,6 +68,11 @@ def to_h buffer_initial_length: buffer_initial_length, } + allow_duplicate_key = allow_duplicate_key? + unless allow_duplicate_key.nil? + result[:allow_duplicate_key] = allow_duplicate_key + end + instance_variables.each do |iv| iv = iv.to_s[1..-1] result[iv.to_sym] = self[iv] diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 6facf0364..8be312f7a 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -271,6 +271,12 @@ def configure(opts) false end + if opts.key?(:allow_duplicate_key) + @allow_duplicate_key = !!opts[:allow_duplicate_key] + else + @allow_duplicate_key = nil # nil is deprecation + end + @strict = !!opts[:strict] if opts.key?(:strict) if !opts.key?(:max_nesting) # defaults to 100 @@ -284,6 +290,10 @@ def configure(opts) end alias merge configure + def allow_duplicate_key? # :nodoc: + @allow_duplicate_key + end + # Returns the configuration instance variables as a hash, that can be # passed to the configure method. def to_h @@ -292,6 +302,11 @@ def to_h iv = iv.to_s[1..-1] result[iv.to_sym] = self[iv] end + + if result[:allow_duplicate_key].nil? + result.delete(:allow_duplicate_key) + end + result end @@ -330,8 +345,17 @@ def generate_new(obj, anIO = nil) # :nodoc: when Hash buf << '{' first = true + key_type = nil obj.each_pair do |k,v| - buf << ',' unless first + if first + key_type = k.class + else + if key_type && !@allow_duplicate_key && key_type != k.class + key_type = nil # stop checking + JSON.send(:on_mixed_keys_hash, obj, !@allow_duplicate_key.nil?) + end + buf << ',' + end key_str = k.to_s if key_str.class == String @@ -471,9 +495,18 @@ def json_transform(state) delim = ",#{state.object_nl}" result = +"{#{state.object_nl}" first = true + key_type = nil indent = !state.object_nl.empty? each { |key, value| - result << delim unless first + if first + key_type = key.class + else + if key_type && !state.allow_duplicate_key? && key_type != key.class + key_type = nil # stop checking + JSON.send(:on_mixed_keys_hash, self, state.allow_duplicate_key? == false) + end + result << delim + end result << state.indent * depth if indent if state.strict? && !(Symbol === key || String === key) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 963350ea4..4315d109d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -234,6 +234,24 @@ def test_state_defaults :space => "", :space_before => "", }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) + + state = JSON::State.new(allow_duplicate_key: true) + assert_equal({ + :allow_duplicate_key => true, + :allow_nan => false, + :array_nl => "", + :as_json => false, + :ascii_only => false, + :buffer_initial_length => 1024, + :depth => 0, + :script_safe => false, + :strict => false, + :indent => "", + :max_nesting => 100, + :object_nl => "", + :space => "", + :space_before => "", + }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end def test_allow_nan @@ -828,4 +846,24 @@ def test_numbers_of_various_sizes assert_equal "[#{number}]", JSON.generate([number]) end end + + def test_generate_duplicate_keys_allowed + hash = { foo: 1, "foo" => 2 } + assert_equal %({"foo":1,"foo":2}), JSON.generate(hash, allow_duplicate_key: true) + end + + def test_generate_duplicate_keys_deprecated + hash = { foo: 1, "foo" => 2 } + assert_deprecated_warning(/allow_duplicate_key/) do + assert_equal %({"foo":1,"foo":2}), JSON.generate(hash) + end + end + + def test_generate_duplicate_keys_disallowed + hash = { foo: 1, "foo" => 2 } + error = assert_raise JSON::GeneratorError do + JSON.generate(hash, allow_duplicate_key: false) + end + assert_equal %(detected duplicate key "foo" in #{hash.inspect}), error.message + end end From f0dda861c586f36ae523ee0293d0a7c0acadc328 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 24 Aug 2025 17:39:10 +0200 Subject: [PATCH 121/506] Optimize `fbuffer_append_str_repeat` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helps with pretty printting performance: ``` == Encoding activitypub.json (52595 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 1.746k i/100ms Calculating ------------------------------------- after 17.481k (± 1.0%) i/s (57.20 μs/i) - 89.046k in 5.094341s Comparison: before: 16038.4 i/s after: 17481.1 i/s - 1.09x faster == Encoding citm_catalog.json (500298 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 60.000 i/100ms Calculating ------------------------------------- after 608.157 (± 2.3%) i/s (1.64 ms/i) - 3.060k in 5.034238s Comparison: before: 525.3 i/s after: 608.2 i/s - 1.16x faster == Encoding twitter.json (466906 bytes) ruby 3.4.2 (2025-02-15 revision d2930f8e7a) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 160.000 i/100ms Calculating ------------------------------------- after 1.606k (± 0.5%) i/s (622.70 μs/i) - 8.160k in 5.081406s Comparison: before: 1410.3 i/s after: 1605.9 i/s - 1.14x faster ``` --- ext/json/ext/fbuffer/fbuffer.h | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 1f6461b0b..d5fd8ac6d 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -177,12 +177,17 @@ static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested) } } -static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len) +static inline void fbuffer_append_reserved(FBuffer *fb, const char *newstr, unsigned long len) +{ + MEMCPY(fb->ptr + fb->len, newstr, char, len); + fbuffer_consumed(fb, len); +} + +static inline void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len) { if (len > 0) { fbuffer_inc_capa(fb, len); - MEMCPY(fb->ptr + fb->len, newstr, char, len); - fbuffer_consumed(fb, len); + fbuffer_append_reserved(fb, newstr, len); } } @@ -210,13 +215,15 @@ static void fbuffer_append_str(FBuffer *fb, VALUE str) static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) { + const char *newstr = StringValuePtr(str); unsigned long len = RSTRING_LEN(str); - size_t total = repeat * len; - fbuffer_inc_capa(fb, total); - + fbuffer_inc_capa(fb, repeat * len); while (repeat) { - fbuffer_append_str(fb, str); +#ifdef JSON_DEBUG + fb->requested = len; +#endif + fbuffer_append_reserved(fb, newstr, len); repeat--; } } From 19bcfdd8d8f50caf09605623fd00ee3bd8848019 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 25 Aug 2025 09:21:05 +0200 Subject: [PATCH 122/506] Remove reference to fast_generate --- lib/json/common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 7222a9d2d..45200a83b 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -410,7 +410,7 @@ def load_file!(filespec, opts = nil) # # Returns a \String containing the generated \JSON data. # - # See also JSON.fast_generate, JSON.pretty_generate. + # See also JSON.pretty_generate. # # Argument +obj+ is the Ruby object to be converted to \JSON. # From 3187c88c065e2fb28444d2e86680106bc9962365 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 25 Aug 2025 09:39:47 +0200 Subject: [PATCH 123/506] Improve generation options documentation --- .gitignore | 2 +- lib/json.rb | 22 ++++++++++++++++++++++ lib/json/ext/generator/state.rb | 16 ++-------------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 8ae6ac119..bb35f362a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,4 @@ Gemfile.lock .byebug_history *.log /tmp - +/doc/ diff --git a/lib/json.rb b/lib/json.rb index 0ebff2f94..43d96afa9 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -307,6 +307,25 @@ # # --- # +# Option +allow_duplicate_key+ (boolean) specifies whether +# hashes with duplicate keys should be allowed or produce an error. +# defaults to emit a deprecation warning. +# +# With the default, (not set): +# Warning[:deprecated] = true +# JSON.generate({ foo: 1, "foo" => 2 }) +# # warning: detected duplicate key "foo" in {foo: 1, "foo" => 2}. +# # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` +# # => '{"foo":1,"foo":2}' +# +# With false +# JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: false) +# # detected duplicate key "foo" in {foo: 1, "foo" => 2} (JSON::GeneratorError) +# +# In version 3.0, false will become the default. +# +# --- +# # Option +max_nesting+ (\Integer) specifies the maximum nesting depth # in +obj+; defaults to +100+. # @@ -384,6 +403,9 @@ # # == \JSON Additions # +# Note that JSON Additions must only be used with trusted data, and is +# deprecated. +# # When you "round trip" a non-\String object from Ruby to \JSON and back, # you have a new \String, instead of the object you began with: # ruby0 = Range.new(0, 2) diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index d2af1631a..1f56e6c68 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -8,20 +8,8 @@ class State # # Instantiates a new State object, configured by _opts_. # - # _opts_ can have the following keys: - # - # * *indent*: a string used to indent levels (default: ''), - # * *space*: a string that is put after, a : or , delimiter (default: ''), - # * *space_before*: a string that is put before a : pair delimiter (default: ''), - # * *object_nl*: a string that is put at the end of a JSON object (default: ''), - # * *array_nl*: a string that is put at the end of a JSON array (default: ''), - # * *allow_nan*: true if NaN, Infinity, and -Infinity should be - # generated, otherwise an exception is thrown, if these values are - # encountered. This options defaults to false. - # * *ascii_only*: true if only ASCII characters should be generated. This - # option defaults to false. - # * *buffer_initial_length*: sets the initial length of the generator's - # internal buffer. + # Argument +opts+, if given, contains a \Hash of options for the generation. + # See {Generating Options}[#module-JSON-label-Generating+Options]. def initialize(opts = nil) if opts && !opts.empty? configure(opts) From d3f7f0452bf3f18723ab2c7b2d5000a1991fc37c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 27 Aug 2025 20:43:20 +0200 Subject: [PATCH 124/506] Fix a -Wreturn-type warning Fix: https://github.com/ruby/json/pull/843 Co-Authored-By: Takashi Kokubun --- ext/json/ext/generator/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d810927c5..8fcc980d4 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1846,7 +1846,7 @@ static VALUE cState_allow_duplicate_key_p(VALUE self) return Qtrue; case JSON_DEPRECATED: return Qnil; - case JSON_RAISE: + default: return Qfalse; } } From 5274d5d19f8ae20e1e31c4021a74c640badd3123 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Mon, 11 Aug 2025 22:02:12 -0500 Subject: [PATCH 125/506] Allow for segmented output streams and a SWAR-based basic StringEncoder implementation. --- Rakefile | 2 +- .../AbstractByteListDirectOutputStream.java | 31 +++++ .../json/ext/ByteListDirectOutputStream.java | 3 +- java/src/json/ext/Generator.java | 4 +- ...edSegmentedByteListDirectOutputStream.java | 106 +++++++++++++++ .../SegmentedByteListDirectOutputStream.java | 79 ++++++++++++ java/src/json/ext/StringEncoder.java | 121 +++++++++++++++++- 7 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 java/src/json/ext/AbstractByteListDirectOutputStream.java create mode 100644 java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java create mode 100644 java/src/json/ext/SegmentedByteListDirectOutputStream.java diff --git a/Rakefile b/Rakefile index 52e178ed1..616e7e98e 100644 --- a/Rakefile +++ b/Rakefile @@ -116,7 +116,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' file JRUBY_GENERATOR_JAR => :compile do cd 'java/src' do generator_classes = FileList[ - "json/ext/ByteList*.class", + "json/ext/*ByteList*.class", "json/ext/OptionsReader*.class", "json/ext/Generator*.class", "json/ext/RuntimeInfo*.class", diff --git a/java/src/json/ext/AbstractByteListDirectOutputStream.java b/java/src/json/ext/AbstractByteListDirectOutputStream.java new file mode 100644 index 000000000..64ca29d35 --- /dev/null +++ b/java/src/json/ext/AbstractByteListDirectOutputStream.java @@ -0,0 +1,31 @@ +package json.ext; + +import java.io.OutputStream; + +import org.jcodings.Encoding; +import org.jruby.util.ByteList; + +abstract class AbstractByteListDirectOutputStream extends OutputStream { + + private static final String PROP_SEGMENTED_BUFFER = "json.useSegmentedOutputStream"; + private static final String PROP_SEGMENTED_BUFFER_DEFAULT = "true"; + + private static final boolean USE_SEGMENTED_BUFFER; + + static { + String useSegmentedOutputStream = System.getProperty(PROP_SEGMENTED_BUFFER, PROP_SEGMENTED_BUFFER_DEFAULT); + USE_SEGMENTED_BUFFER = Boolean.parseBoolean(useSegmentedOutputStream); + // XXX Is there a logger we can use here? + // System.out.println("Using segmented output stream: " + USE_SEGMENTED_BUFFER); + } + + public static AbstractByteListDirectOutputStream create(int estimatedSize) { + if (USE_SEGMENTED_BUFFER) { + return new SegmentedByteListDirectOutputStream(estimatedSize); + } else { + return new ByteListDirectOutputStream(estimatedSize); + } + } + + public abstract ByteList toByteListDirect(Encoding encoding); +} diff --git a/java/src/json/ext/ByteListDirectOutputStream.java b/java/src/json/ext/ByteListDirectOutputStream.java index b22d48128..a92753ca3 100644 --- a/java/src/json/ext/ByteListDirectOutputStream.java +++ b/java/src/json/ext/ByteListDirectOutputStream.java @@ -4,10 +4,9 @@ import org.jruby.util.ByteList; import java.io.IOException; -import java.io.OutputStream; import java.util.Arrays; -public class ByteListDirectOutputStream extends OutputStream { +public class ByteListDirectOutputStream extends AbstractByteListDirectOutputStream { private byte[] buffer; private int length; diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 94436cc4a..5f5177bea 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -26,6 +26,8 @@ import org.jruby.util.IOOutputStream; import org.jruby.util.TypeConverter; +import json.ext.ByteListDirectOutputStream; + import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -252,7 +254,7 @@ int guessSize(ThreadContext context, Session session, T object) { } RubyString generateNew(ThreadContext context, Session session, T object) { - ByteListDirectOutputStream buffer = new ByteListDirectOutputStream(guessSize(context, session, object)); + AbstractByteListDirectOutputStream buffer = AbstractByteListDirectOutputStream.create(guessSize(context, session, object)); generateToBuffer(context, session, object, buffer); return RubyString.newString(context.runtime, buffer.toByteListDirect(UTF8Encoding.INSTANCE)); } diff --git a/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java b/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java new file mode 100644 index 000000000..83a1d89a3 --- /dev/null +++ b/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java @@ -0,0 +1,106 @@ +package json.ext; + +import org.jcodings.Encoding; +import org.jruby.util.ByteList; + +import java.io.IOException; + +public class LinkedSegmentedByteListDirectOutputStream extends AbstractByteListDirectOutputStream { + private Segment head; + private int length; + private Segment current; + private int numSegments; + + private static class Segment { + static final int DEFAULT_SEGMENT_SIZE = 1024; + byte[] buffer; + int length; + Segment next; + + Segment() { + this(DEFAULT_SEGMENT_SIZE); + } + + Segment(int size) { + if (size <= 0) { + size = DEFAULT_SEGMENT_SIZE; + } + buffer = new byte[Math.max(size, DEFAULT_SEGMENT_SIZE)]; + } + } + + LinkedSegmentedByteListDirectOutputStream() { + this(Segment.DEFAULT_SEGMENT_SIZE); + } + + LinkedSegmentedByteListDirectOutputStream(int size) { + current = head = new Segment(size); + } + + public ByteList toByteListDirect(Encoding encoding) { + byte[] buffer = new byte[length]; + Segment segment = head; + int pos = 0; + while (segment != null) { + System.arraycopy(segment.buffer, 0, buffer, pos, segment.length); + pos += segment.length; + segment = segment.next; + } + return new ByteList(buffer, 0, length, encoding, false); + } + + @Override + public void write(int b) throws IOException { + Segment c = current; + if (c.length == c.buffer.length) { + // This check is deliberately in the case the current segment is full. We want to + // avoid this check in the common case where we have space in the current segment. + if (this.length + 1 < 0) { + throw new IOException("Total length exceeds maximum length of an array."); + } + if (c.next == null) { + numSegments++; + c.next = new Segment(c.buffer.length * 2); + } + c = c.next; + current = c; + } + c.buffer[c.length++] = (byte)b; + length++; + } + + @Override + public void write(byte[] bytes, int start, int length) throws IOException { + Segment c = current; + int remaining = length; + + while (remaining > 0) { + if (c.length == c.buffer.length) { + // This check is deliberately in the case the current segment is full. We want to + // avoid this check in the common case where we have space in the current segment. + if (this.length + remaining < 0) { + throw new IOException("Total length exceeds maximum length of an array."); + } + if (c.next == null) { + numSegments++; + c.next = new Segment(c.buffer.length * 2); + } + c = c.next; + current = c; + } + int currentLength = c.length; + int currentCapacity = c.buffer.length; + int copyLength = Math.min(remaining, currentCapacity - currentLength); + System.arraycopy(bytes, start, c.buffer, currentLength, copyLength); + c.length += copyLength; + this.length += copyLength; + start += copyLength; + remaining -= copyLength; + } + } + + @Override + public void write(byte[] bytes) throws IOException { + write(bytes, 0, bytes.length); + } +} diff --git a/java/src/json/ext/SegmentedByteListDirectOutputStream.java b/java/src/json/ext/SegmentedByteListDirectOutputStream.java new file mode 100644 index 000000000..86d38cd15 --- /dev/null +++ b/java/src/json/ext/SegmentedByteListDirectOutputStream.java @@ -0,0 +1,79 @@ +package json.ext; + +import org.jcodings.Encoding; +import org.jruby.util.ByteList; + +import java.io.IOException; + +public class SegmentedByteListDirectOutputStream extends AbstractByteListDirectOutputStream { + private static final int DEFAULT_CAPACITY = 1024; + + private int totalLength; + private byte[][] segments = new byte[21][]; + private int currentSegmentIndex; + private int currentSegmentLength; + private byte[] currentSegment; + + SegmentedByteListDirectOutputStream(int size) { + currentSegment = new byte[Math.max(size, DEFAULT_CAPACITY)]; + segments[0] = currentSegment; + } + + public ByteList toByteListDirect(Encoding encoding) { + byte[] buffer = new byte[totalLength]; + int pos = 0; + // We handle the current segment separately. + for (int i = 0; i < currentSegmentIndex; i++) { + byte[] segment = segments[i]; + System.arraycopy(segment, 0, buffer, pos, segment.length); + pos += segment.length; + } + System.arraycopy(currentSegment, 0, buffer, pos, currentSegmentLength); + return new ByteList(buffer, 0, totalLength, encoding, false); + } + + @Override + public void write(int b) throws IOException { + if (currentSegmentLength == currentSegment.length) { + if (totalLength + 1 < 0) { + throw new IOException("Total length exceeds maximum length of an array."); + } + currentSegmentIndex++; + int capacity = currentSegment.length * 2; + currentSegment = new byte[capacity]; + currentSegmentLength = 0; + segments[currentSegmentIndex] = currentSegment; + } + currentSegment[currentSegmentLength++] = (byte) b; + totalLength++; + } + + @Override + public void write(byte[] bytes, int start, int length) throws IOException { + int remaining = length; + + while (remaining > 0) { + if (currentSegmentLength == currentSegment.length) { + if (totalLength + remaining < 0) { + throw new IOException("Total length exceeds maximum length of an array."); + } + currentSegmentIndex++; + int capacity = currentSegment.length * 2; + currentSegment = new byte[capacity]; + currentSegmentLength = 0; + segments[currentSegmentIndex] = currentSegment; + } + int toWrite = Math.min(remaining, currentSegment.length - currentSegmentLength); + System.arraycopy(bytes, start, currentSegment, currentSegmentLength, toWrite); + currentSegmentLength += toWrite; + start += toWrite; + remaining -= toWrite; + } + totalLength += length; + } + + @Override + public void write(byte[] bytes) throws IOException { + write(bytes, 0, bytes.length); + } +} diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index d178d0bd8..6783be9de 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -5,6 +5,11 @@ */ package json.ext; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + import org.jcodings.Encoding; import org.jcodings.specific.ASCIIEncoding; import org.jcodings.specific.USASCIIEncoding; @@ -17,10 +22,6 @@ import org.jruby.util.ByteList; import org.jruby.util.StringSupport; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - /** * An encoder that reads from the given source and outputs its representation * to another ByteList. The source string is fully checked for UTF-8 validity, @@ -114,6 +115,17 @@ class StringEncoder extends ByteListTranscoder { protected final byte[] escapeTable; + private static final String USE_SWAR_BASIC_ENCODER_PROP = "json.useSWARBasicEncoder"; + private static final String USE_SWAR_BASIC_ENCODER_DEFAULT = "true"; + private static final boolean USE_BASIC_SWAR_ENCODER; + + static { + USE_BASIC_SWAR_ENCODER = Boolean.parseBoolean( + System.getProperty(USE_SWAR_BASIC_ENCODER_PROP, USE_SWAR_BASIC_ENCODER_DEFAULT)); + // XXX Is there a logger we can use here? + // System.out.println("Using SWAR basic encoder: " + USE_BASIC_SWAR_ENCODER); + } + OutputStream out; // Escaped characters will reuse this array, to avoid new allocations @@ -198,8 +210,109 @@ private static RubyString tryWeirdEncodings(ThreadContext context, RubyString st return str; } + void encodeBasicSWAR(ByteList src) throws IOException { + byte[] hexdig = HEX; + byte[] scratch = aux; + + byte[] ptrBytes = src.unsafeBytes(); + int ptr = src.begin(); + int len = src.realSize(); + + int beg = 0; + int pos = 0; + + ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, len); + while (pos + 8 <= len) { + long x = bb.getLong(ptr + pos); + long is_ascii = 0x8080808080808080L & ~x; + long xor2 = x ^ 0x0202020202020202L; + long lt32_or_eq34 = xor2 - 0x2121212121212121L; + long sub92 = x ^ 0x5C5C5C5C5C5C5C5CL; + long eq92 = (sub92 - 0x0101010101010101L); + boolean needs_escape = ((lt32_or_eq34 | eq92) & is_ascii) != 0; + if (needs_escape) { + // Find the exact byte that needs escaping + for (int i = 0; i < 8; i++) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos + i]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos + i, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + break; + } + } + continue; + } + + pos += 8; + } + + // Handle remaining bytes one by one + while (pos < len) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + } else { + pos++; + } + } + + while (pos < len) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + } else { + pos++; + } + } + + if (beg < len) { + append(ptrBytes, ptr + beg, len - beg); + } + } + + void encodeBasic(ByteList src) throws IOException{ + byte[] hexdig = HEX; + byte[] scratch = aux; + + byte[] ptrBytes = src.unsafeBytes(); + int ptr = src.begin(); + int len = src.realSize(); + + int beg = 0; + int pos = 0; + + while (pos < len) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + } else { + pos++; + } + } + + if (beg < len) { + append(ptrBytes, ptr + beg, len - beg); + } + } + // C: convert_UTF8_to_JSON void encode(ByteList src) throws IOException { + if (escapeTable == ESCAPE_TABLE) { + if (USE_BASIC_SWAR_ENCODER) { + encodeBasicSWAR(src); + } else { + encodeBasic(src); + } + return; + } + byte[] hexdig = HEX; byte[] scratch = aux; byte[] escapeTable = this.escapeTable; From b2644fff8aff8986ba324317000609412b9578e5 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 13 Aug 2025 08:09:48 -0500 Subject: [PATCH 126/506] Handle the case were the capacity overflows Integer.MAX_VALUE. --- ...inkedSegmentedByteListDirectOutputStream.java | 16 ++++++++++------ .../ext/SegmentedByteListDirectOutputStream.java | 6 ++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java b/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java index 83a1d89a3..125bb11ef 100644 --- a/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java +++ b/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java @@ -58,10 +58,12 @@ public void write(int b) throws IOException { if (this.length + 1 < 0) { throw new IOException("Total length exceeds maximum length of an array."); } - if (c.next == null) { - numSegments++; - c.next = new Segment(c.buffer.length * 2); + numSegments++; + int capacity = c.buffer.length * 2; + if (capacity < 0) { + capacity = Integer.MAX_VALUE - length; } + c.next = new Segment(capacity); c = c.next; current = c; } @@ -81,10 +83,12 @@ public void write(byte[] bytes, int start, int length) throws IOException { if (this.length + remaining < 0) { throw new IOException("Total length exceeds maximum length of an array."); } - if (c.next == null) { - numSegments++; - c.next = new Segment(c.buffer.length * 2); + numSegments++; + int capacity = c.buffer.length * 2; + if (capacity < 0) { + capacity = Integer.MAX_VALUE - length; } + c.next = new Segment(capacity); c = c.next; current = c; } diff --git a/java/src/json/ext/SegmentedByteListDirectOutputStream.java b/java/src/json/ext/SegmentedByteListDirectOutputStream.java index 86d38cd15..63112fe86 100644 --- a/java/src/json/ext/SegmentedByteListDirectOutputStream.java +++ b/java/src/json/ext/SegmentedByteListDirectOutputStream.java @@ -40,6 +40,9 @@ public void write(int b) throws IOException { } currentSegmentIndex++; int capacity = currentSegment.length * 2; + if (capacity < 0) { + capacity = Integer.MAX_VALUE - totalLength; + } currentSegment = new byte[capacity]; currentSegmentLength = 0; segments[currentSegmentIndex] = currentSegment; @@ -59,6 +62,9 @@ public void write(byte[] bytes, int start, int length) throws IOException { } currentSegmentIndex++; int capacity = currentSegment.length * 2; + if (capacity < 0) { + capacity = Integer.MAX_VALUE - totalLength; + } currentSegment = new byte[capacity]; currentSegmentLength = 0; segments[currentSegmentIndex] = currentSegment; From 32f328728cba8b9b6a969d8c9a6582e5c8ae7d62 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 13 Aug 2025 21:45:42 -0500 Subject: [PATCH 127/506] Remove the LinkedSegmentedByteListDirectOutputStream in favor of the SegmentedByteListDirectOutputStream. --- ...edSegmentedByteListDirectOutputStream.java | 110 ------------------ 1 file changed, 110 deletions(-) delete mode 100644 java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java diff --git a/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java b/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java deleted file mode 100644 index 125bb11ef..000000000 --- a/java/src/json/ext/LinkedSegmentedByteListDirectOutputStream.java +++ /dev/null @@ -1,110 +0,0 @@ -package json.ext; - -import org.jcodings.Encoding; -import org.jruby.util.ByteList; - -import java.io.IOException; - -public class LinkedSegmentedByteListDirectOutputStream extends AbstractByteListDirectOutputStream { - private Segment head; - private int length; - private Segment current; - private int numSegments; - - private static class Segment { - static final int DEFAULT_SEGMENT_SIZE = 1024; - byte[] buffer; - int length; - Segment next; - - Segment() { - this(DEFAULT_SEGMENT_SIZE); - } - - Segment(int size) { - if (size <= 0) { - size = DEFAULT_SEGMENT_SIZE; - } - buffer = new byte[Math.max(size, DEFAULT_SEGMENT_SIZE)]; - } - } - - LinkedSegmentedByteListDirectOutputStream() { - this(Segment.DEFAULT_SEGMENT_SIZE); - } - - LinkedSegmentedByteListDirectOutputStream(int size) { - current = head = new Segment(size); - } - - public ByteList toByteListDirect(Encoding encoding) { - byte[] buffer = new byte[length]; - Segment segment = head; - int pos = 0; - while (segment != null) { - System.arraycopy(segment.buffer, 0, buffer, pos, segment.length); - pos += segment.length; - segment = segment.next; - } - return new ByteList(buffer, 0, length, encoding, false); - } - - @Override - public void write(int b) throws IOException { - Segment c = current; - if (c.length == c.buffer.length) { - // This check is deliberately in the case the current segment is full. We want to - // avoid this check in the common case where we have space in the current segment. - if (this.length + 1 < 0) { - throw new IOException("Total length exceeds maximum length of an array."); - } - numSegments++; - int capacity = c.buffer.length * 2; - if (capacity < 0) { - capacity = Integer.MAX_VALUE - length; - } - c.next = new Segment(capacity); - c = c.next; - current = c; - } - c.buffer[c.length++] = (byte)b; - length++; - } - - @Override - public void write(byte[] bytes, int start, int length) throws IOException { - Segment c = current; - int remaining = length; - - while (remaining > 0) { - if (c.length == c.buffer.length) { - // This check is deliberately in the case the current segment is full. We want to - // avoid this check in the common case where we have space in the current segment. - if (this.length + remaining < 0) { - throw new IOException("Total length exceeds maximum length of an array."); - } - numSegments++; - int capacity = c.buffer.length * 2; - if (capacity < 0) { - capacity = Integer.MAX_VALUE - length; - } - c.next = new Segment(capacity); - c = c.next; - current = c; - } - int currentLength = c.length; - int currentCapacity = c.buffer.length; - int copyLength = Math.min(remaining, currentCapacity - currentLength); - System.arraycopy(bytes, start, c.buffer, currentLength, copyLength); - c.length += copyLength; - this.length += copyLength; - start += copyLength; - remaining -= copyLength; - } - } - - @Override - public void write(byte[] bytes) throws IOException { - write(bytes, 0, bytes.length); - } -} From 44b1d8774ad4e1433a691d0218ec029f7ea8f8bc Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 13 Aug 2025 22:20:12 -0500 Subject: [PATCH 128/506] Use a ternary to determine the capacity of the next segment when growing the output buffer. --- .../json/ext/SegmentedByteListDirectOutputStream.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/java/src/json/ext/SegmentedByteListDirectOutputStream.java b/java/src/json/ext/SegmentedByteListDirectOutputStream.java index 63112fe86..abad16612 100644 --- a/java/src/json/ext/SegmentedByteListDirectOutputStream.java +++ b/java/src/json/ext/SegmentedByteListDirectOutputStream.java @@ -40,9 +40,7 @@ public void write(int b) throws IOException { } currentSegmentIndex++; int capacity = currentSegment.length * 2; - if (capacity < 0) { - capacity = Integer.MAX_VALUE - totalLength; - } + capacity = (capacity < 0) ? DEFAULT_CAPACITY : capacity; currentSegment = new byte[capacity]; currentSegmentLength = 0; segments[currentSegmentIndex] = currentSegment; @@ -61,10 +59,9 @@ public void write(byte[] bytes, int start, int length) throws IOException { throw new IOException("Total length exceeds maximum length of an array."); } currentSegmentIndex++; - int capacity = currentSegment.length * 2; - if (capacity < 0) { - capacity = Integer.MAX_VALUE - totalLength; - } + int capacity = currentSegment.length << 1; + capacity = (capacity < 0) ? DEFAULT_CAPACITY : capacity; + capacity = (capacity < remaining) ? remaining : capacity; currentSegment = new byte[capacity]; currentSegmentLength = 0; segments[currentSegmentIndex] = currentSegment; From a458201a7e960ada025bab8efaa9e1c96eeb256a Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 14 Aug 2025 23:07:34 -0500 Subject: [PATCH 129/506] Use SWAR if there is still at least 4 bytes remaining. --- java/src/json/ext/StringEncoder.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index 6783be9de..5c19ea8a5 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -247,15 +247,16 @@ void encodeBasicSWAR(ByteList src) throws IOException { pos += 8; } - // Handle remaining bytes one by one - while (pos < len) { - int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); - int ch_len = ESCAPE_TABLE[ch]; - if (ch_len > 0) { - beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); - escapeAscii(ch, scratch, hexdig); - } else { - pos++; + if (pos + 4 <= len) { + int x = bb.getInt(ptr + pos); + int is_ascii = 0x808080 & ~x; + int xor2 = x ^ 0x020202; + int lt32_or_eq34 = xor2 - 0x212121; + int sub92 = x ^ 0x5C5C5C; + int eq92 = (sub92 - 0x010101); + boolean skip_chunk = ((lt32_or_eq34 | eq92) & is_ascii) == 0; + if (skip_chunk) { + pos += 4; } } From 9ebe1051441b937abfe965cd78d09c8f0ed5593f Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 15 Aug 2025 09:23:21 -0500 Subject: [PATCH 130/506] Ensure the SWAR encoder in the java extension checks every byte. --- java/src/json/ext/StringEncoder.java | 10 +++++----- test/json/json_generator_test.rb | 12 ++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index 5c19ea8a5..cca547b7b 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -249,11 +249,11 @@ void encodeBasicSWAR(ByteList src) throws IOException { if (pos + 4 <= len) { int x = bb.getInt(ptr + pos); - int is_ascii = 0x808080 & ~x; - int xor2 = x ^ 0x020202; - int lt32_or_eq34 = xor2 - 0x212121; - int sub92 = x ^ 0x5C5C5C; - int eq92 = (sub92 - 0x010101); + int is_ascii = 0x80808080 & ~x; + int xor2 = x ^ 0x02020202; + int lt32_or_eq34 = xor2 - 0x21212121; + int sub92 = x ^ 0x5C5C5C5C; + int eq92 = (sub92 - 0x01010101); boolean skip_chunk = ((lt32_or_eq34 | eq92) & is_ascii) == 0; if (skip_chunk) { pos += 4; diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 4315d109d..6b42de2ad 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -504,6 +504,18 @@ def test_backslash json = '["\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""]' assert_equal json, generate(data) # + data = '"""""' + json = '"\"\"\"\"\""' + assert_equal json, generate(data) + # + data = "abc\n" + json = '"abc\\n"' + assert_equal json, generate(data) + # + data = "\nabc" + json = '"\\nabc"' + assert_equal json, generate(data) + # data = ["'"] json = '["\\\'"]' assert_equal '["\'"]', generate(data) From 052198a7be9c399d4a416bdf1d2ba78f35b414d6 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Sun, 17 Aug 2025 20:31:56 -0500 Subject: [PATCH 131/506] Refactor the SWAR logic into a separate subclass of StringEncoder. --- Rakefile | 2 +- java/src/json/ext/Generator.java | 2 +- java/src/json/ext/SWARBasicStringEncoder.java | 84 +++++++++++++++++++ java/src/json/ext/StringEncoder.java | 83 +++--------------- 4 files changed, 96 insertions(+), 75 deletions(-) create mode 100644 java/src/json/ext/SWARBasicStringEncoder.java diff --git a/Rakefile b/Rakefile index 616e7e98e..57b1d0ec5 100644 --- a/Rakefile +++ b/Rakefile @@ -120,7 +120,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' "json/ext/OptionsReader*.class", "json/ext/Generator*.class", "json/ext/RuntimeInfo*.class", - "json/ext/StringEncoder*.class", + "json/ext/*StringEncoder*.class", "json/ext/Utils*.class" ] sh 'jar', 'cf', File.basename(JRUBY_GENERATOR_JAR), *generator_classes diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 5f5177bea..c8452ade6 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -234,7 +234,7 @@ public StringEncoder getStringEncoder(ThreadContext context) { GeneratorState state = getState(context); stringEncoder = state.asciiOnly() ? new StringEncoderAsciiOnly(state.scriptSafe()) : - new StringEncoder(state.scriptSafe()); + (state.scriptSafe()) ? new StringEncoder(state.scriptSafe()) : StringEncoder.createBasicEncoder(); } return stringEncoder; } diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java new file mode 100644 index 000000000..9d6d4f2b9 --- /dev/null +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -0,0 +1,84 @@ +package json.ext; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.jruby.util.ByteList; + +public class SWARBasicStringEncoder extends StringEncoder { + + public SWARBasicStringEncoder() { + super(ESCAPE_TABLE); + } + + @Override + void encode(ByteList src) throws IOException { + byte[] hexdig = HEX; + byte[] scratch = aux; + + byte[] ptrBytes = src.unsafeBytes(); + int ptr = src.begin(); + int len = src.realSize(); + + int beg = 0; + int pos = 0; + + ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, len); + while (pos + 8 <= len) { + long x = bb.getLong(ptr + pos); + if (skipChunk(x)) { + pos += 8; + continue; + } + for (int i = 0; i < 8; i++) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos + i]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos + i, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + break; + } + } + } + + if (pos + 4 <= len) { + int x = bb.getInt(ptr + pos); + if (skipChunk(x)) { + pos += 4; + } + } + + while (pos < len) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, scratch, hexdig); + } else { + pos++; + } + } + + if (beg < len) { + append(ptrBytes, ptr + beg, len - beg); + } + } + + private boolean skipChunk(long x) { + long is_ascii = 0x8080808080808080L & ~x; + long xor2 = x ^ 0x0202020202020202L; + long lt32_or_eq34 = xor2 - 0x2121212121212121L; + long sub92 = x ^ 0x5C5C5C5C5C5C5C5CL; + long eq92 = (sub92 - 0x0101010101010101L); + return ((lt32_or_eq34 | eq92) & is_ascii) == 0; + } + + private boolean skipChunk(int x) { + int is_ascii = 0x80808080 & ~x; + int xor2 = x ^ 0x02020202; + int lt32_or_eq34 = xor2 - 0x21212121; + int sub92 = x ^ 0x5C5C5C5C; + int eq92 = (sub92 - 0x01010101); + return ((lt32_or_eq34 | eq92) & is_ascii) == 0; + } +} diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index cca547b7b..047b829ff 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import org.jcodings.Encoding; @@ -150,6 +149,14 @@ class StringEncoder extends ByteListTranscoder { this.escapeTable = escapeTable; } + static StringEncoder createBasicEncoder() { + if (USE_BASIC_SWAR_ENCODER) { + return new SWARBasicStringEncoder(); + } else { + return new StringEncoder(false); + } + } + // C: generate_json_string void generate(ThreadContext context, RubyString object, OutputStream buffer) throws IOException { object = ensureValidEncoding(context, object); @@ -210,73 +217,7 @@ private static RubyString tryWeirdEncodings(ThreadContext context, RubyString st return str; } - void encodeBasicSWAR(ByteList src) throws IOException { - byte[] hexdig = HEX; - byte[] scratch = aux; - - byte[] ptrBytes = src.unsafeBytes(); - int ptr = src.begin(); - int len = src.realSize(); - - int beg = 0; - int pos = 0; - - ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, len); - while (pos + 8 <= len) { - long x = bb.getLong(ptr + pos); - long is_ascii = 0x8080808080808080L & ~x; - long xor2 = x ^ 0x0202020202020202L; - long lt32_or_eq34 = xor2 - 0x2121212121212121L; - long sub92 = x ^ 0x5C5C5C5C5C5C5C5CL; - long eq92 = (sub92 - 0x0101010101010101L); - boolean needs_escape = ((lt32_or_eq34 | eq92) & is_ascii) != 0; - if (needs_escape) { - // Find the exact byte that needs escaping - for (int i = 0; i < 8; i++) { - int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos + i]); - int ch_len = ESCAPE_TABLE[ch]; - if (ch_len > 0) { - beg = pos = flushPos(pos + i, beg, ptrBytes, ptr, 1); - escapeAscii(ch, scratch, hexdig); - break; - } - } - continue; - } - - pos += 8; - } - - if (pos + 4 <= len) { - int x = bb.getInt(ptr + pos); - int is_ascii = 0x80808080 & ~x; - int xor2 = x ^ 0x02020202; - int lt32_or_eq34 = xor2 - 0x21212121; - int sub92 = x ^ 0x5C5C5C5C; - int eq92 = (sub92 - 0x01010101); - boolean skip_chunk = ((lt32_or_eq34 | eq92) & is_ascii) == 0; - if (skip_chunk) { - pos += 4; - } - } - - while (pos < len) { - int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); - int ch_len = ESCAPE_TABLE[ch]; - if (ch_len > 0) { - beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); - escapeAscii(ch, scratch, hexdig); - } else { - pos++; - } - } - - if (beg < len) { - append(ptrBytes, ptr + beg, len - beg); - } - } - - void encodeBasic(ByteList src) throws IOException{ + void encodeBasic(ByteList src) throws IOException { byte[] hexdig = HEX; byte[] scratch = aux; @@ -306,11 +247,7 @@ void encodeBasic(ByteList src) throws IOException{ // C: convert_UTF8_to_JSON void encode(ByteList src) throws IOException { if (escapeTable == ESCAPE_TABLE) { - if (USE_BASIC_SWAR_ENCODER) { - encodeBasicSWAR(src); - } else { - encodeBasic(src); - } + encodeBasic(src); return; } From 43a8a8345681586fdc016572cf3f6f38a12977ec Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Sun, 17 Aug 2025 20:37:22 -0500 Subject: [PATCH 132/506] Refactor the logic to evaluate every byte in the chunk if there is a byte in that chunk that needs escaping. --- .../json/ext/AbstractByteListDirectOutputStream.java | 2 +- java/src/json/ext/SWARBasicStringEncoder.java | 10 ++++++---- .../json/ext/SegmentedByteListDirectOutputStream.java | 2 ++ java/src/json/ext/StringEncoder.java | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/java/src/json/ext/AbstractByteListDirectOutputStream.java b/java/src/json/ext/AbstractByteListDirectOutputStream.java index 64ca29d35..c3175c849 100644 --- a/java/src/json/ext/AbstractByteListDirectOutputStream.java +++ b/java/src/json/ext/AbstractByteListDirectOutputStream.java @@ -7,7 +7,7 @@ abstract class AbstractByteListDirectOutputStream extends OutputStream { - private static final String PROP_SEGMENTED_BUFFER = "json.useSegmentedOutputStream"; + private static final String PROP_SEGMENTED_BUFFER = "jruby.json.useSegmentedOutputStream"; private static final String PROP_SEGMENTED_BUFFER_DEFAULT = "true"; private static final boolean USE_SEGMENTED_BUFFER; diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java index 9d6d4f2b9..8021d73b8 100644 --- a/java/src/json/ext/SWARBasicStringEncoder.java +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -30,13 +30,15 @@ void encode(ByteList src) throws IOException { pos += 8; continue; } - for (int i = 0; i < 8; i++) { - int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos + i]); + int chunkEnd = pos + 8; + while (pos < chunkEnd) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); int ch_len = ESCAPE_TABLE[ch]; if (ch_len > 0) { - beg = pos = flushPos(pos + i, beg, ptrBytes, ptr, 1); + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); escapeAscii(ch, scratch, hexdig); - break; + } else { + pos++; } } } diff --git a/java/src/json/ext/SegmentedByteListDirectOutputStream.java b/java/src/json/ext/SegmentedByteListDirectOutputStream.java index abad16612..0e89e50df 100644 --- a/java/src/json/ext/SegmentedByteListDirectOutputStream.java +++ b/java/src/json/ext/SegmentedByteListDirectOutputStream.java @@ -9,6 +9,8 @@ public class SegmentedByteListDirectOutputStream extends AbstractByteListDirectO private static final int DEFAULT_CAPACITY = 1024; private int totalLength; + // Why 21? The minimum segment size is 1024 bytes. If we double the segment size each time + // we need a new segment, we only need 21 segments to reach the maximum array size in Java. private byte[][] segments = new byte[21][]; private int currentSegmentIndex; private int currentSegmentLength; diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index 047b829ff..7f75476d9 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -114,7 +114,7 @@ class StringEncoder extends ByteListTranscoder { protected final byte[] escapeTable; - private static final String USE_SWAR_BASIC_ENCODER_PROP = "json.useSWARBasicEncoder"; + private static final String USE_SWAR_BASIC_ENCODER_PROP = "jruby.json.useSWARBasicEncoder"; private static final String USE_SWAR_BASIC_ENCODER_DEFAULT = "true"; private static final boolean USE_BASIC_SWAR_ENCODER; From cd5f2cdeb8e11abfcf9eccbb83d3641284aa93ad Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 Aug 2025 09:43:12 +0200 Subject: [PATCH 133/506] CI: enable bundler cache --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c2aafaac..acc699779 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,25 +39,25 @@ jobs: - { os: windows-latest, ruby: jruby } - { os: windows-latest, ruby: jruby-head } + env: + BUNDLE_WITHOUT: benchmark + steps: - uses: actions/checkout@v5 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 with: + bundler-cache: true ruby-version: ${{ matrix.ruby }} apt-get: "${{ startsWith(matrix.ruby, 'jruby') && 'ragel' || '' }}" brew: "${{ startsWith(matrix.ruby, 'jruby') && 'ragel' || '' }}" - - run: | - bundle config --without benchmark - bundle install - - - run: rake compile ${{ matrix.env }} + - run: bundle exec rake compile ${{ matrix.env }} - - run: rake test JSON_COMPACT=1 ${{ matrix.env }} + - run: bundle exec rake test JSON_COMPACT=1 ${{ matrix.env }} - - run: rake build + - run: bundle exec rake build - run: gem install pkg/*.gem if: ${{ matrix.ruby != '3.2' }} From 7ce2610f8fd839e9f7c1cae097f12353a23cc72e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 Aug 2025 13:20:13 +0200 Subject: [PATCH 134/506] More bundler caching on CI --- .github/workflows/ci.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acc699779..fcfc7c392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,6 @@ jobs: - { os: windows-latest, ruby: jruby } - { os: windows-latest, ruby: jruby-head } - env: - BUNDLE_WITHOUT: benchmark - steps: - uses: actions/checkout@v5 @@ -74,13 +71,10 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 with: + bundler-cache: true ruby-version: "3.3" apt-get: valgrind - - run: | - bundle config --without benchmark - bundle install - - - run: rake compile + - run: bundle exec rake compile - - run: rake valgrind JSON_COMPACT=1 + - run: bundle exec rake valgrind JSON_COMPACT=1 From 124c48316295321358adab4a1bbe527f2d3ac2bb Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 Aug 2025 13:29:47 +0200 Subject: [PATCH 135/506] Implement automated releases --- .github/workflows/ci.yml | 2 + .github/workflows/push_gem.yml | 85 ++++++++++++++++++++++++++++++++++ Gemfile | 17 ++++--- Rakefile | 63 +++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/push_gem.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcfc7c392..b4070118b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: needs: ruby-versions name: ${{ matrix.os }} ${{ matrix.ruby }} ${{ matrix.env }} runs-on: ${{ matrix.os }} + env: + BUNDLE_WITHOUT: release strategy: fail-fast: false matrix: diff --git a/.github/workflows/push_gem.yml b/.github/workflows/push_gem.yml new file mode 100644 index 000000000..cb4a260e5 --- /dev/null +++ b/.github/workflows/push_gem.yml @@ -0,0 +1,85 @@ +name: Publish + +on: + workflow_run: + workflows: [CI] + types: [completed] + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + outputs: + run_publish: ${{ steps.precheck.outputs.run_publish }} + + steps: + - uses: actions/checkout@v5 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + + - id: precheck + name: Release precheck + run: bundle exec rake ci:check_release + + push: + needs: check + + if: | + needs.check.outputs.run_publish == 'true' && + github.repository == 'ruby/json' && + github.ref_name == github.event.repository.default_branch + runs-on: ubuntu-latest + + environment: + name: rubygems.org + url: https://rubygems.org/gems/json + + permissions: + contents: write + id-token: write + + strategy: + matrix: + ruby: ["ruby", "jruby"] + + steps: + - name: Harden Runner + uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Ruby + uses: ruby/setup-ruby@eaecf785f6a34567a6d97f686bbb7bccc1ac1e5c # v1.237.0 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + + # https://github.com/rubygems/rubygems/issues/5882 + - name: Install dependencies and build for JRuby + run: | + sudo apt install default-jdk maven + gem update --system + gem install ruby-maven rake-compiler --no-document + rake compile + if: matrix.ruby == 'jruby' + + - name: Install dependencies + run: bundle install --jobs 4 --retry 3 + + - name: Publish to RubyGems + uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1 + + - name: Create GitHub release + run: | + bundle exec rake ci:create_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: matrix.ruby != 'jruby' diff --git a/Gemfile b/Gemfile index 98956c7e2..f93eed212 100644 --- a/Gemfile +++ b/Gemfile @@ -10,12 +10,17 @@ group :development do gem "test-unit" gem "test-unit-ruby-core" gem "all_images", "~> 0" unless RUBY_PLATFORM =~ /java/ +end + +group :release do + gem "net-http" + gem "uri" +end - if ENV['BENCHMARK'] - gem "benchmark-ips" - unless RUBY_PLATFORM =~ /java/ - gem "oj" - gem "rapidjson" - end +if ENV['BENCHMARK'] + gem "benchmark-ips" + unless RUBY_PLATFORM =~ /java/ + gem "oj" + gem "rapidjson" end end diff --git a/Rakefile b/Rakefile index 57b1d0ec5..c43acbc69 100644 --- a/Rakefile +++ b/Rakefile @@ -12,6 +12,69 @@ PKG_VERSION = File.foreach(File.join(__dir__, "lib/json/version.rb")) do | /^\s*VERSION\s*=\s*'(.*)'/ =~ line and break $1 end rescue nil +module CI + extend self + + def gem_published?(name, version) + require 'net/https' + require 'uri' + uri = URI.parse("https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json") + Net::HTTP.get_response(uri).is_a?(Net::HTTPSuccess) + end + + def changelog(version) + changelog_lines = File.readlines(File.expand_path("CHANGES.md", __dir__)) + changelog_lines = changelog_lines.drop_while { |line| !(line.start_with?("### ") && line.include?("(#{version})")) } + if changelog_lines.empty? + return false + end + changelog_lines.shift + changelog_lines = changelog_lines.take_while { |line| !(line.start_with?("### ") ) } + changelog_lines.join + end + + def prerelease?(version) + !version.match?(/\A[\d.]+\z/) + end +end + +namespace :ci do + task :check_release do + unless PKG_VERSION + abort("Gem version couldn't be read") + end + + if CI.gem_published?("json", PKG_VERSION) + $stderr.puts "Version #{PKG_VERSION} was already released. Nothing to do." + exit 0 + end + + if changelog = CI.changelog(PKG_VERSION) + puts "Changelog:" + puts changelog + else + abort("Could not find version #{PKG_VERSION} in CHANGES.md") + end + + if ENV["GITHUB_OUTPUT"] + $stderr.puts "Triggering release" + File.open(ENV["GITHUB_OUTPUT"], "a+") do |f| + f.puts "run_publish=true" + end + end + end + + task :create_release do + tag = "v#{PKG_VERSION}" + args = [ + "--title", tag, + "--notes", changelog, + ] + args << "--prerelease" << "--latest=false" if CI.prerelease?(PKG_VERSION) + system("gh", "release", "create", tag, "--draft", *args) + end +end + JAVA_DIR = "java/src/json/ext" JAVA_RAGEL_PATH = "#{JAVA_DIR}/ParserConfig.rl" JAVA_PARSER_SRC = "#{JAVA_DIR}/ParserConfig.java" From 036e30fbbe65e026ab67ec130697bf1aab54973b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 Aug 2025 16:24:03 +0200 Subject: [PATCH 136/506] Remove push_gem workflow --- .github/workflows/push_gem.yml | 85 ---------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 .github/workflows/push_gem.yml diff --git a/.github/workflows/push_gem.yml b/.github/workflows/push_gem.yml deleted file mode 100644 index cb4a260e5..000000000 --- a/.github/workflows/push_gem.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Publish - -on: - workflow_run: - workflows: [CI] - types: [completed] - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - outputs: - run_publish: ${{ steps.precheck.outputs.run_publish }} - - steps: - - uses: actions/checkout@v5 - - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: "3.4" - bundler-cache: true - - - id: precheck - name: Release precheck - run: bundle exec rake ci:check_release - - push: - needs: check - - if: | - needs.check.outputs.run_publish == 'true' && - github.repository == 'ruby/json' && - github.ref_name == github.event.repository.default_branch - runs-on: ubuntu-latest - - environment: - name: rubygems.org - url: https://rubygems.org/gems/json - - permissions: - contents: write - id-token: write - - strategy: - matrix: - ruby: ["ruby", "jruby"] - - steps: - - name: Harden Runner - uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 - with: - egress-policy: audit - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set up Ruby - uses: ruby/setup-ruby@eaecf785f6a34567a6d97f686bbb7bccc1ac1e5c # v1.237.0 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - # https://github.com/rubygems/rubygems/issues/5882 - - name: Install dependencies and build for JRuby - run: | - sudo apt install default-jdk maven - gem update --system - gem install ruby-maven rake-compiler --no-document - rake compile - if: matrix.ruby == 'jruby' - - - name: Install dependencies - run: bundle install --jobs 4 --retry 3 - - - name: Publish to RubyGems - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1 - - - name: Create GitHub release - run: | - bundle exec rake ci:create_release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: matrix.ruby != 'jruby' From 77292cbc9b4b63248caaaff0fa2973b53e1eb9cd Mon Sep 17 00:00:00 2001 From: Robin Miller Date: Mon, 1 Sep 2025 22:40:34 -0600 Subject: [PATCH 137/506] Added testing for JSON.unsafe_load. Fixes NoMethodError when passing proc to JSON.unsafe_load, matching the changes made in 73d2137fd3ad18e2a524553e80531f434cf26dde. --- CHANGES.md | 1 + lib/json/common.rb | 10 ++- test/json/json_common_interface_test.rb | 81 +++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 779dd3ef7..14c770606 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,7 @@ ``` * Fix `JSON.generate` `strict: true` mode to also restrict hash keys. * Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. +* Fix `JSON.unsafe_load` usage with proc ### 2025-07-28 (2.13.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index 45200a83b..f8aefcd28 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -703,9 +703,13 @@ def unsafe_load(source, proc = nil, options = nil) if opts[:allow_blank] && (source.nil? || source.empty?) source = 'null' end - result = parse(source, opts) - recurse_proc(result, &proc) if proc - result + + if proc + opts = opts.dup + opts[:on_load] = proc.to_proc + end + + parse(source, opts) end # :call-seq: diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 745400faa..03e152265 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -162,6 +162,87 @@ def test_load_null assert_raise(JSON::ParserError) { JSON.load('', nil, :allow_blank => false) } end + def test_unsafe_load + string_able_klass = Class.new do + def initialize(str) + @str = str + end + + def to_str + @str + end + end + + io_able_klass = Class.new do + def initialize(str) + @str = str + end + + def to_io + StringIO.new(@str) + end + end + + assert_equal @hash, JSON.unsafe_load(@json) + tempfile = Tempfile.open('@json') + tempfile.write @json + tempfile.rewind + assert_equal @hash, JSON.unsafe_load(tempfile) + stringio = StringIO.new(@json) + stringio.rewind + assert_equal @hash, JSON.unsafe_load(stringio) + string_able = string_able_klass.new(@json) + assert_equal @hash, JSON.unsafe_load(string_able) + io_able = io_able_klass.new(@json) + assert_equal @hash, JSON.unsafe_load(io_able) + assert_equal nil, JSON.unsafe_load(nil) + assert_equal nil, JSON.unsafe_load('') + ensure + tempfile.close! + end + + def test_unsafe_load_with_proc + visited = [] + JSON.unsafe_load('{"foo": [1, 2, 3], "bar": {"baz": "plop"}}', proc { |o| visited << JSON.dump(o); o }) + + expected = [ + '"foo"', + '1', + '2', + '3', + '[1,2,3]', + '"bar"', + '"baz"', + '"plop"', + '{"baz":"plop"}', + '{"foo":[1,2,3],"bar":{"baz":"plop"}}', + ] + assert_equal expected, visited + end + + def test_unsafe_load_default_options + too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' + assert JSON.unsafe_load(too_deep, nil).is_a?(Array) + nan_json = '{ "foo": NaN }' + assert JSON.unsafe_load(nan_json, nil)['foo'].nan? + assert_equal nil, JSON.unsafe_load(nil, nil) + t = Time.now + assert_equal t, JSON.unsafe_load(JSON(t)) + end + + def test_unsafe_load_with_options + nan_json = '{ "foo": NaN }' + assert_raise(JSON::ParserError) { JSON.unsafe_load(nan_json, nil, :allow_nan => false)['foo'].nan? } + # make sure it still uses the defaults when something is provided + assert JSON.unsafe_load(nan_json, nil, :allow_blank => true)['foo'].nan? + end + + def test_unsafe_load_null + assert_equal nil, JSON.unsafe_load(nil, nil, :allow_blank => true) + assert_raise(TypeError) { JSON.unsafe_load(nil, nil, :allow_blank => false) } + assert_raise(JSON::ParserError) { JSON.unsafe_load('', nil, :allow_blank => false) } + end + def test_dump too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' obj = eval(too_deep) From 92654cd99bdacb6b5a47f8c8b2a0bf3c4bbc08f6 Mon Sep 17 00:00:00 2001 From: Robin Miller Date: Mon, 1 Sep 2025 22:52:16 -0600 Subject: [PATCH 138/506] Update method docs for JSON.load and JSON.unsafe_load to show the correct use of proc argument. --- lib/json/common.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/json/common.rb b/lib/json/common.rb index f8aefcd28..ae9b787a0 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -662,6 +662,7 @@ def pretty_generate(obj, opts = nil) # when Array # obj.map! {|v| deserialize_obj v } # end + # obj # }) # pp ruby # Output: @@ -826,6 +827,7 @@ def unsafe_load(source, proc = nil, options = nil) # when Array # obj.map! {|v| deserialize_obj v } # end + # obj # }) # pp ruby # Output: From 7a8ec8258f4ff281a55ab2cbad813d66324b190d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 3 Sep 2025 14:41:10 +0200 Subject: [PATCH 139/506] Use Time.new because JSON.unsafe_load losts fractions of a second and JSON.unsafe_load with Time object returns String ``` 1) Failure: JSONCommonInterfaceTest#test_unsafe_load_default_options [/path/to/ruby/test/json/json_common_interface_test.rb:230]: <2025-09-03 14:28:31.293635 +0200> expected but was <"2025-09-03 14:28:31 +0200">. ``` --- test/json/json_common_interface_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 03e152265..077d7e153 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -226,8 +226,8 @@ def test_unsafe_load_default_options nan_json = '{ "foo": NaN }' assert JSON.unsafe_load(nan_json, nil)['foo'].nan? assert_equal nil, JSON.unsafe_load(nil, nil) - t = Time.now - assert_equal t, JSON.unsafe_load(JSON(t)) + t = Time.new(2025, 9, 3, 14, 50, 0) + assert_equal t.to_s, JSON.unsafe_load(JSON(t)).to_s end def test_unsafe_load_with_options From f37e73cf44556b0c3fef4b9b47e5baf2181cba22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 12:11:22 +0200 Subject: [PATCH 140/506] Enable test coverage when simplecov is available Co-authored-by: Jean Boussier --- Gemfile | 1 + test/json/test_helper.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/Gemfile b/Gemfile index f93eed212..c02e4d494 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,7 @@ group :development do gem "test-unit" gem "test-unit-ruby-core" gem "all_images", "~> 0" unless RUBY_PLATFORM =~ /java/ + gem "simplecov", require: false end group :release do diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index d849e28b9..cf592debf 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,5 +1,13 @@ $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) +begin + require 'simplecov' +rescue LoadError + # Don't fail Ruby's test suite +else + SimpleCov.start +end + require 'json' require 'test/unit' From 1397bf8cc9459859b0467128e13b5199575cc824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 12:14:14 +0200 Subject: [PATCH 141/506] Don't call json_create if json_creatable? returns false Fix f411ddf1ceaf1ea82c8470c94da522919a4fb3c9 Co-authored-by: Jean Boussier --- lib/json/common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index ae9b787a0..1f82c4624 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -73,7 +73,7 @@ def create_additions_proc(opts) if opts[:create_additions] != false if class_name = object[JSON.create_id] klass = JSON.deep_const_get(class_name) - if (klass.respond_to?(:json_creatable?) && klass.json_creatable?) || klass.respond_to?(:json_create) + if klass.respond_to?(:json_creatable?) ? klass.json_creatable? : klass.respond_to?(:json_create) create_additions_warning if create_additions.nil? object = klass.json_create(object) end From 7a2a24d663887a765388b107f3ec9b4dab12413c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 12:12:55 +0200 Subject: [PATCH 142/506] Fix JSON::GenericObject tests Co-authored-by: Jean Boussier --- test/json/json_generic_object_test.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb index 471534192..333da7047 100644 --- a/test/json/json_generic_object_test.rb +++ b/test/json/json_generic_object_test.rb @@ -4,7 +4,7 @@ class JSONGenericObjectTest < Test::Unit::TestCase def setup - if defined?(GenericObject) + if defined?(JSON::GenericObject) @go = JSON::GenericObject[ :a => 1, :b => 2 ] else omit("JSON::GenericObject is not available") @@ -40,10 +40,10 @@ def test_parse_json ) assert_equal 1, l.a assert_equal @go, - l = JSON('{ "a": 1, "b": 2 }', :object_class => GenericObject) + l = JSON('{ "a": 1, "b": 2 }', :object_class => JSON::GenericObject) assert_equal 1, l.a - assert_equal GenericObject[:a => GenericObject[:b => 2]], - l = JSON('{ "a": { "b": 2 } }', :object_class => GenericObject) + assert_equal JSON::GenericObject[:a => JSON::GenericObject[:b => 2]], + l = JSON('{ "a": { "b": 2 } }', :object_class => JSON::GenericObject) assert_equal 2, l.a.b end end @@ -51,12 +51,12 @@ def test_parse_json def test_from_hash result = JSON::GenericObject.from_hash( :foo => { :bar => { :baz => true }, :quux => [ { :foobar => true } ] }) - assert_kind_of GenericObject, result.foo - assert_kind_of GenericObject, result.foo.bar + assert_kind_of JSON::GenericObject, result.foo + assert_kind_of JSON::GenericObject, result.foo.bar assert_equal true, result.foo.bar.baz - assert_kind_of GenericObject, result.foo.quux.first + assert_kind_of JSON::GenericObject, result.foo.quux.first assert_equal true, result.foo.quux.first.foobar - assert_equal true, GenericObject.from_hash(true) + assert_equal true, JSON::GenericObject.from_hash(true) end def test_json_generic_object_load From 2ec31a736375b5b1f63dc447c82a5e10034aa812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 12:17:24 +0200 Subject: [PATCH 143/506] Fix deprecation warning backtrace The gem root was including the test/ directory, so lines from tests were skipped, and the deprecation warnings were shown as coming from the test framework. Co-authored-by: Jean Boussier --- lib/json/common.rb | 2 +- test/json/json_parser_test.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 1f82c4624..60178c2a5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -97,7 +97,7 @@ def create_additions_warning class << self def deprecation_warning(message, uplevel = 3) # :nodoc: - gem_root = File.expand_path("../../../", __FILE__) + "/" + gem_root = File.expand_path("..", __dir__) + "/" caller_locations(uplevel, 10).each do |frame| if frame.path.nil? || frame.path.start_with?(gem_root) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") uplevel += 1 diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3c60fec87..45474c982 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -347,7 +347,7 @@ def test_parse_duplicate_key assert_equal expected_sym, parse('{"a": 1, "a": 2}', symbolize_names: true) end - if RUBY_ENGINE == 'RUBY_ENGINE' + if RUBY_ENGINE == 'ruby' assert_deprecated_warning(/#{File.basename(__FILE__)}\:#{__LINE__ + 1}/) do assert_equal expected, parse('{"a": 1, "a": 2}') end From efc61682eec8a19d9c21c381a703f5ae25ccd8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 14:50:18 +0200 Subject: [PATCH 144/506] Change invalid encoding test to use assert_raise Co-authored-by: Jean Boussier --- test/json/json_encoding_test.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index 873e96fdd..caf335d52 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -145,19 +145,11 @@ def test_very_large_json_strings end def test_invalid_utf8_sequences - # Create strings with invalid UTF-8 sequences invalid_utf8 = "\xFF\xFF" - - # Test that generating JSON with invalid UTF-8 raises an error - # Different JSON implementations may handle this differently, - # so we'll check if any exception is raised - begin + error = assert_raise(JSON::GeneratorError) do generate(invalid_utf8) - raise "Expected an exception when generating JSON with invalid UTF8" - rescue StandardError => e - assert true - assert_match(%r{source sequence is illegal/malformed utf-8}, e.message) end + assert_match(%r{source sequence is illegal/malformed utf-8}, error.message) end def test_surrogate_pair_handling From a9e05d6ff348360bb0fdba934c2580c95f875d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 15:22:21 +0200 Subject: [PATCH 145/506] Stop checking for String#@- Co-authored-by: Jean Boussier --- test/json/json_parser_test.rb | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 45474c982..2217f2b9e 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -397,10 +397,8 @@ def test_freeze assert_predicate parse('[]', :freeze => true), :frozen? assert_predicate parse('"foo"', :freeze => true), :frozen? - if string_deduplication_available? - assert_same(-'foo', parse('"foo"', :freeze => true)) - assert_same(-'foo', parse('{"foo": 1}', :freeze => true).keys.first) - end + assert_same(-'foo', parse('"foo"', :freeze => true)) + assert_same(-'foo', parse('{"foo": 1}', :freeze => true).keys.first) end def test_parse_comments @@ -794,16 +792,6 @@ def test_parse_leading_slash private - def string_deduplication_available? - r1 = rand.to_s - r2 = r1.dup - begin - (-r1).equal?(-r2) - rescue NoMethodError - false # No String#-@ - end - end - def assert_equal_float(expected, actual, delta = 1e-2) Array === expected and expected = expected.first Array === actual and actual = actual.first From 3232e8df4153edfa988b5e2189410d76a58d776c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 15:26:25 +0200 Subject: [PATCH 146/506] Stop checking for OpenStruct#[] and #[]= Co-authored-by: Jean Boussier --- lib/json/generic_object.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/json/generic_object.rb b/lib/json/generic_object.rb index ec5aa9dcb..5c8ace354 100644 --- a/lib/json/generic_object.rb +++ b/lib/json/generic_object.rb @@ -52,14 +52,6 @@ def to_hash table end - def [](name) - __send__(name) - end unless method_defined?(:[]) - - def []=(name, value) - __send__("#{name}=", value) - end unless method_defined?(:[]=) - def |(other) self.class[other.to_hash.merge(to_hash)] end From bae760aa195bdc59006c76802bafcaea563ea6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 8 Sep 2025 15:27:02 +0200 Subject: [PATCH 147/506] Improve test coverage Test SubArrayWrapper#[] and SubOpenStruct#[] Co-authored-by: Jean Boussier --- test/json/json_parser_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 2217f2b9e..a9b0624f6 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -635,6 +635,7 @@ def test_parse_array_custom_array_derived_class def test_parse_array_custom_non_array_derived_class res = parse('[1,2]', :array_class => SubArrayWrapper) assert_equal([1,2], res.data) + assert_equal(1, res[0]) assert_equal(SubArrayWrapper, res.class) assert res.shifted? end @@ -696,6 +697,7 @@ def item_set? def test_parse_object_custom_non_hash_derived_class res = parse('{"foo":"bar"}', :object_class => SubOpenStruct) assert_equal "bar", res.foo + assert_equal "bar", res[:foo] assert_equal(SubOpenStruct, res.class) assert res.item_set? end From ca72019fd3d2f3dd4c5f30466a9a441252310afd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 11 Sep 2025 10:04:40 +0200 Subject: [PATCH 148/506] Enable coverage before loading simplecov Fix: https://github.com/ruby/json/pull/853 Simplecov end up requiring json so we need to start collecting coverage before. --- test/json/test_helper.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index cf592debf..a788804dd 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,5 +1,8 @@ $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) +require 'coverage' +Coverage.start + begin require 'simplecov' rescue LoadError From 6f6a4cdfd793abb6e5748a26dcbdddf61490ffbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 15 Sep 2025 16:26:43 +0200 Subject: [PATCH 149/506] Better handle missing ostruct In the Ruby test suite, this test class is causing trouble because ostruct is not available. Having an autoload for JSON::GenericObject but causing it not to define the constant causes a warning. See 0dc1cd407e7775610f2bcaef6c1282369867f91c and caa5d8cdd7483647013af5e3d1701b0ed3ec8785 in ruby. We can skip defining the test class entirely instead when ostruct is not available. --- test/json/json_generic_object_test.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb index 333da7047..57e3bf3c5 100644 --- a/test/json/json_generic_object_test.rb +++ b/test/json/json_generic_object_test.rb @@ -1,8 +1,14 @@ # frozen_string_literal: true require_relative 'test_helper' -class JSONGenericObjectTest < Test::Unit::TestCase +# ostruct is required to test JSON::GenericObject +begin + require "ostruct" +rescue LoadError + return +end +class JSONGenericObjectTest < Test::Unit::TestCase def setup if defined?(JSON::GenericObject) @go = JSON::GenericObject[ :a => 1, :b => 2 ] From 5855f4f6030b4ce730ae081ad6659c3dfb025f42 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 10:45:44 +0200 Subject: [PATCH 150/506] parser: Reject invalid surogate pairs more consistently. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 7 ++++++- test/json/json_parser_test.rb | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 14c770606..0a123ab9d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ * Fix `JSON.generate` `strict: true` mode to also restrict hash keys. * Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. * Fix `JSON.unsafe_load` usage with proc +* Fix the parser to more consistently reject invalid UTF-16 surogate pairs. ### 2025-07-28 (2.13.2) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e34f1999d..297031dcf 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -713,11 +713,16 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c } if (pe[0] == '\\' && pe[1] == 'u') { uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2); + + if ((sur & 0xFC00) != 0xDC00) { + raise_parse_error_at("invalid surrogate pair at %s", state, p); + } + ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); pe += 5; } else { - unescape = (char *) "?"; + raise_parse_error_at("incomplete surrogate pair at %s", state, p); break; } } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index a9b0624f6..9d387cb80 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -319,6 +319,12 @@ def test_invalid_unicode_escape assert_raise(JSON::ParserError) { parse('"\u111___"') } end + def test_invalid_surogates + assert_raise(JSON::ParserError) { parse('"\\uD800"') } + assert_raise(JSON::ParserError) { parse('"\\uD800_________________"') } + assert_raise(JSON::ParserError) { parse('"\\uD800\\u0041"') } + end + def test_parse_big_integers json1 = JSON(orig = (1 << 31) - 1) assert_equal orig, parse(json1) From db89486fff9b7decf059f63329e6752dbd2e9b95 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 10:53:10 +0200 Subject: [PATCH 151/506] Regenerate the Java parser --- java/src/json/ext/ParserConfig.java | 84 ++++++++++++++--------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 4a44f9be8..2bd03babc 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -301,11 +301,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int return parsingError(context, "unexpected token at '", absStart, absEnd); } - + // line 328 "ParserConfig.rl" - + // line 310 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { @@ -427,14 +427,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject result = null; - + // line 432 "ParserConfig.java" { cs = JSON_value_start; } // line 441 "ParserConfig.rl" - + // line 439 "ParserConfig.java" { int _klen; @@ -671,7 +671,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } - + // line 676 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { @@ -733,7 +733,7 @@ private static byte[] init__JSON_integer_index_offsets_0() private static final byte _JSON_integer_index_offsets[] = init__JSON_integer_index_offsets_0(); -private static byte[] init__JSON_integer_indices_0() +private static byte[] init__JSON_integer_indicies_0() { return new byte [] { 0, 2, 3, 1, 2, 3, 1, 1, 4, 1, 3, 4, @@ -741,7 +741,7 @@ private static byte[] init__JSON_integer_indices_0() }; } -private static final byte _JSON_integer_indices[] = init__JSON_integer_indices_0(); +private static final byte _JSON_integer_indicies[] = init__JSON_integer_indicies_0(); private static byte[] init__JSON_integer_trans_targs_0() @@ -787,7 +787,7 @@ void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { int parseIntegerInternal(int p, int pe) { int cs; - + // line 792 "ParserConfig.java" { cs = JSON_integer_start; @@ -795,7 +795,7 @@ int parseIntegerInternal(int p, int pe) { // line 480 "ParserConfig.rl" int memo = p; - + // line 800 "ParserConfig.java" { int _klen; @@ -866,7 +866,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_integer_indices[_trans]; + _trans = _JSON_integer_indicies[_trans]; cs = _JSON_integer_trans_targs[_trans]; if ( _JSON_integer_trans_actions[_trans] != 0 ) { @@ -922,7 +922,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { return ConvertBytes.byteListToInum(runtime, num, 10, true); } - + // line 927 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { @@ -986,7 +986,7 @@ private static byte[] init__JSON_float_index_offsets_0() private static final byte _JSON_float_index_offsets[] = init__JSON_float_index_offsets_0(); -private static byte[] init__JSON_float_indices_0() +private static byte[] init__JSON_float_indicies_0() { return new byte [] { 0, 2, 3, 1, 2, 3, 1, 4, 5, 5, 1, 6, @@ -995,7 +995,7 @@ private static byte[] init__JSON_float_indices_0() }; } -private static final byte _JSON_float_indices[] = init__JSON_float_indices_0(); +private static final byte _JSON_float_indicies[] = init__JSON_float_indicies_0(); private static byte[] init__JSON_float_trans_targs_0() @@ -1043,7 +1043,7 @@ void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { int parseFloatInternal(int p, int pe) { int cs; - + // line 1048 "ParserConfig.java" { cs = JSON_float_start; @@ -1051,7 +1051,7 @@ int parseFloatInternal(int p, int pe) { // line 533 "ParserConfig.rl" int memo = p; - + // line 1056 "ParserConfig.java" { int _klen; @@ -1122,7 +1122,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_float_indices[_trans]; + _trans = _JSON_float_indicies[_trans]; cs = _JSON_float_trans_targs[_trans]; if ( _JSON_float_trans_actions[_trans] != 0 ) { @@ -1168,7 +1168,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) return p; } - + // line 1173 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { @@ -1232,7 +1232,7 @@ private static byte[] init__JSON_string_index_offsets_0() private static final byte _JSON_string_index_offsets[] = init__JSON_string_index_offsets_0(); -private static byte[] init__JSON_string_indices_0() +private static byte[] init__JSON_string_indicies_0() { return new byte [] { 0, 1, 2, 3, 1, 0, 4, 1, 0, 5, 5, 5, @@ -1241,7 +1241,7 @@ private static byte[] init__JSON_string_indices_0() }; } -private static final byte _JSON_string_indices[] = init__JSON_string_indices_0(); +private static final byte _JSON_string_indicies[] = init__JSON_string_indicies_0(); private static byte[] init__JSON_string_trans_targs_0() @@ -1278,7 +1278,7 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { int cs; IRubyObject result = null; - + // line 1283 "ParserConfig.java" { cs = JSON_string_start; @@ -1286,7 +1286,7 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { // line 581 "ParserConfig.rl" int memo = p; - + // line 1291 "ParserConfig.java" { int _klen; @@ -1357,7 +1357,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_string_indices[_trans]; + _trans = _JSON_string_indicies[_trans]; cs = _JSON_string_trans_targs[_trans]; if ( _JSON_string_trans_actions[_trans] != 0 ) { @@ -1429,7 +1429,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } } - + // line 1434 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { @@ -1549,7 +1549,7 @@ private static short[] init__JSON_array_index_offsets_0() private static final short _JSON_array_index_offsets[] = init__JSON_array_index_offsets_0(); -private static byte[] init__JSON_array_indices_0() +private static byte[] init__JSON_array_indicies_0() { return new byte [] { 0, 1, 0, 0, 2, 2, 3, 2, 2, 2, 4, 2, @@ -1565,7 +1565,7 @@ private static byte[] init__JSON_array_indices_0() }; } -private static final byte _JSON_array_indices[] = init__JSON_array_indices_0(); +private static final byte _JSON_array_indicies[] = init__JSON_array_indicies_0(); private static byte[] init__JSON_array_trans_targs_0() @@ -1610,14 +1610,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); - + // line 1615 "ParserConfig.java" { cs = JSON_array_start; } // line 650 "ParserConfig.rl" - + // line 1622 "ParserConfig.java" { int _klen; @@ -1660,7 +1660,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) switch ( _JSON_array_cond_spaces[_JSON_array_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { case 0: { _widec = 65536 + (data[p] - 0); - if ( + if ( // line 608 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; @@ -1720,7 +1720,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_array_indices[_trans]; + _trans = _JSON_array_indicies[_trans]; cs = _JSON_array_trans_targs[_trans]; if ( _JSON_array_trans_actions[_trans] != 0 ) { @@ -1779,7 +1779,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } } - + // line 1784 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { @@ -1906,7 +1906,7 @@ private static short[] init__JSON_object_index_offsets_0() private static final short _JSON_object_index_offsets[] = init__JSON_object_index_offsets_0(); -private static byte[] init__JSON_object_indices_0() +private static byte[] init__JSON_object_indicies_0() { return new byte [] { 0, 1, 0, 0, 2, 3, 4, 0, 1, 5, 5, 6, @@ -1923,7 +1923,7 @@ private static byte[] init__JSON_object_indices_0() }; } -private static final byte _JSON_object_indices[] = init__JSON_object_indices_0(); +private static final byte _JSON_object_indicies[] = init__JSON_object_indicies_0(); private static byte[] init__JSON_object_trans_targs_0() @@ -1973,14 +1973,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { // allocator test at OptionsReader#getClass IRubyObject result = RubyHash.newHash(context.runtime); - + // line 1978 "ParserConfig.java" { cs = JSON_object_start; } // line 737 "ParserConfig.rl" - + // line 1985 "ParserConfig.java" { int _klen; @@ -2023,7 +2023,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) switch ( _JSON_object_cond_spaces[_JSON_object_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { case 0: { _widec = 65536 + (data[p] - 0); - if ( + if ( // line 665 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; @@ -2083,7 +2083,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_object_indices[_trans]; + _trans = _JSON_object_indicies[_trans]; cs = _JSON_object_trans_targs[_trans]; if ( _JSON_object_trans_actions[_trans] != 0 ) { @@ -2174,7 +2174,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) res.update(config.onLoad(context, result), p + 1); } - + // line 2179 "ParserConfig.java" private static byte[] init__JSON_actions_0() { @@ -2238,7 +2238,7 @@ private static byte[] init__JSON_index_offsets_0() private static final byte _JSON_index_offsets[] = init__JSON_index_offsets_0(); -private static byte[] init__JSON_indices_0() +private static byte[] init__JSON_indicies_0() { return new byte [] { 0, 0, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, @@ -2248,7 +2248,7 @@ private static byte[] init__JSON_indices_0() }; } -private static final byte _JSON_indices[] = init__JSON_indices_0(); +private static final byte _JSON_indicies[] = init__JSON_indicies_0(); private static byte[] init__JSON_trans_targs_0() @@ -2287,7 +2287,7 @@ public IRubyObject parseImplementation(ThreadContext context) { IRubyObject result = null; ParserResult res = new ParserResult(); - + // line 2292 "ParserConfig.java" { cs = JSON_start; @@ -2296,7 +2296,7 @@ public IRubyObject parseImplementation(ThreadContext context) { // line 776 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); - + // line 2301 "ParserConfig.java" { int _klen; @@ -2367,7 +2367,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } while (false); - _trans = _JSON_indices[_trans]; + _trans = _JSON_indicies[_trans]; cs = _JSON_trans_targs[_trans]; if ( _JSON_trans_actions[_trans] != 0 ) { From 6bded942c4fa21eb15b4e5663f0980a8f947786a Mon Sep 17 00:00:00 2001 From: Robin Miller Date: Sat, 13 Sep 2025 16:11:20 -0600 Subject: [PATCH 152/506] Add branch test coverage when available. Force track all files to prevent implicit file discovery missing files. --- test/json/test_helper.rb | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index a788804dd..24d98170b 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,14 +1,29 @@ $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) require 'coverage' -Coverage.start + +branches_supported = Coverage.respond_to?(:supported?) && Coverage.supported?(:branches) + +# Coverage module must be started before SimpleCov to work around the cyclic require order. +# Track both branches and lines, or else SimpleCov misleadingly reports 0/0 = 100% for non-branching files. +Coverage.start(lines: true, + branches: branches_supported) begin require 'simplecov' rescue LoadError # Don't fail Ruby's test suite else - SimpleCov.start + SimpleCov.start do + # Enabling both coverage types to let SimpleCov know to output them together in reports + enable_coverage :line + enable_coverage :branch if branches_supported + + # Can't always trust SimpleCov to find files implicitly + track_files 'lib/**/*.rb' + + add_filter 'lib/json/truffle_ruby' unless RUBY_ENGINE == 'truffleruby' + end end require 'json' From 08b9eb0ee66abff31000fffae19ab74c907c4576 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 11:20:21 +0200 Subject: [PATCH 153/506] Only enable test coverage when running the test suite standalone --- Rakefile | 8 ++++++++ test/json/test_helper.rb | 18 ++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Rakefile b/Rakefile index c43acbc69..c1768853c 100644 --- a/Rakefile +++ b/Rakefile @@ -6,6 +6,14 @@ include RbConfig require 'rake/testtask' class UndocumentedTestTask < Rake::TestTask def desc(*) end + + def ruby(...) + env_was = ENV["JSON_COVERAGE"] + ENV["JSON_COVERAGE"] = "1" + ret = super + ENV["JSON_COVERAGE"] = env_was + ret + end end PKG_VERSION = File.foreach(File.join(__dir__, "lib/json/version.rb")) do |line| diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index 24d98170b..24cde4348 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -1,19 +1,17 @@ $LOAD_PATH.unshift(File.expand_path('../../../ext', __FILE__), File.expand_path('../../../lib', __FILE__)) -require 'coverage' +if ENV["JSON_COVERAGE"] + # This test helper is loaded inside Ruby's own test suite, so we try to not mess it up. + require 'coverage' -branches_supported = Coverage.respond_to?(:supported?) && Coverage.supported?(:branches) + branches_supported = Coverage.respond_to?(:supported?) && Coverage.supported?(:branches) -# Coverage module must be started before SimpleCov to work around the cyclic require order. -# Track both branches and lines, or else SimpleCov misleadingly reports 0/0 = 100% for non-branching files. -Coverage.start(lines: true, - branches: branches_supported) + # Coverage module must be started before SimpleCov to work around the cyclic require order. + # Track both branches and lines, or else SimpleCov misleadingly reports 0/0 = 100% for non-branching files. + Coverage.start(lines: true, + branches: branches_supported) -begin require 'simplecov' -rescue LoadError - # Don't fail Ruby's test suite -else SimpleCov.start do # Enabling both coverage types to let SimpleCov know to output them together in reports enable_coverage :line From 55552cafe2c4d54658fddd7a8c6134a42b0200b5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 11:28:05 +0200 Subject: [PATCH 154/506] Release 2.14.0 --- lib/json/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/version.rb b/lib/json/version.rb index f9ac3e17a..165b93cff 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.13.2' + VERSION = '2.14.0' end From 1d52d184892c9c40802c1f79e79b4dbf8d859327 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 16:18:50 +0200 Subject: [PATCH 155/506] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0a123ab9d..79398951b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-09-18 (2.14.0) + * Add new `allow_duplicate_key` generator options. By default a warning is now emitted when a duplicated key is encountered. In `json 3.0` an error will be raised. ```ruby From 67ebabec7587ee689b4293895606ac652432e665 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 18 Sep 2025 08:50:55 -0500 Subject: [PATCH 156/506] fix issue reading off the end of the ByteBuffer if ptr > 0 Fix: https://github.com/ruby/json/issues/859 Co-Authored-By: Jean Boussier --- CHANGES.md | 2 ++ java/src/json/ext/SWARBasicStringEncoder.java | 8 ++++---- test/json/json_encoding_test.rb | 6 ++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 79398951b..942602b9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix `IndexOutOfBoundsException` in the JRuby extension when encoding shared strings. + ### 2025-09-18 (2.14.0) * Add new `allow_duplicate_key` generator options. By default a warning is now emitted when a duplicated key is encountered. diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java index 8021d73b8..08cd2e473 100644 --- a/java/src/json/ext/SWARBasicStringEncoder.java +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -24,14 +24,14 @@ void encode(ByteList src) throws IOException { int pos = 0; ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, len); - while (pos + 8 <= len) { + while (ptr + pos + 8 <= len) { long x = bb.getLong(ptr + pos); if (skipChunk(x)) { pos += 8; continue; } - int chunkEnd = pos + 8; - while (pos < chunkEnd) { + int chunkEnd = ptr + pos + 8; + while (ptr + pos < chunkEnd) { int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); int ch_len = ESCAPE_TABLE[ch]; if (ch_len > 0) { @@ -43,7 +43,7 @@ void encode(ByteList src) throws IOException { } } - if (pos + 4 <= len) { + if (ptr + pos + 4 <= len) { int x = bb.getInt(ptr + pos); if (skipChunk(x)) { pos += 4; diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index caf335d52..8dce81da5 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -31,6 +31,12 @@ def test_generate assert_equal @generated, JSON.generate(@utf_16_data, ascii_only: true) end + def test_generate_shared_string + # Ref: https://github.com/ruby/json/issues/859 + s = "01234567890" + assert_equal '"234567890"', JSON.dump(s[2..-1]) + end + def test_unicode assert_equal '""', ''.to_json assert_equal '"\\b"', "\b".to_json From 51ce76ea662ea29a2a223698d53e1eacd542fadd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 16:26:02 +0200 Subject: [PATCH 157/506] Release 2.14.1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 942602b9c..5f50531b3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-09-18 (2.14.1) + * Fix `IndexOutOfBoundsException` in the JRuby extension when encoding shared strings. ### 2025-09-18 (2.14.0) diff --git a/lib/json/version.rb b/lib/json/version.rb index 165b93cff..9c928e394 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.14.0' + VERSION = '2.14.1' end From 1566cd01a60bf290d75b00b9b4308f396ba3827a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 18:57:26 +0200 Subject: [PATCH 158/506] Avoid scientific notation before exponent 15 Fix: https://github.com/ruby/json/issues/861 It's not incorrect to use scientific notation, but it tend to throw people off a bit, so it's best to keep it for very large numbers. --- CHANGES.md | 2 ++ ext/json/ext/generator/generator.c | 5 ++- ext/json/ext/vendor/fpconv.c | 23 ++++++------- test/json/json_generator_test.rb | 53 +++++++++++++++++++----------- 4 files changed, 50 insertions(+), 33 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5f50531b3..7120367f0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Tuned the floating point number generator to not use scientific notation as agressively. + ### 2025-09-18 (2.14.1) * Fix `IndexOutOfBoundsException` in the JRuby extension when encoding shared strings. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 8fcc980d4..9b67629f9 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1345,12 +1345,11 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data } /* This implementation writes directly into the buffer. We reserve - * the 28 characters that fpconv_dtoa states as its maximum. + * the 32 characters that fpconv_dtoa states as its maximum. */ - fbuffer_inc_capa(buffer, 28); + fbuffer_inc_capa(buffer, 32); char* d = buffer->ptr + buffer->len; int len = fpconv_dtoa(value, d); - /* fpconv_dtoa converts a float to its shortest string representation, * but it adds a ".0" if this is a plain integer. */ diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 75efd46f1..e91c7889c 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -29,6 +29,10 @@ #include #include +#ifdef JSON_DEBUG +#include +#endif + #define npowers 87 #define steppowers 8 #define firstpower -348 /* 10 ^ -348 */ @@ -320,15 +324,7 @@ static int emit_digits(char* digits, int ndigits, char* dest, int K, bool neg) { int exp = absv(K + ndigits - 1); - int max_trailing_zeros = 7; - - if(neg) { - max_trailing_zeros -= 1; - } - - /* write plain integer */ - if(K >= 0 && (exp < (ndigits + max_trailing_zeros))) { - + if(K >= 0 && exp < 15) { memcpy(dest, digits, ndigits); memset(dest + ndigits, '0', K); @@ -432,10 +428,12 @@ static int filter_special(double fp, char* dest) * * Input: * fp -> the double to convert, dest -> destination buffer. - * The generated string will never be longer than 28 characters. - * Make sure to pass a pointer to at least 28 bytes of memory. + * The generated string will never be longer than 32 characters. + * Make sure to pass a pointer to at least 32 bytes of memory. * The emitted string will not be null terminated. * + * + * * Output: * The number of written characters. * @@ -474,6 +472,9 @@ static int fpconv_dtoa(double d, char dest[28]) int ndigits = grisu2(d, digits, &K); str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); +#ifdef JSON_DEBUG + assert(str_len <= 32); +#endif return str_len; } diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 6b42de2ad..4fdfa12b0 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -825,26 +825,41 @@ def test_json_generate_as_json_convert_to_proc assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: :object_id) end - def test_json_generate_float - values = [-1.0, 1.0, 0.0, 12.2, 7.5 / 3.2, 12.0, 100.0, 1000.0] - expecteds = ["-1.0", "1.0", "0.0", "12.2", "2.34375", "12.0", "100.0", "1000.0"] - - if RUBY_ENGINE == "jruby" - values << 1746861937.7842371 - expecteds << "1.7468619377842371E9" - else - values << 1746861937.7842371 - expecteds << "1746861937.7842371" - end - - if RUBY_ENGINE == "ruby" - values << -2.2471348024634545e-08 << -2.2471348024634545e-09 << -2.2471348024634545e-10 - expecteds << "-0.000000022471348024634545" << "-0.0000000022471348024634545" << "-2.2471348024634546e-10" - end + def assert_float_roundtrip(expected, actual) + assert_equal(expected, JSON.generate(actual)) + assert_equal(actual, JSON.parse(JSON.generate(actual)), "JSON: #{JSON.generate(actual)}") + end - values.zip(expecteds).each do |value, expected| - assert_equal expected, value.to_json - end + def test_json_generate_float + assert_float_roundtrip "-1.0", -1.0 + assert_float_roundtrip "1.0", 1.0 + assert_float_roundtrip "0.0", 0.0 + assert_float_roundtrip "12.2", 12.2 + assert_float_roundtrip "2.34375", 7.5 / 3.2 + assert_float_roundtrip "12.0", 12.0 + assert_float_roundtrip "100.0", 100.0 + assert_float_roundtrip "1000.0", 1000.0 + + if RUBY_ENGINE == "jruby" + assert_float_roundtrip "1.7468619377842371E9", 1746861937.7842371 + else + assert_float_roundtrip "1746861937.7842371", 1746861937.7842371 + end + + if RUBY_ENGINE == "ruby" + assert_float_roundtrip "100000000000000.0", 100000000000000.0 + assert_float_roundtrip "1e+15", 1e+15 + assert_float_roundtrip "-100000000000000.0", -100000000000000.0 + assert_float_roundtrip "-1e+15", -1e+15 + assert_float_roundtrip "1111111111111111.1", 1111111111111111.1 + assert_float_roundtrip "1.1111111111111112e+16", 11111111111111111.1 + assert_float_roundtrip "-1111111111111111.1", -1111111111111111.1 + assert_float_roundtrip "-1.1111111111111112e+16", -11111111111111111.1 + + assert_float_roundtrip "-0.000000022471348024634545", -2.2471348024634545e-08 + assert_float_roundtrip "-0.0000000022471348024634545", -2.2471348024634545e-09 + assert_float_roundtrip "-2.2471348024634546e-10", -2.2471348024634545e-10 + end end def test_numbers_of_various_sizes From 7982b6148cba71d50ab9aa26d84681c0e19ef135 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 20:33:41 +0200 Subject: [PATCH 159/506] `JSON::Coder` callback now recieve a second argument to mark object keys e.g. ```ruby { 1 => 2 } ``` The callback will be invoked for `1` as while it has a native JSON equivalent, it's not legal as an object name. --- CHANGES.md | 1 + README.md | 4 +++- ext/json/ext/generator/generator.c | 15 ++++++++++++--- java/src/json/ext/Generator.java | 6 +++--- lib/json/truffle_ruby/generator.rb | 10 +++++----- test/json/json_coder_test.rb | 12 ++++++++---- test/json/json_generator_test.rb | 2 +- 7 files changed, 33 insertions(+), 17 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7120367f0..33929954f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* `JSON::Coder` callback now receive a second argument to convey whether the object is a hash key. * Tuned the floating point number generator to not use scientific notation as agressively. ### 2025-09-18 (2.14.1) diff --git a/README.md b/README.md index 119327213..eed71ba43 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Instead it is recommended to use the newer `JSON::Coder` API: ```ruby module MyApp - API_JSON_CODER = JSON::Coder.new do |object| + API_JSON_CODER = JSON::Coder.new do |object, is_object_key| case object when Time object.iso8601(3) @@ -113,6 +113,8 @@ puts MyApp::API_JSON_CODER.dump(Time.now.utc) # => "2025-01-21T08:41:44.286Z" The provided block is called for all objects that don't have a native JSON equivalent, and must return a Ruby object that has a native JSON equivalent. +It is also called for objects that do have a JSON equivalent, but are used as Hash keys, for instance `{ 1 => 2}`. + ## Combining JSON fragments To combine JSON fragments into a bigger JSON document, you can use `JSON::Fragment`: diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9b67629f9..6a38cc60a 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -29,6 +29,7 @@ typedef struct JSON_Generator_StateStruct { enum duplicate_key_action on_duplicate_key; + bool as_json_single_arg; bool allow_nan; bool ascii_only; bool script_safe; @@ -1033,6 +1034,13 @@ json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) } } +static VALUE +json_call_as_json(JSON_Generator_State *state, VALUE object, VALUE is_key) +{ + VALUE proc_args[2] = {object, is_key}; + return rb_proc_call_with_block(state->as_json, 2, proc_args, Qnil); +} + static int json_object_i(VALUE key, VALUE val, VALUE _arg) { @@ -1086,7 +1094,7 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) default: if (data->state->strict) { if (RTEST(data->state->as_json) && !as_json_called) { - key = rb_proc_call_with_block(data->state->as_json, 1, &key, Qnil); + key = json_call_as_json(data->state, key, Qtrue); key_type = rb_type(key); as_json_called = true; goto start; @@ -1328,7 +1336,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data /* for NaN and Infinity values we either raise an error or rely on Float#to_s. */ if (!allow_nan) { if (data->state->strict && data->state->as_json) { - VALUE casted_obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil); + VALUE casted_obj = json_call_as_json(data->state, obj, Qfalse); if (casted_obj != obj) { increase_depth(data); generate_json(buffer, data, casted_obj); @@ -1416,7 +1424,7 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALU general: if (data->state->strict) { if (RTEST(data->state->as_json) && !as_json_called) { - obj = rb_proc_call_with_block(data->state->as_json, 1, &obj, Qnil); + obj = json_call_as_json(data->state, obj, Qfalse); as_json_called = true; goto start; } else { @@ -1942,6 +1950,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_allow_duplicate_key) { state->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_as_json) { VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; + state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; state_write_value(data, &state->as_json, proc); } return ST_CONTINUE; diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index c8452ade6..be0e82579 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -396,7 +396,7 @@ static void generateFloat(ThreadContext context, Session session, RubyFloat obje if (!state.allowNaN()) { if (state.strict() && state.getAsJSON() != null) { - IRubyObject castedValue = state.getAsJSON().call(context, object); + IRubyObject castedValue = state.getAsJSON().call(context, object, context.getRuntime().getFalse()); if (castedValue != object) { getHandlerFor(context.runtime, castedValue).generate(context, session, castedValue, buffer); return; @@ -623,7 +623,7 @@ private static void processEntry(ThreadContext context, Session session, OutputS GeneratorState state = session.getState(context); if (state.strict()) { if (state.getAsJSON() != null) { - key = state.getAsJSON().call(context, key); + key = state.getAsJSON().call(context, key, context.getRuntime().getTrue()); keyStr = castKey(context, key); } @@ -760,7 +760,7 @@ static RubyString generateGenericNew(ThreadContext context, Session session, IRu GeneratorState state = session.getState(context); if (state.strict()) { if (state.getAsJSON() != null) { - IRubyObject value = state.getAsJSON().call(context, object); + IRubyObject value = state.getAsJSON().call(context, object, context.getRuntime().getFalse()); Handler handler = getHandlerFor(context.runtime, value); if (handler == GENERIC_HANDLER) { throw Utils.buildGeneratorError(context, object, value + " returned by as_json not allowed in JSON").toThrowable(); diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 8be312f7a..0e935d50a 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -450,7 +450,7 @@ def to_json(state = nil, *) value = self if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) if state.as_json - value = state.as_json.call(value) + value = state.as_json.call(value, false) unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end @@ -511,7 +511,7 @@ def json_transform(state) if state.strict? && !(Symbol === key || String === key) if state.as_json - key = state.as_json.call(key) + key = state.as_json.call(key, true) end unless Symbol === key || String === key @@ -529,7 +529,7 @@ def json_transform(state) result = +"#{result}#{key_json}#{state.space_before}:#{state.space}" if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) if state.as_json - value = state.as_json.call(value) + value = state.as_json.call(value, false) unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end @@ -590,7 +590,7 @@ def json_transform(state) result << state.indent * depth if indent if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value || Symbol == value) if state.as_json - value = state.as_json.call(value) + value = state.as_json.call(value, false) unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value || Symbol === value raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end @@ -625,7 +625,7 @@ def to_json(state = nil, *args) if state.allow_nan? to_s elsif state.strict? && state.as_json - casted_value = state.as_json.call(self) + casted_value = state.as_json.call(self, false) if casted_value.equal?(self) raise GeneratorError.new("#{self} not allowed in JSON", self) diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index fc4aba296..c72483537 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -12,7 +12,8 @@ def test_json_coder_with_proc end def test_json_coder_with_proc_with_unsupported_value - coder = JSON::Coder.new do |object| + coder = JSON::Coder.new do |object, is_key| + assert_equal false, is_key Object.new end assert_raise(JSON::GeneratorError) { coder.dump([Object.new]) } @@ -20,7 +21,10 @@ def test_json_coder_with_proc_with_unsupported_value def test_json_coder_hash_key obj = Object.new - coder = JSON::Coder.new(&:to_s) + coder = JSON::Coder.new do |obj, is_key| + assert_equal true, is_key + obj.to_s + end assert_equal %({#{obj.to_s.inspect}:1}), coder.dump({ obj => 1 }) coder = JSON::Coder.new { 42 } @@ -49,14 +53,14 @@ def test_json_coder_load_options end def test_json_coder_dump_NaN_or_Infinity - coder = JSON::Coder.new(&:inspect) + coder = JSON::Coder.new { |o| o.inspect } assert_equal "NaN", coder.load(coder.dump(Float::NAN)) assert_equal "Infinity", coder.load(coder.dump(Float::INFINITY)) assert_equal "-Infinity", coder.load(coder.dump(-Float::INFINITY)) end def test_json_coder_dump_NaN_or_Infinity_loop - coder = JSON::Coder.new(&:itself) + coder = JSON::Coder.new { |o| o.itself } error = assert_raise JSON::GeneratorError do coder.dump(Float::NAN) end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 4fdfa12b0..a6950f888 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -822,7 +822,7 @@ def test_fragment def test_json_generate_as_json_convert_to_proc object = Object.new - assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: :object_id) + assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: -> (o, is_key) { o.object_id }) end def assert_float_roundtrip(expected, actual) From 4d9068c1a6bee90235dd4aa8bdfb7c4416688de6 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Sep 2025 21:31:40 +0200 Subject: [PATCH 160/506] Refactor Truffle generator type checks --- lib/json/truffle_ruby/generator.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 0e935d50a..2d01ad1b5 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -47,6 +47,14 @@ module Generator SCRIPT_SAFE_ESCAPE_PATTERN = /[\/"\\\x0-\x1f\u2028-\u2029]/ + def self.native_type?(value) # :nodoc: + (false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) + end + + def self.native_key?(key) # :nodoc: + (Symbol === key || String === key) + end + # Convert a UTF8 encoded Ruby string _string_ to a JSON string, encoded with # UTF16 big endian characters as \u????, and return it. def self.utf8_to_json(string, script_safe = false) # :nodoc: @@ -448,10 +456,10 @@ def to_json(state = nil, *) state = State.from_state(state) if state if state&.strict? value = self - if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) + if state.strict? && !Generator.native_type?(value) if state.as_json value = state.as_json.call(value, false) - unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value + unless Generator.native_type?(value) raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end value.to_json(state) @@ -509,12 +517,12 @@ def json_transform(state) end result << state.indent * depth if indent - if state.strict? && !(Symbol === key || String === key) + if state.strict? && !Generator.native_key?(key) if state.as_json key = state.as_json.call(key, true) end - unless Symbol === key || String === key + unless Generator.native_key?(key) raise GeneratorError.new("#{key.class} not allowed as object key in JSON", value) end end @@ -527,10 +535,10 @@ def json_transform(state) end result = +"#{result}#{key_json}#{state.space_before}:#{state.space}" - if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) + if state.strict? && !Generator.native_type?(value) if state.as_json value = state.as_json.call(value, false) - unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value + unless Generator.native_type?(value) raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end result << value.to_json(state) @@ -588,10 +596,10 @@ def json_transform(state) each { |value| result << delim unless first result << state.indent * depth if indent - if state.strict? && !(false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value || Symbol == value) + if state.strict? && !Generator.native_type?(value) if state.as_json value = state.as_json.call(value, false) - unless false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value || Symbol === value + unless Generator.native_type?(value) raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end result << value.to_json(state) From efd67e7dfd32ba31cfe0c764a4825816f0e56a28 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Sep 2025 00:05:04 +0200 Subject: [PATCH 161/506] Update changelog --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 33929954f..60273d26e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,7 @@ * Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. * Fix `JSON.unsafe_load` usage with proc * Fix the parser to more consistently reject invalid UTF-16 surogate pairs. +* Stop defining `String.json_create`, `String#to_json_raw`, `String#to_json_raw_object` when `json/add` isn't loaded. ### 2025-07-28 (2.13.2) From 0a9478a715ea766d627ff3a72cf7a4f033fe536f Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 18 Sep 2025 21:15:40 -0500 Subject: [PATCH 162/506] implement a better fix for an out of bounds exception --- java/src/json/ext/SWARBasicStringEncoder.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java index 08cd2e473..02a3bfcde 100644 --- a/java/src/json/ext/SWARBasicStringEncoder.java +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -23,9 +23,13 @@ void encode(ByteList src) throws IOException { int beg = 0; int pos = 0; - ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, len); - while (ptr + pos + 8 <= len) { - long x = bb.getLong(ptr + pos); + // There are optimizations in JRuby where ptr > 0 will be true. For example, if we + // slice a string, the underlying byte array is the same, but the + // begin index and real size are different. When reading from the ptrBytes + // array, we need to always add ptr to the index. + ByteBuffer bb = ByteBuffer.wrap(ptrBytes, ptr, len); + while (pos + 8 <= len) { + long x = bb.getLong(pos); if (skipChunk(x)) { pos += 8; continue; @@ -43,8 +47,8 @@ void encode(ByteList src) throws IOException { } } - if (ptr + pos + 4 <= len) { - int x = bb.getInt(ptr + pos); + if (pos + 4 <= len) { + int x = bb.getInt(pos); if (skipChunk(x)) { pos += 4; } From a2aa1cb3d0de7b7a81253191e9e7e0baa39f0bf4 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Sat, 20 Sep 2025 04:25:34 +0530 Subject: [PATCH 163/506] Add JRuby build output to CLEAN and CLOBBER --- Rakefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Rakefile b/Rakefile index c1768853c..64a440db7 100644 --- a/Rakefile +++ b/Rakefile @@ -91,6 +91,12 @@ JAVA_CLASSES = [] JRUBY_PARSER_JAR = File.expand_path("lib/json/ext/parser.jar") JRUBY_GENERATOR_JAR = File.expand_path("lib/json/ext/generator.jar") +CLEAN.concat FileList["java/src/**/*.class"] +CLEAN << JRUBY_PARSER_JAR +CLEAN << JRUBY_GENERATOR_JAR + +CLOBBER << JAVA_PARSER_SRC + which = lambda { |c| w = `which #{c}` break w.chomp unless w.empty? From b1cc12605b5b5331b9c6c36774c37464d694b6cd Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Sat, 20 Sep 2025 04:26:03 +0530 Subject: [PATCH 164/506] Use --release flag on Java 9+ The check for the modules dir here indicates whether we are on JDK 9+, since 1.8 did not have modules. Fixes #862. See #859 for some background. --- Rakefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index 64a440db7..9681be0d7 100644 --- a/Rakefile +++ b/Rakefile @@ -146,7 +146,11 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator obj = src.sub(/\.java\Z/, '.class') file obj => src do - sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src + if File.exist?(File.join(ENV['JAVA_HOME'], "lib", "modules")) + sh 'javac', '-classpath', classpath, '--release', '8', src + else + sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src + end end JAVA_CLASSES << obj end From 3d1ed18b65c7581874ef65a7e197ef7b80b5ae49 Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Sat, 20 Sep 2025 04:37:36 +0530 Subject: [PATCH 165/506] Add macos platform for JRuby CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4070118b..b9b6bd96e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } - { os: macos-13, ruby: 3.4 } - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI + - { os: macos-latest , ruby: jruby-9.4 } # Ruby 3.1 - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 - { os: macos-latest , ruby: truffleruby-head } - { os: ubuntu-latest , ruby: truffleruby-head } From 4abfad090d8398bfd30b4fcb253595151eb7f397 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 22 Sep 2025 12:02:24 +0200 Subject: [PATCH 166/506] Release 2.15.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 60273d26e..747b8059b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-09-22 (2.15.0) + * `JSON::Coder` callback now receive a second argument to convey whether the object is a hash key. * Tuned the floating point number generator to not use scientific notation as agressively. diff --git a/lib/json/version.rb b/lib/json/version.rb index 9c928e394..e2db923e3 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.14.1' + VERSION = '2.15.0' end From ec858510effa1f5ed4c409854f2280c51f1d07bd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 22 Sep 2025 12:05:51 +0200 Subject: [PATCH 167/506] Fix a typo in the changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 747b8059b..c161547d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ ### 2025-09-22 (2.15.0) * `JSON::Coder` callback now receive a second argument to convey whether the object is a hash key. -* Tuned the floating point number generator to not use scientific notation as agressively. +* Tuned the floating point number generator to not use scientific notation as aggressively. ### 2025-09-18 (2.14.1) From d867e393bbbd2c50261e747296c1ceb232ad60bd Mon Sep 17 00:00:00 2001 From: Charles Oliver Nutter Date: Fri, 3 Oct 2025 12:05:58 -0500 Subject: [PATCH 168/506] Run jruby-head on Windows Last few failures on Windows caused by jruby/jruby#9018 were fixed in jruby/ruby#9019. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9b6bd96e..c8c08f196 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,6 @@ jobs: - { os: ubuntu-latest , ruby: truffleruby-head } exclude: - { os: windows-latest, ruby: jruby } - - { os: windows-latest, ruby: jruby-head } steps: - uses: actions/checkout@v5 From d7baf015d98c30a71597934508d6728d66752d6c Mon Sep 17 00:00:00 2001 From: tompng Date: Sun, 5 Oct 2025 02:26:46 +0900 Subject: [PATCH 169/506] Fix sliced string escaping --- java/src/json/ext/SWARBasicStringEncoder.java | 7 ++++--- test/json/json_encoding_test.rb | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java index 02a3bfcde..a6695d99c 100644 --- a/java/src/json/ext/SWARBasicStringEncoder.java +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -27,9 +27,10 @@ void encode(ByteList src) throws IOException { // slice a string, the underlying byte array is the same, but the // begin index and real size are different. When reading from the ptrBytes // array, we need to always add ptr to the index. - ByteBuffer bb = ByteBuffer.wrap(ptrBytes, ptr, len); + ByteBuffer bb = ByteBuffer.wrap(ptrBytes, 0, ptr + len); + while (pos + 8 <= len) { - long x = bb.getLong(pos); + long x = bb.getLong(ptr + pos); if (skipChunk(x)) { pos += 8; continue; @@ -48,7 +49,7 @@ void encode(ByteList src) throws IOException { } if (pos + 4 <= len) { - int x = bb.getInt(pos); + int x = bb.getInt(ptr + pos); if (skipChunk(x)) { pos += 4; } diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index 8dce81da5..2789e94b5 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -35,6 +35,8 @@ def test_generate_shared_string # Ref: https://github.com/ruby/json/issues/859 s = "01234567890" assert_equal '"234567890"', JSON.dump(s[2..-1]) + s = '01234567890123456789"a"b"c"d"e"f"g"h' + assert_equal '"\"a\"b\"c\"d\"e\"f\"g\""', JSON.dump(s[20, 15]) end def test_unicode From 1b1647fb2cecf848b5c2630782846875a707532e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 5 Oct 2025 09:48:48 +0200 Subject: [PATCH 170/506] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index c161547d7..bc8016d6b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix incorrect escaping in the JRuby extension when encoding shared strings. + ### 2025-09-22 (2.15.0) * `JSON::Coder` callback now receive a second argument to convey whether the object is a hash key. From 1e19097a71bb12ede5a4683581298246f73b9242 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 6 Oct 2025 16:48:41 -0700 Subject: [PATCH 171/506] Add a workflow to sync commits to ruby/ruby (#872) --- .github/workflows/sync-ruby.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/sync-ruby.yml diff --git a/.github/workflows/sync-ruby.yml b/.github/workflows/sync-ruby.yml new file mode 100644 index 000000000..29c434491 --- /dev/null +++ b/.github/workflows/sync-ruby.yml @@ -0,0 +1,33 @@ +name: Sync ruby +on: + push: + branches: [master] +jobs: + sync: + name: Sync ruby + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'ruby' }} + steps: + - uses: actions/checkout@v5 + + - name: Create GitHub App token + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: 2060836 + private-key: ${{ secrets.RUBY_SYNC_DEFAULT_GEMS_PRIVATE_KEY }} + owner: ruby + repositories: ruby + + - name: Sync to ruby/ruby + uses: convictional/trigger-workflow-and-wait@v1.6.5 + with: + owner: ruby + repo: ruby + workflow_file_name: sync_default_gems.yml + github_token: ${{ steps.app-token.outputs.token }} + ref: master + client_payload: | + {"gem":"${{ github.event.repository.name }}","before":"${{ github.event.before }}","after":"${{ github.event.after }}"} + propagate_failure: true + wait_interval: 10 From 9e6067bb55d60d893fb95dafa777af880459c7c2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 7 Oct 2025 10:30:28 +0200 Subject: [PATCH 172/506] Release 2.15.1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index bc8016d6b..1f0b78733 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-10-07 (2.15.1) + * Fix incorrect escaping in the JRuby extension when encoding shared strings. ### 2025-09-22 (2.15.0) diff --git a/lib/json/version.rb b/lib/json/version.rb index e2db923e3..be5daf4c5 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.15.0' + VERSION = '2.15.1' end From aefa671eca2d514bfa685f385c5fa3e1a13305f4 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 25 Oct 2025 10:31:04 +0200 Subject: [PATCH 173/506] Fix concurrent usage of JSON::Coder#dump Fix: https://github.com/rails/rails/commit/90616277e3d8fc46c9cf35d6a7470ff1ea0092f7#r168784389 Because the `depth` counter is inside `JSON::State` it can't be used concurrently, and in case of a circular reference the counter may be left at the max value. The depth counter should be moved outside `JSON_Generator_State` and into `struct generate_json_data`, but it's a larger refactor. In the meantime, `JSON::Coder` calls `State#generate_new` so I changed that method so that it first copy the state on the stack. --- CHANGES.md | 3 +++ ext/json/ext/generator/generator.c | 39 +++++++++++++++++++++++---- java/src/json/ext/GeneratorState.java | 24 ++++++++++++++--- lib/json/truffle_ruby/generator.rb | 6 ++++- test/json/json_coder_test.rb | 10 +++++++ 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1f0b78733..4434d815c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +* Fix `JSON::Coder` to have one dedicated depth counter per invocation. + After encountering a circular reference in `JSON::Coder#dump`, any further `#dump` call would raise `JSON::NestingError`. + ### 2025-10-07 (2.15.1) * Fix incorrect escaping in the JRuby extension when encoding shared strings. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 6a38cc60a..f3f27b29d 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1124,7 +1124,7 @@ static inline long increase_depth(struct generate_json_data *data) JSON_Generator_State *state = data->state; long depth = ++state->depth; if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) { - rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); + rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", --state->depth); } return depth; } @@ -1491,10 +1491,39 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self) rb_check_arity(argc, 1, 2); VALUE obj = argv[0]; VALUE io = argc > 1 ? argv[1] : Qnil; - VALUE result = cState_partial_generate(self, obj, generate_json, io); + return cState_partial_generate(self, obj, generate_json, io); +} + +static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) +{ + rb_check_arity(argc, 1, 2); + VALUE obj = argv[0]; + VALUE io = argc > 1 ? argv[1] : Qnil; + GET_STATE(self); - (void)state; - return result; + + JSON_Generator_State new_state; + MEMCPY(&new_state, state, JSON_Generator_State, 1); + + // FIXME: depth shouldn't be part of JSON_Generator_State, as that prevents it from being used concurrently. + new_state.depth = 0; + + char stack_buffer[FBUFFER_STACK_SIZE]; + FBuffer buffer = { + .io = RTEST(io) ? io : Qfalse, + }; + fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE); + + struct generate_json_data data = { + .buffer = &buffer, + .vstate = Qfalse, + .state = &new_state, + .obj = obj, + .func = generate_json + }; + rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); + + return fbuffer_finalize(&buffer); } static VALUE cState_initialize(int argc, VALUE *argv, VALUE self) @@ -2072,7 +2101,7 @@ void Init_generator(void) rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0); rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1); rb_define_method(cState, "generate", cState_generate, -1); - rb_define_alias(cState, "generate_new", "generate"); // :nodoc: + rb_define_method(cState, "generate_new", cState_generate_new, -1); // :nodoc: rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index f9e8234e6..fd1478b0e 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -231,7 +231,7 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { * the result. If no valid JSON document can be created this method raises * a GeneratorError exception. */ - @JRubyMethod(alias="generate_new") + @JRubyMethod public IRubyObject generate(ThreadContext context, IRubyObject obj, IRubyObject io) { IRubyObject result = Generator.generateJson(context, obj, this, io); RuntimeInfo info = RuntimeInfo.forRuntime(context.runtime); @@ -251,11 +251,25 @@ public IRubyObject generate(ThreadContext context, IRubyObject obj, IRubyObject return resultString; } - @JRubyMethod(alias="generate_new") + @JRubyMethod public IRubyObject generate(ThreadContext context, IRubyObject obj) { return generate(context, obj, context.nil); } + @JRubyMethod + public IRubyObject generate_new(ThreadContext context, IRubyObject obj, IRubyObject io) { + GeneratorState newState = (GeneratorState)dup(); + newState.resetDepth(); + return newState.generate(context, obj, io); + } + + @JRubyMethod + public IRubyObject generate_new(ThreadContext context, IRubyObject obj) { + GeneratorState newState = (GeneratorState)dup(); + newState.resetDepth(); + return newState.generate(context, obj, context.nil); + } + @JRubyMethod(name="[]") public IRubyObject op_aref(ThreadContext context, IRubyObject vName) { String name = vName.asJavaString(); @@ -478,6 +492,10 @@ public IRubyObject depth_set(IRubyObject vDepth) { return vDepth; } + public void resetDepth() { + depth = 0; + } + private ByteList prepareByteList(ThreadContext context, IRubyObject value) { RubyString str = value.convertToString(); if (str.getEncoding() != UTF8Encoding.INSTANCE) { @@ -610,7 +628,7 @@ public int decreaseDepth() { private void checkMaxNesting(ThreadContext context) { if (maxNesting != 0 && depth > maxNesting) { depth--; - throw Utils.newException(context, Utils.M_NESTING_ERROR, "nesting of " + depth + " is too deep"); + throw Utils.newException(context, Utils.M_NESTING_ERROR, "nesting of " + depth + " is too deep. Did you try to serialize objects with circular references?"); } } } diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 2d01ad1b5..703211cbc 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -212,7 +212,7 @@ def check_max_nesting # :nodoc: return if @max_nesting.zero? current_nesting = depth + 1 current_nesting > @max_nesting and - raise NestingError, "nesting of #{current_nesting} is too deep" + raise NestingError, "nesting of #{current_nesting} is too deep. Did you try to serialize objects with circular references?" end # Returns true, if circular data structures are checked, @@ -347,6 +347,10 @@ def generate_new(obj, anIO = nil) # :nodoc: dup.generate(obj, anIO) end + private def initialize_copy(_orig) + @depth = 0 + end + # Handles @allow_nan, @buffer_initial_length, other ivars must be the default value (see above) private def generate_json(obj, buf) case obj diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index c72483537..fb9d7b30a 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -66,4 +66,14 @@ def test_json_coder_dump_NaN_or_Infinity_loop end assert_include error.message, "NaN not allowed in JSON" end + + def test_nesting_recovery + coder = JSON::Coder.new + ary = [] + ary << ary + assert_raise JSON::NestingError do + coder.dump(ary) + end + assert_equal '{"a":1}', coder.dump({ a: 1 }) + end end From 5e61cd7dce7ce8748ee0753cd9ef415424abc248 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 25 Oct 2025 10:59:15 +0200 Subject: [PATCH 174/506] Release 2.15.2 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 4434d815c..305515fa6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-10-25 (2.15.2) + * Fix `JSON::Coder` to have one dedicated depth counter per invocation. After encountering a circular reference in `JSON::Coder#dump`, any further `#dump` call would raise `JSON::NestingError`. diff --git a/lib/json/version.rb b/lib/json/version.rb index be5daf4c5..dc01ba290 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.15.1' + VERSION = '2.15.2' end From 1ba1e9bef9707da37459039bdb3e5336e65162cc Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Oct 2025 09:52:58 +0100 Subject: [PATCH 175/506] Use locale indepenent version of `islapha` --- ext/json/ext/parser/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 297031dcf..2522c5622 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -124,7 +124,7 @@ static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const lon return Qfalse; } - if (RB_UNLIKELY(!isalpha((unsigned char)str[0]))) { + if (RB_UNLIKELY(!rb_isalpha((unsigned char)str[0]))) { // Simple heuristic, if the first character isn't a letter, // we're much less likely to see this string again. // We mostly want to cache strings that are likely to be repeated. @@ -176,7 +176,7 @@ static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const lon return Qfalse; } - if (RB_UNLIKELY(!isalpha((unsigned char)str[0]))) { + if (RB_UNLIKELY(!rb_isalpha((unsigned char)str[0]))) { // Simple heuristic, if the first character isn't a letter, // we're much less likely to see this string again. // We mostly want to cache strings that are likely to be repeated. From f228b306359e61e543b2d3db4974e57f88c6f5a5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Oct 2025 10:22:04 +0100 Subject: [PATCH 176/506] parser.c: Fix indentation in json_decode_integer --- ext/json/ext/parser/parser.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2522c5622..538cddb6d 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -790,11 +790,11 @@ static VALUE json_decode_large_integer(const char *start, long len) static inline VALUE json_decode_integer(const char *start, const char *end) { - long len = end - start; - if (RB_LIKELY(len < MAX_FAST_INTEGER_SIZE)) { - return fast_decode_integer(start, end); - } - return json_decode_large_integer(start, len); + long len = end - start; + if (RB_LIKELY(len < MAX_FAST_INTEGER_SIZE)) { + return fast_decode_integer(start, end); + } + return json_decode_large_integer(start, len); } static VALUE json_decode_large_float(const char *start, long len) From d40b2703a87bddc99db43e38a23a4be47400681e Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 28 Oct 2025 08:37:32 +0100 Subject: [PATCH 177/506] Use Vector API in the Java Extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overview This PR uses the [jdk.incubator.vector module](https://docs.oracle.com/en/java/javase/20/docs/api/jdk.incubator.vector/jdk/incubator/vector/package-summary.html) as mentioned in [issue #739](https://github.com/ruby/json/issues/739) to accelerate generating JSON with the same algorithm as the C extension. The PR as it exists right now, it will attempt to build the `json.ext.VectorizedEscapeScanner` class with a target release of `16`. This is the first version of Java with support for the `jdk.incubator.vector` module. The remaining code is built for Java 1.8. The code will attempt to load the `json.ext.VectorizedEscapeScanner` only if the `json.enableVectorizedEscapeScanner` system property is set to `true` (or `1`). I'm not entirely sure how this is packaged / included with JRuby so I'd love @byroot and @headius's (and others?) thought about how to potential package and/or structure the JARs. I did consider adding the `json.ext.VectorizedEscapeScanner` to a separate `generator-vectorized.jar` but I thought I'd solicit feedback before spending any more time on the build / package process. Benchmarks Machine M1 Macbook Air Note: I've had trouble modifying the `compare.rb` I was using for the C extension to work reliability with the Java extension. I'll probably spend more time trying to get it to work, but as of right now these are pretty raw benchmarks. Below are two sample runs of the real-world benchmarks. The benchmarks are much more variable then the C extension for some reason. I'm not sure if HotSpot is doing something slightly different per execution. Vector API Enabled ``` scott@Scotts-MacBook-Air json % ONLY=json JAVA_OPTS='--add-modules jdk.incubator.vector -Djson.enableVectorizedEscapeScanner=true' ruby -I"lib" benchmark/encoder-realworld.rb WARNING: Using incubator modules: jdk.incubator.vector == Encoding activitypub.json (52595 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 1.384k i/100ms Calculating ------------------------------------- json 15.289k (± 0.8%) i/s (65.41 μs/i) - 153.624k in 10.048481s == Encoding citm_catalog.json (500298 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 76.000 i/100ms Calculating ------------------------------------- json 753.787 (± 3.6%) i/s (1.33 ms/i) - 7.524k in 9.997059s == Encoding twitter.json (466906 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 173.000 i/100ms Calculating ------------------------------------- json 1.751k (± 1.1%) i/s (571.24 μs/i) - 17.646k in 10.081260s == Encoding ohai.json (20147 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 2.390k i/100ms Calculating ------------------------------------- json 23.829k (± 0.8%) i/s (41.97 μs/i) - 239.000k in 10.030503s ``` Vector API Disabled ``` scott@Scotts-MacBook-Air json % ONLY=json JAVA_OPTS='--add-modules jdk.incubator.vector -Djson.enableVectorizedEscapeScanner=false' ruby -I"lib" benchmark/encoder-realworld.rb WARNING: Using incubator modules: jdk.incubator.vector VectorizedEscapeScanner disabled. == Encoding activitypub.json (52595 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 1.204k i/100ms Calculating ------------------------------------- json 12.937k (± 1.1%) i/s (77.30 μs/i) - 130.032k in 10.052234s == Encoding citm_catalog.json (500298 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 80.000 i/100ms Calculating ------------------------------------- json 817.378 (± 1.0%) i/s (1.22 ms/i) - 8.240k in 10.082058s == Encoding twitter.json (466906 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 147.000 i/100ms Calculating ------------------------------------- json 1.499k (± 1.3%) i/s (667.08 μs/i) - 14.994k in 10.004181s == Encoding ohai.json (20147 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 2.269k i/100ms Calculating ------------------------------------- json 22.366k (± 5.7%) i/s (44.71 μs/i) - 224.631k in 10.097069s ``` `master` as of commit `c5af1b68c582335c2a82bbc4bfa5b3e41ead1eba` ``` scott@Scotts-MacBook-Air json % ONLY=json ruby -I"lib" benchmark/encoder-realworld.rb == Encoding activitypub.json (52595 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 886.000 i/100ms Calculating ------------------------------------- json^C% scott@Scotts-MacBook-Air json % ONLY=json ruby -I"lib" benchmark/encoder-realworld.rb == Encoding activitypub.json (52595 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 1.031k i/100ms Calculating ------------------------------------- json 10.812k (± 1.3%) i/s (92.49 μs/i) - 108.255k in 10.014260s == Encoding citm_catalog.json (500298 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 82.000 i/100ms Calculating ------------------------------------- json 824.921 (± 1.0%) i/s (1.21 ms/i) - 8.282k in 10.040787s == Encoding twitter.json (466906 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 141.000 i/100ms Calculating ------------------------------------- json 1.421k (± 0.7%) i/s (703.85 μs/i) - 14.241k in 10.023979s == Encoding ohai.json (20147 bytes) jruby 9.4.12.0 (3.1.4) 2025-02-11 f4ab75096a Java HotSpot(TM) 64-Bit Server VM 21.0.7+8-LTS-245 on 21.0.7+8-LTS-245 +jit [arm64-darwin] Warming up -------------------------------------- json 2.274k i/100ms Calculating ------------------------------------- json 22.612k (± 0.9%) i/s (44.22 μs/i) - 227.400k in 10.057516s ``` Observations `activitypub.json` and `twitter.json` seem to be consistently faster with the Vector API enabled. `citm_catalog.json` seems consistently a bit slower and `ohai.json` is fairly close to even. --- Rakefile | 21 +++- .../AbstractByteListDirectOutputStream.java | 2 - java/src/json/ext/SWARBasicStringEncoder.java | 4 +- java/src/json/ext/StringEncoder.java | 37 ++++++- .../src/json/ext/VectorizedStringEncoder.java | 104 ++++++++++++++++++ test/json/json_encoding_test.rb | 4 + 6 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 java/src/json/ext/VectorizedStringEncoder.java diff --git a/Rakefile b/Rakefile index 9681be0d7..d09e81940 100644 --- a/Rakefile +++ b/Rakefile @@ -86,7 +86,8 @@ end JAVA_DIR = "java/src/json/ext" JAVA_RAGEL_PATH = "#{JAVA_DIR}/ParserConfig.rl" JAVA_PARSER_SRC = "#{JAVA_DIR}/ParserConfig.java" -JAVA_SOURCES = FileList["#{JAVA_DIR}/*.java"] +JAVA_SOURCES = FileList["#{JAVA_DIR}/*.java"].exclude("#{JAVA_DIR}/Vectorized*.java") +JAVA_VEC_SOURCES = FileList["#{JAVA_DIR}/Vectorized*.java"] JAVA_CLASSES = [] JRUBY_PARSER_JAR = File.expand_path("lib/json/ext/parser.jar") JRUBY_GENERATOR_JAR = File.expand_path("lib/json/ext/generator.jar") @@ -142,8 +143,8 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' JRUBY_JAR = File.join(CONFIG["libdir"], "jruby.jar") if File.exist?(JRUBY_JAR) + classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator JAVA_SOURCES.each do |src| - classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator obj = src.sub(/\.java\Z/, '.class') file obj => src do if File.exist?(File.join(ENV['JAVA_HOME'], "lib", "modules")) @@ -154,6 +155,20 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' end JAVA_CLASSES << obj end + + JAVA_VEC_SOURCES.each do |src| + obj = src.sub(/\.java\Z/, '.class') + file obj => src do + sh 'javac', '--add-modules', 'jdk.incubator.vector', '-classpath', classpath, '--release', '16', src do |success, status| + if success + puts "*** 'jdk.incubator.vector' support enabled ***" + else + puts "*** 'jdk.incubator.vector' support disabled ***" + end + end + end + JAVA_CLASSES << obj + end else warn "WARNING: Cannot find jruby in path => Cannot build jruby extension!" end @@ -199,11 +214,13 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' generator_classes = FileList[ "json/ext/*ByteList*.class", "json/ext/OptionsReader*.class", + "json/ext/EscapeScanner*.class", "json/ext/Generator*.class", "json/ext/RuntimeInfo*.class", "json/ext/*StringEncoder*.class", "json/ext/Utils*.class" ] + puts "Creating generator jar with classes: #{generator_classes.join(', ')}" sh 'jar', 'cf', File.basename(JRUBY_GENERATOR_JAR), *generator_classes mv File.basename(JRUBY_GENERATOR_JAR), File.dirname(JRUBY_GENERATOR_JAR) end diff --git a/java/src/json/ext/AbstractByteListDirectOutputStream.java b/java/src/json/ext/AbstractByteListDirectOutputStream.java index c3175c849..b94d7f59a 100644 --- a/java/src/json/ext/AbstractByteListDirectOutputStream.java +++ b/java/src/json/ext/AbstractByteListDirectOutputStream.java @@ -15,8 +15,6 @@ abstract class AbstractByteListDirectOutputStream extends OutputStream { static { String useSegmentedOutputStream = System.getProperty(PROP_SEGMENTED_BUFFER, PROP_SEGMENTED_BUFFER_DEFAULT); USE_SEGMENTED_BUFFER = Boolean.parseBoolean(useSegmentedOutputStream); - // XXX Is there a logger we can use here? - // System.out.println("Using segmented output stream: " + USE_SEGMENTED_BUFFER); } public static AbstractByteListDirectOutputStream create(int estimatedSize) { diff --git a/java/src/json/ext/SWARBasicStringEncoder.java b/java/src/json/ext/SWARBasicStringEncoder.java index a6695d99c..bd0d1a70f 100644 --- a/java/src/json/ext/SWARBasicStringEncoder.java +++ b/java/src/json/ext/SWARBasicStringEncoder.java @@ -71,7 +71,7 @@ void encode(ByteList src) throws IOException { } } - private boolean skipChunk(long x) { + boolean skipChunk(long x) { long is_ascii = 0x8080808080808080L & ~x; long xor2 = x ^ 0x0202020202020202L; long lt32_or_eq34 = xor2 - 0x2121212121212121L; @@ -80,7 +80,7 @@ private boolean skipChunk(long x) { return ((lt32_or_eq34 | eq92) & is_ascii) == 0; } - private boolean skipChunk(int x) { + boolean skipChunk(int x) { int is_ascii = 0x80808080 & ~x; int xor2 = x ^ 0x02020202; int lt32_or_eq34 = xor2 - 0x21212121; diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index 7f75476d9..b874ad785 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -7,6 +7,8 @@ import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import org.jcodings.Encoding; @@ -114,15 +116,37 @@ class StringEncoder extends ByteListTranscoder { protected final byte[] escapeTable; + private static final String VECTORIZED_STRING_ENCODER_CLASS = "json.ext.VectorizedStringEncoder"; + private static final String USE_VECTORIZED_BASIC_ENCODER_PROP = "jruby.json.useVectorizedBasicEncoder"; + private static final String USE_VECTORIZED_BASIC_ENCODER_DEFAULT = "false"; + private static final boolean USE_VECTORIZED_BASIC_ENCODER; + private static final StringEncoder VECTORIZED_SCANNER; + private static final String USE_SWAR_BASIC_ENCODER_PROP = "jruby.json.useSWARBasicEncoder"; private static final String USE_SWAR_BASIC_ENCODER_DEFAULT = "true"; private static final boolean USE_BASIC_SWAR_ENCODER; static { + String enableVectorizedScanner = System.getProperty(USE_VECTORIZED_BASIC_ENCODER_PROP, USE_VECTORIZED_BASIC_ENCODER_DEFAULT); + if ("true".equalsIgnoreCase(enableVectorizedScanner) || "1".equalsIgnoreCase(enableVectorizedScanner)) { + StringEncoder scanner; + try { + Class vectorizedStringEncoderClass = StringEncoder.class.getClassLoader().loadClass(VECTORIZED_STRING_ENCODER_CLASS); + Constructor vectorizedStringEncoderConstructor = vectorizedStringEncoderClass.getDeclaredConstructor(); + scanner = (StringEncoder) vectorizedStringEncoderConstructor.newInstance(); + } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { + // Fallback to the StringEncoder if we cannot load the VectorizedStringEncoder. + scanner = null; + } + VECTORIZED_SCANNER = scanner; + USE_VECTORIZED_BASIC_ENCODER = scanner != null; + } else { + VECTORIZED_SCANNER = null; + USE_VECTORIZED_BASIC_ENCODER = false; + } + USE_BASIC_SWAR_ENCODER = Boolean.parseBoolean( System.getProperty(USE_SWAR_BASIC_ENCODER_PROP, USE_SWAR_BASIC_ENCODER_DEFAULT)); - // XXX Is there a logger we can use here? - // System.out.println("Using SWAR basic encoder: " + USE_BASIC_SWAR_ENCODER); } OutputStream out; @@ -149,8 +173,15 @@ class StringEncoder extends ByteListTranscoder { this.escapeTable = escapeTable; } + @Override + public StringEncoder clone() { + return new StringEncoder(escapeTable); + } + static StringEncoder createBasicEncoder() { - if (USE_BASIC_SWAR_ENCODER) { + if (USE_VECTORIZED_BASIC_ENCODER) { + return (StringEncoder) VECTORIZED_SCANNER.clone(); + } else if (USE_BASIC_SWAR_ENCODER) { return new SWARBasicStringEncoder(); } else { return new StringEncoder(false); diff --git a/java/src/json/ext/VectorizedStringEncoder.java b/java/src/json/ext/VectorizedStringEncoder.java new file mode 100644 index 000000000..14b3c8d7c --- /dev/null +++ b/java/src/json/ext/VectorizedStringEncoder.java @@ -0,0 +1,104 @@ +package json.ext; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.jruby.util.ByteList; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.VectorMask; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +class VectorizedStringEncoder extends SWARBasicStringEncoder { + private static final VectorSpecies SP = ByteVector.SPECIES_PREFERRED; + private static final ByteVector ZERO = ByteVector.zero(SP); + private static final ByteVector TWO = ByteVector.broadcast(SP, 2); + private static final ByteVector THIRTY_THREE = ByteVector.broadcast(SP, 33); + private static final ByteVector BACKSLASH = ByteVector.broadcast(SP, '\\'); + + @Override + public StringEncoder clone() { + return new VectorizedStringEncoder(); + } + + @Override + void encode(ByteList src) throws IOException { + byte[] ptrBytes = src.unsafeBytes(); + int ptr = src.begin(); + int len = src.realSize(); + int beg = 0; + int pos = ptr; + + while ((pos + SP.length() <= len)) { + ByteVector chunk = ByteVector.fromArray(SP, ptrBytes, ptr + pos); + // bytes are signed in java, so we need to remove negative values + VectorMask negative = chunk.lt(ZERO); + VectorMask tooLowOrDblQuote = chunk.lanewise(VectorOperators.XOR, TWO).lt(THIRTY_THREE).andNot(negative); + VectorMask needsEscape = chunk.eq(BACKSLASH).or(tooLowOrDblQuote); + if (needsEscape.anyTrue()) { + int chunkStart = pos; + long mask = needsEscape.toLong(); + + while(mask > 0) { + // nextMatch inlined + int index = Long.numberOfTrailingZeros(mask); + mask &= (mask - 1); + pos = chunkStart + index; + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, aux, HEX); + } + + // Skip over any remaining characters in the current chunk + pos = chunkStart + SP.length(); + continue; + } + + pos += SP.length(); + } + + ByteBuffer bb = ByteBuffer.wrap(ptrBytes, ptr, len); + if (pos + 8 <= len) { + long x = bb.getLong(ptr + pos); + if (skipChunk(x)) { + pos += 8; + } else { + int chunkEnd = ptr + pos + 8; + while (pos < chunkEnd) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, aux, HEX); + } else { + pos++; + } + } + } + } + + if (pos + 4 <= len) { + int x = bb.getInt(ptr + pos); + if (skipChunk(x)) { + pos += 4; + } + } + + while (pos < len) { + int ch = Byte.toUnsignedInt(ptrBytes[ptr + pos]); + int ch_len = ESCAPE_TABLE[ch]; + if (ch_len > 0) { + beg = pos = flushPos(pos, beg, ptrBytes, ptr, 1); + escapeAscii(ch, aux, HEX); + } else { + pos++; + } + } + + if (beg < len) { + append(ptrBytes, ptr + beg, len - beg); + } + } +} diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index 2789e94b5..7ac06b2a7 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -37,6 +37,10 @@ def test_generate_shared_string assert_equal '"234567890"', JSON.dump(s[2..-1]) s = '01234567890123456789"a"b"c"d"e"f"g"h' assert_equal '"\"a\"b\"c\"d\"e\"f\"g\""', JSON.dump(s[20, 15]) + s = "0123456789001234567890012345678900123456789001234567890" + assert_equal '"23456789001234567890012345678900123456789001234567890"', JSON.dump(s[2..-1]) + s = "0123456789001234567890012345678900123456789001234567890" + assert_equal '"567890012345678900123456789001234567890012345678"', JSON.dump(s[5..-3]) end def test_unicode From 965ba6c5d407b4306d2a8ec41a0af0746687d9df Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 30 Oct 2025 11:33:26 +0100 Subject: [PATCH 178/506] Fix GeneratorError messages to be UTF-8 encoded --- ext/json/ext/generator/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index f3f27b29d..d04c8a907 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -90,6 +90,7 @@ RBIMPL_ATTR_NORETURN() #endif static void raise_generator_error_str(VALUE invalid_object, VALUE str) { + rb_enc_associate_index(str, utf8_encindex); VALUE exc = rb_exc_new_str(eGeneratorError, str); rb_ivar_set(exc, rb_intern("@invalid_object"), invalid_object); rb_exc_raise(exc); From 9c4db3190817f3ebcadc0961f8f104dce6a599ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Sun, 26 Oct 2025 21:09:06 +0100 Subject: [PATCH 179/506] Add ryu float parser. Co-Authored-By: Jean Boussier --- CHANGES.md | 2 + LEGAL | 12 + benchmark/parser.rb | 5 - ext/json/ext/parser/parser.c | 141 +++-- ext/json/ext/vendor/ryu.h | 819 ++++++++++++++++++++++++++++ test/json/json_ryu_fallback_test.rb | 169 ++++++ 6 files changed, 1091 insertions(+), 57 deletions(-) create mode 100644 ext/json/ext/vendor/ryu.h create mode 100644 test/json/json_ryu_fallback_test.rb diff --git a/CHANGES.md b/CHANGES.md index 305515fa6..858792f52 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Optimized floating point number parsing by integrating the ryu algorithm. + ### 2025-10-25 (2.15.2) * Fix `JSON::Coder` to have one dedicated depth counter per invocation. diff --git a/LEGAL b/LEGAL index 737d18cb8..126ccc000 100644 --- a/LEGAL +++ b/LEGAL @@ -6,3 +6,15 @@ All the files in this distribution are covered under either the Ruby's license (see the file COPYING) or public-domain except some files mentioned below. + +ext/json/ext/vendor/fpconv.h:: + This files is adapted from https://github.com/night-shift/fpconv + It is licensed under Boost Software License 1.0. + +ext/json/ext/vendor/jeaiii-ltoa.h:: + This files is adapted from https://github.com/jeaiii/itoa + It is licensed under the MIT License + +ext/json/ext/vendor/ryu.h:: + This file is adapted from the Ryu algorithm by Ulf Adams https://github.com/ulfjack/ryu. + It is dual-licensed under Apache License 2.0 OR Boost Software License 1.0. diff --git a/benchmark/parser.rb b/benchmark/parser.rb index a2bd17efb..71c527ea4 100644 --- a/benchmark/parser.rb +++ b/benchmark/parser.rb @@ -57,9 +57,4 @@ def benchmark_parsing(name, json_output) benchmark_parsing "activitypub.json", File.read("#{__dir__}/data/activitypub.json") benchmark_parsing "twitter.json", File.read("#{__dir__}/data/twitter.json") benchmark_parsing "citm_catalog.json", File.read("#{__dir__}/data/citm_catalog.json") - -# rapidjson is 8x faster thanks to its much more performant float parser. -# Unfortunately, there isn't a lot of existing fast float parsers in pure C, -# and including C++ is problematic. -# Aside from that, we're close to the alternatives here. benchmark_parsing "float parsing", File.read("#{__dir__}/data/canada.json") diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 538cddb6d..ca8f50153 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1,5 +1,6 @@ #include "ruby.h" #include "ruby/encoding.h" +#include "../vendor/ryu.h" /* shims */ /* This is the fallback definition from Ruby 3.4 */ @@ -20,6 +21,16 @@ typedef unsigned char _Bool; #endif #endif +#if SIZEOF_UINT64_T == SIZEOF_LONG_LONG +# define INT64T2NUM(x) LL2NUM(x) +# define UINT64T2NUM(x) ULL2NUM(x) +#elif SIZEOF_UINT64_T == SIZEOF_LONG +# define INT64T2NUM(x) LONG2NUM(x) +# define UINT64T2NUM(x) ULONG2NUM(x) +#else +# error No uint64_t conversion +#endif + #include "../simd/simd.h" #ifndef RB_UNLIKELY @@ -755,26 +766,6 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c } #define MAX_FAST_INTEGER_SIZE 18 -static inline VALUE fast_decode_integer(const char *p, const char *pe) -{ - bool negative = false; - if (*p == '-') { - negative = true; - p++; - } - - long long memo = 0; - while (p < pe) { - memo *= 10; - memo += *p - '0'; - p++; - } - - if (negative) { - memo = -memo; - } - return LL2NUM(memo); -} static VALUE json_decode_large_integer(const char *start, long len) { @@ -788,17 +779,27 @@ static VALUE json_decode_large_integer(const char *start, long len) } static inline VALUE -json_decode_integer(const char *start, const char *end) +json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end) { - long len = end - start; - if (RB_LIKELY(len < MAX_FAST_INTEGER_SIZE)) { - return fast_decode_integer(start, end); + if (RB_LIKELY(mantissa_digits < MAX_FAST_INTEGER_SIZE)) { + if (negative) { + return INT64T2NUM(-((int64_t)mantissa)); + } + return UINT64T2NUM(mantissa); } - return json_decode_large_integer(start, len); + + return json_decode_large_integer(start, end - start); } static VALUE json_decode_large_float(const char *start, long len) { + if (RB_LIKELY(len < 64)) { + char buffer[64]; + MEMCPY(buffer, start, char, len); + buffer[len] = '\0'; + return DBL2NUM(rb_cstr_to_dbl(buffer, 1)); + } + VALUE buffer_v; char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1); MEMCPY(buffer, start, char, len); @@ -808,21 +809,24 @@ static VALUE json_decode_large_float(const char *start, long len) return number; } -static VALUE json_decode_float(JSON_ParserConfig *config, const char *start, const char *end) +/* Ruby JSON optimized float decoder using vendored Ryu algorithm + * Accepts pre-extracted mantissa and exponent from first-pass validation + */ +static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int32_t exponent, bool negative, + const char *start, const char *end) { - long len = end - start; - if (RB_UNLIKELY(config->decimal_class)) { - VALUE text = rb_str_new(start, len); + VALUE text = rb_str_new(start, end - start); return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text); - } else if (RB_LIKELY(len < 64)) { - char buffer[64]; - MEMCPY(buffer, start, char, len); - buffer[len] = '\0'; - return DBL2NUM(rb_cstr_to_dbl(buffer, 1)); - } else { - return json_decode_large_float(start, len); } + + // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case) + // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308) + if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) { + return json_decode_large_float(start, end - start); + } + + return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, exponent, negative)); } static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count) @@ -1082,57 +1086,90 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { bool integer = true; + // Variables for Ryu optimization - extract digits during parsing + uint64_t mantissa = 0; + int mantissa_digits = 0; + int32_t exponent = 0; + bool negative = false; + int decimal_point_pos = -1; + // /\A-?(0|[1-9]\d*)(\.\d+)?([Ee][-+]?\d+)?/ const char *start = state->cursor; - state->cursor++; - while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { + // Handle optional negative sign + if (*state->cursor == '-') { + negative = true; state->cursor++; + if (state->cursor >= state->end || !rb_isdigit(*state->cursor)) { + raise_parse_error_at("invalid number: %s", state, start); + } } - long integer_length = state->cursor - start; + // Parse integer part and extract mantissa digits + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + mantissa = mantissa * 10 + (*state->cursor - '0'); + mantissa_digits++; + state->cursor++; + } - if (RB_UNLIKELY(start[0] == '0' && integer_length > 1)) { + if (RB_UNLIKELY(start[0] == '0' && mantissa_digits > 1)) { raise_parse_error_at("invalid number: %s", state, start); - } else if (RB_UNLIKELY(integer_length > 2 && start[0] == '-' && start[1] == '0')) { - raise_parse_error_at("invalid number: %s", state, start); - } else if (RB_UNLIKELY(integer_length == 1 && start[0] == '-')) { + } else if (RB_UNLIKELY(mantissa_digits > 1 && negative && start[1] == '0')) { raise_parse_error_at("invalid number: %s", state, start); } + // Parse fractional part if ((state->cursor < state->end) && (*state->cursor == '.')) { integer = false; + decimal_point_pos = mantissa_digits; // Remember position of decimal point state->cursor++; - if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state); + if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { + raise_parse_error_at("invalid number: %s", state, start); } - while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + mantissa = mantissa * 10 + (*state->cursor - '0'); + mantissa_digits++; state->cursor++; } } + // Parse exponent if ((state->cursor < state->end) && ((*state->cursor == 'e') || (*state->cursor == 'E'))) { integer = false; state->cursor++; - if ((state->cursor < state->end) && ((*state->cursor == '+') || (*state->cursor == '-'))) { + + bool negative_exponent = false; + if ((state->cursor < state->end) && ((*state->cursor == '-') || (*state->cursor == '+'))) { + negative_exponent = (*state->cursor == '-'); state->cursor++; } - if (state->cursor == state->end || *state->cursor < '0' || *state->cursor > '9') { - raise_parse_error("invalid number: %s", state); + if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { + raise_parse_error_at("invalid number: %s", state, start); } - while ((state->cursor < state->end) && (*state->cursor >= '0') && (*state->cursor <= '9')) { + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + exponent = exponent * 10 + (*state->cursor - '0'); state->cursor++; } + + if (negative_exponent) { + exponent = -exponent; + } } if (integer) { - return json_push_value(state, config, json_decode_integer(start, state->cursor)); + return json_push_value(state, config, json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor)); } - return json_push_value(state, config, json_decode_float(config, start, state->cursor)); + + // Adjust exponent based on decimal point position + if (decimal_point_pos >= 0) { + exponent -= (mantissa_digits - decimal_point_pos); + } + + return json_push_value(state, config, json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor)); } case '"': { // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} diff --git a/ext/json/ext/vendor/ryu.h b/ext/json/ext/vendor/ryu.h new file mode 100644 index 000000000..f06ec814b --- /dev/null +++ b/ext/json/ext/vendor/ryu.h @@ -0,0 +1,819 @@ +// Copyright 2018 Ulf Adams +// +// The contents of this file may be used under the terms of the Apache License, +// Version 2.0. +// +// Alternatively, the contents of this file may be used under the terms of +// the Boost Software License, Version 1.0. +// +// Unless required by applicable law or agreed to in writing, this software +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. +// +// --- +// +// Apache License +// Version 2.0, January 2004 +// http://www.apache.org/licenses/ +// +// TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +// +// 1. Definitions. +// +// "License" shall mean the terms and conditions for use, reproduction, +// and distribution as defined by Sections 1 through 9 of this document. +// +// "Licensor" shall mean the copyright owner or entity authorized by +// the copyright owner that is granting the License. +// +// "Legal Entity" shall mean the union of the acting entity and all +// other entities that control, are controlled by, or are under common +// control with that entity. For the purposes of this definition, +// "control" means (i) the power, direct or indirect, to cause the +// direction or management of such entity, whether by contract or +// otherwise, or (ii) ownership of fifty percent (50%) or more of the +// outstanding shares, or (iii) beneficial ownership of such entity. +// +// "You" (or "Your") shall mean an individual or Legal Entity +// exercising permissions granted by this License. +// +// "Source" form shall mean the preferred form for making modifications, +// including but not limited to software source code, documentation +// source, and configuration files. +// +// "Object" form shall mean any form resulting from mechanical +// transformation or translation of a Source form, including but +// not limited to compiled object code, generated documentation, +// and conversions to other media types. +// +// "Work" shall mean the work of authorship, whether in Source or +// Object form, made available under the License, as indicated by a +// copyright notice that is included in or attached to the work +// (an example is provided in the Appendix below). +// +// "Derivative Works" shall mean any work, whether in Source or Object +// form, that is based on (or derived from) the Work and for which the +// editorial revisions, annotations, elaborations, or other modifications +// represent, as a whole, an original work of authorship. For the purposes +// of this License, Derivative Works shall not include works that remain +// separable from, or merely link (or bind by name) to the interfaces of, +// the Work and Derivative Works thereof. +// +// "Contribution" shall mean any work of authorship, including +// the original version of the Work and any modifications or additions +// to that Work or Derivative Works thereof, that is intentionally +// submitted to Licensor for inclusion in the Work by the copyright owner +// or by an individual or Legal Entity authorized to submit on behalf of +// the copyright owner. For the purposes of this definition, "submitted" +// means any form of electronic, verbal, or written communication sent +// to the Licensor or its representatives, including but not limited to +// communication on electronic mailing lists, source code control systems, +// and issue tracking systems that are managed by, or on behalf of, the +// Licensor for the purpose of discussing and improving the Work, but +// excluding communication that is conspicuously marked or otherwise +// designated in writing by the copyright owner as "Not a Contribution." +// +// "Contributor" shall mean Licensor and any individual or Legal Entity +// on behalf of whom a Contribution has been received by Licensor and +// subsequently incorporated within the Work. +// +// 2. Grant of Copyright License. Subject to the terms and conditions of +// this License, each Contributor hereby grants to You a perpetual, +// worldwide, non-exclusive, no-charge, royalty-free, irrevocable +// copyright license to reproduce, prepare Derivative Works of, +// publicly display, publicly perform, sublicense, and distribute the +// Work and such Derivative Works in Source or Object form. +// +// 3. Grant of Patent License. Subject to the terms and conditions of +// this License, each Contributor hereby grants to You a perpetual, +// worldwide, non-exclusive, no-charge, royalty-free, irrevocable +// (except as stated in this section) patent license to make, have made, +// use, offer to sell, sell, import, and otherwise transfer the Work, +// where such license applies only to those patent claims licensable +// by such Contributor that are necessarily infringed by their +// Contribution(s) alone or by combination of their Contribution(s) +// with the Work to which such Contribution(s) was submitted. If You +// institute patent litigation against any entity (including a +// cross-claim or counterclaim in a lawsuit) alleging that the Work +// or a Contribution incorporated within the Work constitutes direct +// or contributory patent infringement, then any patent licenses +// granted to You under this License for that Work shall terminate +// as of the date such litigation is filed. +// +// 4. Redistribution. You may reproduce and distribute copies of the +// Work or Derivative Works thereof in any medium, with or without +// modifications, and in Source or Object form, provided that You +// meet the following conditions: +// +// (a) You must give any other recipients of the Work or +// Derivative Works a copy of this License; and +// +// (b) You must cause any modified files to carry prominent notices +// stating that You changed the files; and +// +// (c) You must retain, in the Source form of any Derivative Works +// that You distribute, all copyright, patent, trademark, and +// attribution notices from the Source form of the Work, +// excluding those notices that do not pertain to any part of +// the Derivative Works; and +// +// (d) If the Work includes a "NOTICE" text file as part of its +// distribution, then any Derivative Works that You distribute must +// include a readable copy of the attribution notices contained +// within such NOTICE file, excluding those notices that do not +// pertain to any part of the Derivative Works, in at least one +// of the following places: within a NOTICE text file distributed +// as part of the Derivative Works; within the Source form or +// documentation, if provided along with the Derivative Works; or, +// within a display generated by the Derivative Works, if and +// wherever such third-party notices normally appear. The contents +// of the NOTICE file are for informational purposes only and +// do not modify the License. You may add Your own attribution +// notices within Derivative Works that You distribute, alongside +// or as an addendum to the NOTICE text from the Work, provided +// that such additional attribution notices cannot be construed +// as modifying the License. +// +// You may add Your own copyright statement to Your modifications and +// may provide additional or different license terms and conditions +// for use, reproduction, or distribution of Your modifications, or +// for any such Derivative Works as a whole, provided Your use, +// reproduction, and distribution of the Work otherwise complies with +// the conditions stated in this License. +// +// 5. Submission of Contributions. Unless You explicitly state otherwise, +// any Contribution intentionally submitted for inclusion in the Work +// by You to the Licensor shall be under the terms and conditions of +// this License, without any additional terms or conditions. +// Notwithstanding the above, nothing herein shall supersede or modify +// the terms of any separate license agreement you may have executed +// with Licensor regarding such Contributions. +// +// 6. Trademarks. This License does not grant permission to use the trade +// names, trademarks, service marks, or product names of the Licensor, +// except as required for reasonable and customary use in describing the +// origin of the Work and reproducing the content of the NOTICE file. +// +// 7. Disclaimer of Warranty. Unless required by applicable law or +// agreed to in writing, Licensor provides the Work (and each +// Contributor provides its Contributions) on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied, including, without limitation, any warranties or conditions +// of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +// PARTICULAR PURPOSE. You are solely responsible for determining the +// appropriateness of using or redistributing the Work and assume any +// risks associated with Your exercise of permissions under this License. +// +// 8. Limitation of Liability. In no event and under no legal theory, +// whether in tort (including negligence), contract, or otherwise, +// unless required by applicable law (such as deliberate and grossly +// negligent acts) or agreed to in writing, shall any Contributor be +// liable to You for damages, including any direct, indirect, special, +// incidental, or consequential damages of any character arising as a +// result of this License or out of the use or inability to use the +// Work (including but not limited to damages for loss of goodwill, +// work stoppage, computer failure or malfunction, or any and all +// other commercial damages or losses), even if such Contributor +// has been advised of the possibility of such damages. +// +// 9. Accepting Warranty or Additional Liability. While redistributing +// the Work or Derivative Works thereof, You may choose to offer, +// and charge a fee for, acceptance of support, warranty, indemnity, +// or other liability obligations and/or rights consistent with this +// License. However, in accepting such obligations, You may act only +// on Your own behalf and on Your sole responsibility, not on behalf +// of any other Contributor, and only if You agree to indemnify, +// defend, and hold each Contributor harmless for any liability +// incurred by, or claims asserted against, such Contributor by reason +// of your accepting any such warranty or additional liability. +// +// END OF TERMS AND CONDITIONS +// +// APPENDIX: How to apply the Apache License to your work. +// +// To apply the Apache License to your work, attach the following +// boilerplate notice, with the fields enclosed by brackets "[]" +// replaced with your own identifying information. (Don't include +// the brackets!) The text should be enclosed in the appropriate +// comment syntax for the file format. We also recommend that a +// file or class name and description of purpose be included on the +// same "printed page" as the copyright notice for easier +// identification within third-party archives. +// +// Copyright [yyyy] [name of copyright owner] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// --- +// +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// --- +// Minimal Ryu implementation adapted for Ruby JSON gem by Josef Šimánek +// Optimized for pre-extracted mantissa/exponent from JSON parsing +// This is a stripped-down version containing only what's needed for +// converting decimal mantissa+exponent to IEEE 754 double precision. + +#ifndef RYU_H +#define RYU_H + +#include +#include +#include + +// Detect __builtin_clzll availability (for floor_log2) +// Note: MSVC doesn't have __builtin_clzll, so we provide a fallback +#ifdef __clang__ + #if __has_builtin(__builtin_clzll) + #define RYU_HAVE_BUILTIN_CLZLL 1 + #else + #define RYU_HAVE_BUILTIN_CLZLL 0 + #endif +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define RYU_HAVE_BUILTIN_CLZLL 1 +#else + #define RYU_HAVE_BUILTIN_CLZLL 0 +#endif + +// Count leading zeros (for floor_log2) +static inline uint32_t ryu_leading_zeros64(uint64_t input) +{ +#if RYU_HAVE_BUILTIN_CLZLL + return __builtin_clzll(input); +#else + // Fallback: binary search for the highest set bit + // This works on MSVC and other compilers without __builtin_clzll + if (input == 0) return 64; + uint32_t n = 0; + if (input <= 0x00000000FFFFFFFFULL) { n += 32; input <<= 32; } + if (input <= 0x0000FFFFFFFFFFFFULL) { n += 16; input <<= 16; } + if (input <= 0x00FFFFFFFFFFFFFFULL) { n += 8; input <<= 8; } + if (input <= 0x0FFFFFFFFFFFFFFFULL) { n += 4; input <<= 4; } + if (input <= 0x3FFFFFFFFFFFFFFFULL) { n += 2; input <<= 2; } + if (input <= 0x7FFFFFFFFFFFFFFFULL) { n += 1; } + return n; +#endif +} + +// These tables are generated by PrintDoubleLookupTable. +#define DOUBLE_POW5_INV_BITCOUNT 125 +#define DOUBLE_POW5_BITCOUNT 125 + +#define DOUBLE_POW5_INV_TABLE_SIZE 342 +#define DOUBLE_POW5_TABLE_SIZE 326 + +static const uint64_t DOUBLE_POW5_INV_SPLIT[DOUBLE_POW5_INV_TABLE_SIZE][2] = { + { 1u, 2305843009213693952u }, { 11068046444225730970u, 1844674407370955161u }, + { 5165088340638674453u, 1475739525896764129u }, { 7821419487252849886u, 1180591620717411303u }, + { 8824922364862649494u, 1888946593147858085u }, { 7059937891890119595u, 1511157274518286468u }, + { 13026647942995916322u, 1208925819614629174u }, { 9774590264567735146u, 1934281311383406679u }, + { 11509021026396098440u, 1547425049106725343u }, { 16585914450600699399u, 1237940039285380274u }, + { 15469416676735388068u, 1980704062856608439u }, { 16064882156130220778u, 1584563250285286751u }, + { 9162556910162266299u, 1267650600228229401u }, { 7281393426775805432u, 2028240960365167042u }, + { 16893161185646375315u, 1622592768292133633u }, { 2446482504291369283u, 1298074214633706907u }, + { 7603720821608101175u, 2076918743413931051u }, { 2393627842544570617u, 1661534994731144841u }, + { 16672297533003297786u, 1329227995784915872u }, { 11918280793837635165u, 2126764793255865396u }, + { 5845275820328197809u, 1701411834604692317u }, { 15744267100488289217u, 1361129467683753853u }, + { 3054734472329800808u, 2177807148294006166u }, { 17201182836831481939u, 1742245718635204932u }, + { 6382248639981364905u, 1393796574908163946u }, { 2832900194486363201u, 2230074519853062314u }, + { 5955668970331000884u, 1784059615882449851u }, { 1075186361522890384u, 1427247692705959881u }, + { 12788344622662355584u, 2283596308329535809u }, { 13920024512871794791u, 1826877046663628647u }, + { 3757321980813615186u, 1461501637330902918u }, { 10384555214134712795u, 1169201309864722334u }, + { 5547241898389809503u, 1870722095783555735u }, { 4437793518711847602u, 1496577676626844588u }, + { 10928932444453298728u, 1197262141301475670u }, { 17486291911125277965u, 1915619426082361072u }, + { 6610335899416401726u, 1532495540865888858u }, { 12666966349016942027u, 1225996432692711086u }, + { 12888448528943286597u, 1961594292308337738u }, { 17689456452638449924u, 1569275433846670190u }, + { 14151565162110759939u, 1255420347077336152u }, { 7885109000409574610u, 2008672555323737844u }, + { 9997436015069570011u, 1606938044258990275u }, { 7997948812055656009u, 1285550435407192220u }, + { 12796718099289049614u, 2056880696651507552u }, { 2858676849947419045u, 1645504557321206042u }, + { 13354987924183666206u, 1316403645856964833u }, { 17678631863951955605u, 2106245833371143733u }, + { 3074859046935833515u, 1684996666696914987u }, { 13527933681774397782u, 1347997333357531989u }, + { 10576647446613305481u, 2156795733372051183u }, { 15840015586774465031u, 1725436586697640946u }, + { 8982663654677661702u, 1380349269358112757u }, { 18061610662226169046u, 2208558830972980411u }, + { 10759939715039024913u, 1766847064778384329u }, { 12297300586773130254u, 1413477651822707463u }, + { 15986332124095098083u, 2261564242916331941u }, { 9099716884534168143u, 1809251394333065553u }, + { 14658471137111155161u, 1447401115466452442u }, { 4348079280205103483u, 1157920892373161954u }, + { 14335624477811986218u, 1852673427797059126u }, { 7779150767507678651u, 1482138742237647301u }, + { 2533971799264232598u, 1185710993790117841u }, { 15122401323048503126u, 1897137590064188545u }, + { 12097921058438802501u, 1517710072051350836u }, { 5988988032009131678u, 1214168057641080669u }, + { 16961078480698431330u, 1942668892225729070u }, { 13568862784558745064u, 1554135113780583256u }, + { 7165741412905085728u, 1243308091024466605u }, { 11465186260648137165u, 1989292945639146568u }, + { 16550846638002330379u, 1591434356511317254u }, { 16930026125143774626u, 1273147485209053803u }, + { 4951948911778577463u, 2037035976334486086u }, { 272210314680951647u, 1629628781067588869u }, + { 3907117066486671641u, 1303703024854071095u }, { 6251387306378674625u, 2085924839766513752u }, + { 16069156289328670670u, 1668739871813211001u }, { 9165976216721026213u, 1334991897450568801u }, + { 7286864317269821294u, 2135987035920910082u }, { 16897537898041588005u, 1708789628736728065u }, + { 13518030318433270404u, 1367031702989382452u }, { 6871453250525591353u, 2187250724783011924u }, + { 9186511415162383406u, 1749800579826409539u }, { 11038557946871817048u, 1399840463861127631u }, + { 10282995085511086630u, 2239744742177804210u }, { 8226396068408869304u, 1791795793742243368u }, + { 13959814484210916090u, 1433436634993794694u }, { 11267656730511734774u, 2293498615990071511u }, + { 5324776569667477496u, 1834798892792057209u }, { 7949170070475892320u, 1467839114233645767u }, + { 17427382500606444826u, 1174271291386916613u }, { 5747719112518849781u, 1878834066219066582u }, + { 15666221734240810795u, 1503067252975253265u }, { 12532977387392648636u, 1202453802380202612u }, + { 5295368560860596524u, 1923926083808324180u }, { 4236294848688477220u, 1539140867046659344u }, + { 7078384693692692099u, 1231312693637327475u }, { 11325415509908307358u, 1970100309819723960u }, + { 9060332407926645887u, 1576080247855779168u }, { 14626963555825137356u, 1260864198284623334u }, + { 12335095245094488799u, 2017382717255397335u }, { 9868076196075591040u, 1613906173804317868u }, + { 15273158586344293478u, 1291124939043454294u }, { 13369007293925138595u, 2065799902469526871u }, + { 7005857020398200553u, 1652639921975621497u }, { 16672732060544291412u, 1322111937580497197u }, + { 11918976037903224966u, 2115379100128795516u }, { 5845832015580669650u, 1692303280103036413u }, + { 12055363241948356366u, 1353842624082429130u }, { 841837113407818570u, 2166148198531886609u }, + { 4362818505468165179u, 1732918558825509287u }, { 14558301248600263113u, 1386334847060407429u }, + { 12225235553534690011u, 2218135755296651887u }, { 2401490813343931363u, 1774508604237321510u }, + { 1921192650675145090u, 1419606883389857208u }, { 17831303500047873437u, 2271371013423771532u }, + { 6886345170554478103u, 1817096810739017226u }, { 1819727321701672159u, 1453677448591213781u }, + { 16213177116328979020u, 1162941958872971024u }, { 14873036941900635463u, 1860707134196753639u }, + { 15587778368262418694u, 1488565707357402911u }, { 8780873879868024632u, 1190852565885922329u }, + { 2981351763563108441u, 1905364105417475727u }, { 13453127855076217722u, 1524291284333980581u }, + { 7073153469319063855u, 1219433027467184465u }, { 11317045550910502167u, 1951092843947495144u }, + { 12742985255470312057u, 1560874275157996115u }, { 10194388204376249646u, 1248699420126396892u }, + { 1553625868034358140u, 1997919072202235028u }, { 8621598323911307159u, 1598335257761788022u }, + { 17965325103354776697u, 1278668206209430417u }, { 13987124906400001422u, 2045869129935088668u }, + { 121653480894270168u, 1636695303948070935u }, { 97322784715416134u, 1309356243158456748u }, + { 14913111714512307107u, 2094969989053530796u }, { 8241140556867935363u, 1675975991242824637u }, + { 17660958889720079260u, 1340780792994259709u }, { 17189487779326395846u, 2145249268790815535u }, + { 13751590223461116677u, 1716199415032652428u }, { 18379969808252713988u, 1372959532026121942u }, + { 14650556434236701088u, 2196735251241795108u }, { 652398703163629901u, 1757388200993436087u }, + { 11589965406756634890u, 1405910560794748869u }, { 7475898206584884855u, 2249456897271598191u }, + { 2291369750525997561u, 1799565517817278553u }, { 9211793429904618695u, 1439652414253822842u }, + { 18428218302589300235u, 2303443862806116547u }, { 7363877012587619542u, 1842755090244893238u }, + { 13269799239553916280u, 1474204072195914590u }, { 10615839391643133024u, 1179363257756731672u }, + { 2227947767661371545u, 1886981212410770676u }, { 16539753473096738529u, 1509584969928616540u }, + { 13231802778477390823u, 1207667975942893232u }, { 6413489186596184024u, 1932268761508629172u }, + { 16198837793502678189u, 1545815009206903337u }, { 5580372605318321905u, 1236652007365522670u }, + { 8928596168509315048u, 1978643211784836272u }, { 18210923379033183008u, 1582914569427869017u }, + { 7190041073742725760u, 1266331655542295214u }, { 436019273762630246u, 2026130648867672343u }, + { 7727513048493924843u, 1620904519094137874u }, { 9871359253537050198u, 1296723615275310299u }, + { 4726128361433549347u, 2074757784440496479u }, { 7470251503888749801u, 1659806227552397183u }, + { 13354898832594820487u, 1327844982041917746u }, { 13989140502667892133u, 2124551971267068394u }, + { 14880661216876224029u, 1699641577013654715u }, { 11904528973500979224u, 1359713261610923772u }, + { 4289851098633925465u, 2175541218577478036u }, { 18189276137874781665u, 1740432974861982428u }, + { 3483374466074094362u, 1392346379889585943u }, { 1884050330976640656u, 2227754207823337509u }, + { 5196589079523222848u, 1782203366258670007u }, { 15225317707844309248u, 1425762693006936005u }, + { 5913764258841343181u, 2281220308811097609u }, { 8420360221814984868u, 1824976247048878087u }, + { 17804334621677718864u, 1459980997639102469u }, { 17932816512084085415u, 1167984798111281975u }, + { 10245762345624985047u, 1868775676978051161u }, { 4507261061758077715u, 1495020541582440929u }, + { 7295157664148372495u, 1196016433265952743u }, { 7982903447895485668u, 1913626293225524389u }, + { 10075671573058298858u, 1530901034580419511u }, { 4371188443704728763u, 1224720827664335609u }, + { 14372599139411386667u, 1959553324262936974u }, { 15187428126271019657u, 1567642659410349579u }, + { 15839291315758726049u, 1254114127528279663u }, { 3206773216762499739u, 2006582604045247462u }, + { 13633465017635730761u, 1605266083236197969u }, { 14596120828850494932u, 1284212866588958375u }, + { 4907049252451240275u, 2054740586542333401u }, { 236290587219081897u, 1643792469233866721u }, + { 14946427728742906810u, 1315033975387093376u }, { 16535586736504830250u, 2104054360619349402u }, + { 5849771759720043554u, 1683243488495479522u }, { 15747863852001765813u, 1346594790796383617u }, + { 10439186904235184007u, 2154551665274213788u }, { 15730047152871967852u, 1723641332219371030u }, + { 12584037722297574282u, 1378913065775496824u }, { 9066413911450387881u, 2206260905240794919u }, + { 10942479943902220628u, 1765008724192635935u }, { 8753983955121776503u, 1412006979354108748u }, + { 10317025513452932081u, 2259211166966573997u }, { 874922781278525018u, 1807368933573259198u }, + { 8078635854506640661u, 1445895146858607358u }, { 13841606313089133175u, 1156716117486885886u }, + { 14767872471458792434u, 1850745787979017418u }, { 746251532941302978u, 1480596630383213935u }, + { 597001226353042382u, 1184477304306571148u }, { 15712597221132509104u, 1895163686890513836u }, + { 8880728962164096960u, 1516130949512411069u }, { 10793931984473187891u, 1212904759609928855u }, + { 17270291175157100626u, 1940647615375886168u }, { 2748186495899949531u, 1552518092300708935u }, + { 2198549196719959625u, 1242014473840567148u }, { 18275073973719576693u, 1987223158144907436u }, + { 10930710364233751031u, 1589778526515925949u }, { 12433917106128911148u, 1271822821212740759u }, + { 8826220925580526867u, 2034916513940385215u }, { 7060976740464421494u, 1627933211152308172u }, + { 16716827836597268165u, 1302346568921846537u }, { 11989529279587987770u, 2083754510274954460u }, + { 9591623423670390216u, 1667003608219963568u }, { 15051996368420132820u, 1333602886575970854u }, + { 13015147745246481542u, 2133764618521553367u }, { 3033420566713364587u, 1707011694817242694u }, + { 6116085268112601993u, 1365609355853794155u }, { 9785736428980163188u, 2184974969366070648u }, + { 15207286772667951197u, 1747979975492856518u }, { 1097782973908629988u, 1398383980394285215u }, + { 1756452758253807981u, 2237414368630856344u }, { 5094511021344956708u, 1789931494904685075u }, + { 4075608817075965366u, 1431945195923748060u }, { 6520974107321544586u, 2291112313477996896u }, + { 1527430471115325346u, 1832889850782397517u }, { 12289990821117991246u, 1466311880625918013u }, + { 17210690286378213644u, 1173049504500734410u }, { 9090360384495590213u, 1876879207201175057u }, + { 18340334751822203140u, 1501503365760940045u }, { 14672267801457762512u, 1201202692608752036u }, + { 16096930852848599373u, 1921924308174003258u }, { 1809498238053148529u, 1537539446539202607u }, + { 12515645034668249793u, 1230031557231362085u }, { 1578287981759648052u, 1968050491570179337u }, + { 12330676829633449412u, 1574440393256143469u }, { 13553890278448669853u, 1259552314604914775u }, + { 3239480371808320148u, 2015283703367863641u }, { 17348979556414297411u, 1612226962694290912u }, + { 6500486015647617283u, 1289781570155432730u }, { 10400777625036187652u, 2063650512248692368u }, + { 15699319729512770768u, 1650920409798953894u }, { 16248804598352126938u, 1320736327839163115u }, + { 7551343283653851484u, 2113178124542660985u }, { 6041074626923081187u, 1690542499634128788u }, + { 12211557331022285596u, 1352433999707303030u }, { 1091747655926105338u, 2163894399531684849u }, + { 4562746939482794594u, 1731115519625347879u }, { 7339546366328145998u, 1384892415700278303u }, + { 8053925371383123274u, 2215827865120445285u }, { 6443140297106498619u, 1772662292096356228u }, + { 12533209867169019542u, 1418129833677084982u }, { 5295740528502789974u, 2269007733883335972u }, + { 15304638867027962949u, 1815206187106668777u }, { 4865013464138549713u, 1452164949685335022u }, + { 14960057215536570740u, 1161731959748268017u }, { 9178696285890871890u, 1858771135597228828u }, + { 14721654658196518159u, 1487016908477783062u }, { 4398626097073393881u, 1189613526782226450u }, + { 7037801755317430209u, 1903381642851562320u }, { 5630241404253944167u, 1522705314281249856u }, + { 814844308661245011u, 1218164251424999885u }, { 1303750893857992017u, 1949062802279999816u }, + { 15800395974054034906u, 1559250241823999852u }, { 5261619149759407279u, 1247400193459199882u }, + { 12107939454356961969u, 1995840309534719811u }, { 5997002748743659252u, 1596672247627775849u }, + { 8486951013736837725u, 1277337798102220679u }, { 2511075177753209390u, 2043740476963553087u }, + { 13076906586428298482u, 1634992381570842469u }, { 14150874083884549109u, 1307993905256673975u }, + { 4194654460505726958u, 2092790248410678361u }, { 18113118827372222859u, 1674232198728542688u }, + { 3422448617672047318u, 1339385758982834151u }, { 16543964232501006678u, 2143017214372534641u }, + { 9545822571258895019u, 1714413771498027713u }, { 15015355686490936662u, 1371531017198422170u }, + { 5577825024675947042u, 2194449627517475473u }, { 11840957649224578280u, 1755559702013980378u }, + { 16851463748863483271u, 1404447761611184302u }, { 12204946739213931940u, 2247116418577894884u }, + { 13453306206113055875u, 1797693134862315907u }, { 3383947335406624054u, 1438154507889852726u }, + { 16482362180876329456u, 2301047212623764361u }, { 9496540929959153242u, 1840837770099011489u }, + { 11286581558709232917u, 1472670216079209191u }, { 5339916432225476010u, 1178136172863367353u }, + { 4854517476818851293u, 1885017876581387765u }, { 3883613981455081034u, 1508014301265110212u }, + { 14174937629389795797u, 1206411441012088169u }, { 11611853762797942306u, 1930258305619341071u }, + { 5600134195496443521u, 1544206644495472857u }, { 15548153800622885787u, 1235365315596378285u }, + { 6430302007287065643u, 1976584504954205257u }, { 16212288050055383484u, 1581267603963364205u }, + { 12969830440044306787u, 1265014083170691364u }, { 9683682259845159889u, 2024022533073106183u }, + { 15125643437359948558u, 1619218026458484946u }, { 8411165935146048523u, 1295374421166787957u }, + { 17147214310975587960u, 2072599073866860731u }, { 10028422634038560045u, 1658079259093488585u }, + { 8022738107230848036u, 1326463407274790868u }, { 9147032156827446534u, 2122341451639665389u }, + { 11006974540203867551u, 1697873161311732311u }, { 5116230817421183718u, 1358298529049385849u }, + { 15564666937357714594u, 2173277646479017358u }, { 1383687105660440706u, 1738622117183213887u }, + { 12174996128754083534u, 1390897693746571109u }, { 8411947361780802685u, 2225436309994513775u }, + { 6729557889424642148u, 1780349047995611020u }, { 5383646311539713719u, 1424279238396488816u }, + { 1235136468979721303u, 2278846781434382106u }, { 15745504434151418335u, 1823077425147505684u }, + { 16285752362063044992u, 1458461940118004547u }, { 5649904260166615347u, 1166769552094403638u }, + { 5350498001524674232u, 1866831283351045821u }, { 591049586477829062u, 1493465026680836657u }, + { 11540886113407994219u, 1194772021344669325u }, { 18673707743239135u, 1911635234151470921u }, + { 14772334225162232601u, 1529308187321176736u }, { 8128518565387875758u, 1223446549856941389u }, + { 1937583260394870242u, 1957514479771106223u }, { 8928764237799716840u, 1566011583816884978u }, + { 14521709019723594119u, 1252809267053507982u }, { 8477339172590109297u, 2004494827285612772u }, + { 17849917782297818407u, 1603595861828490217u }, { 6901236596354434079u, 1282876689462792174u }, + { 18420676183650915173u, 2052602703140467478u }, { 3668494502695001169u, 1642082162512373983u }, + { 10313493231639821582u, 1313665730009899186u }, { 9122891541139893884u, 2101865168015838698u }, + { 14677010862395735754u, 1681492134412670958u }, { 673562245690857633u, 1345193707530136767u } +}; + +static const uint64_t DOUBLE_POW5_SPLIT[DOUBLE_POW5_TABLE_SIZE][2] = { + { 0u, 1152921504606846976u }, { 0u, 1441151880758558720u }, + { 0u, 1801439850948198400u }, { 0u, 2251799813685248000u }, + { 0u, 1407374883553280000u }, { 0u, 1759218604441600000u }, + { 0u, 2199023255552000000u }, { 0u, 1374389534720000000u }, + { 0u, 1717986918400000000u }, { 0u, 2147483648000000000u }, + { 0u, 1342177280000000000u }, { 0u, 1677721600000000000u }, + { 0u, 2097152000000000000u }, { 0u, 1310720000000000000u }, + { 0u, 1638400000000000000u }, { 0u, 2048000000000000000u }, + { 0u, 1280000000000000000u }, { 0u, 1600000000000000000u }, + { 0u, 2000000000000000000u }, { 0u, 1250000000000000000u }, + { 0u, 1562500000000000000u }, { 0u, 1953125000000000000u }, + { 0u, 1220703125000000000u }, { 0u, 1525878906250000000u }, + { 0u, 1907348632812500000u }, { 0u, 1192092895507812500u }, + { 0u, 1490116119384765625u }, { 4611686018427387904u, 1862645149230957031u }, + { 9799832789158199296u, 1164153218269348144u }, { 12249790986447749120u, 1455191522836685180u }, + { 15312238733059686400u, 1818989403545856475u }, { 14528612397897220096u, 2273736754432320594u }, + { 13692068767113150464u, 1421085471520200371u }, { 12503399940464050176u, 1776356839400250464u }, + { 15629249925580062720u, 2220446049250313080u }, { 9768281203487539200u, 1387778780781445675u }, + { 7598665485932036096u, 1734723475976807094u }, { 274959820560269312u, 2168404344971008868u }, + { 9395221924704944128u, 1355252715606880542u }, { 2520655369026404352u, 1694065894508600678u }, + { 12374191248137781248u, 2117582368135750847u }, { 14651398557727195136u, 1323488980084844279u }, + { 13702562178731606016u, 1654361225106055349u }, { 3293144668132343808u, 2067951531382569187u }, + { 18199116482078572544u, 1292469707114105741u }, { 8913837547316051968u, 1615587133892632177u }, + { 15753982952572452864u, 2019483917365790221u }, { 12152082354571476992u, 1262177448353618888u }, + { 15190102943214346240u, 1577721810442023610u }, { 9764256642163156992u, 1972152263052529513u }, + { 17631875447420442880u, 1232595164407830945u }, { 8204786253993389888u, 1540743955509788682u }, + { 1032610780636961552u, 1925929944387235853u }, { 2951224747111794922u, 1203706215242022408u }, + { 3689030933889743652u, 1504632769052528010u }, { 13834660704216955373u, 1880790961315660012u }, + { 17870034976990372916u, 1175494350822287507u }, { 17725857702810578241u, 1469367938527859384u }, + { 3710578054803671186u, 1836709923159824231u }, { 26536550077201078u, 2295887403949780289u }, + { 11545800389866720434u, 1434929627468612680u }, { 14432250487333400542u, 1793662034335765850u }, + { 8816941072311974870u, 2242077542919707313u }, { 17039803216263454053u, 1401298464324817070u }, + { 12076381983474541759u, 1751623080406021338u }, { 5872105442488401391u, 2189528850507526673u }, + { 15199280947623720629u, 1368455531567204170u }, { 9775729147674874978u, 1710569414459005213u }, + { 16831347453020981627u, 2138211768073756516u }, { 1296220121283337709u, 1336382355046097823u }, + { 15455333206886335848u, 1670477943807622278u }, { 10095794471753144002u, 2088097429759527848u }, + { 6309871544845715001u, 1305060893599704905u }, { 12499025449484531656u, 1631326116999631131u }, + { 11012095793428276666u, 2039157646249538914u }, { 11494245889320060820u, 1274473528905961821u }, + { 532749306367912313u, 1593091911132452277u }, { 5277622651387278295u, 1991364888915565346u }, + { 7910200175544436838u, 1244603055572228341u }, { 14499436237857933952u, 1555753819465285426u }, + { 8900923260467641632u, 1944692274331606783u }, { 12480606065433357876u, 1215432671457254239u }, + { 10989071563364309441u, 1519290839321567799u }, { 9124653435777998898u, 1899113549151959749u }, + { 8008751406574943263u, 1186945968219974843u }, { 5399253239791291175u, 1483682460274968554u }, + { 15972438586593889776u, 1854603075343710692u }, { 759402079766405302u, 1159126922089819183u }, + { 14784310654990170340u, 1448908652612273978u }, { 9257016281882937117u, 1811135815765342473u }, + { 16182956370781059300u, 2263919769706678091u }, { 7808504722524468110u, 1414949856066673807u }, + { 5148944884728197234u, 1768687320083342259u }, { 1824495087482858639u, 2210859150104177824u }, + { 1140309429676786649u, 1381786968815111140u }, { 1425386787095983311u, 1727233711018888925u }, + { 6393419502297367043u, 2159042138773611156u }, { 13219259225790630210u, 1349401336733506972u }, + { 16524074032238287762u, 1686751670916883715u }, { 16043406521870471799u, 2108439588646104644u }, + { 803757039314269066u, 1317774742903815403u }, { 14839754354425000045u, 1647218428629769253u }, + { 4714634887749086344u, 2059023035787211567u }, { 9864175832484260821u, 1286889397367007229u }, + { 16941905809032713930u, 1608611746708759036u }, { 2730638187581340797u, 2010764683385948796u }, + { 10930020904093113806u, 1256727927116217997u }, { 18274212148543780162u, 1570909908895272496u }, + { 4396021111970173586u, 1963637386119090621u }, { 5053356204195052443u, 1227273366324431638u }, + { 15540067292098591362u, 1534091707905539547u }, { 14813398096695851299u, 1917614634881924434u }, + { 13870059828862294966u, 1198509146801202771u }, { 12725888767650480803u, 1498136433501503464u }, + { 15907360959563101004u, 1872670541876879330u }, { 14553786618154326031u, 1170419088673049581u }, + { 4357175217410743827u, 1463023860841311977u }, { 10058155040190817688u, 1828779826051639971u }, + { 7961007781811134206u, 2285974782564549964u }, { 14199001900486734687u, 1428734239102843727u }, + { 13137066357181030455u, 1785917798878554659u }, { 11809646928048900164u, 2232397248598193324u }, + { 16604401366885338411u, 1395248280373870827u }, { 16143815690179285109u, 1744060350467338534u }, + { 10956397575869330579u, 2180075438084173168u }, { 6847748484918331612u, 1362547148802608230u }, + { 17783057643002690323u, 1703183936003260287u }, { 17617136035325974999u, 2128979920004075359u }, + { 17928239049719816230u, 1330612450002547099u }, { 17798612793722382384u, 1663265562503183874u }, + { 13024893955298202172u, 2079081953128979843u }, { 5834715712847682405u, 1299426220705612402u }, + { 16516766677914378815u, 1624282775882015502u }, { 11422586310538197711u, 2030353469852519378u }, + { 11750802462513761473u, 1268970918657824611u }, { 10076817059714813937u, 1586213648322280764u }, + { 12596021324643517422u, 1982767060402850955u }, { 5566670318688504437u, 1239229412751781847u }, + { 2346651879933242642u, 1549036765939727309u }, { 7545000868343941206u, 1936295957424659136u }, + { 4715625542714963254u, 1210184973390411960u }, { 5894531928393704067u, 1512731216738014950u }, + { 16591536947346905892u, 1890914020922518687u }, { 17287239619732898039u, 1181821263076574179u }, + { 16997363506238734644u, 1477276578845717724u }, { 2799960309088866689u, 1846595723557147156u }, + { 10973347230035317489u, 1154122327223216972u }, { 13716684037544146861u, 1442652909029021215u }, + { 12534169028502795672u, 1803316136286276519u }, { 11056025267201106687u, 2254145170357845649u }, + { 18439230838069161439u, 1408840731473653530u }, { 13825666510731675991u, 1761050914342066913u }, + { 3447025083132431277u, 2201313642927583642u }, { 6766076695385157452u, 1375821026829739776u }, + { 8457595869231446815u, 1719776283537174720u }, { 10571994836539308519u, 2149720354421468400u }, + { 6607496772837067824u, 1343575221513417750u }, { 17482743002901110588u, 1679469026891772187u }, + { 17241742735199000331u, 2099336283614715234u }, { 15387775227926763111u, 1312085177259197021u }, + { 5399660979626290177u, 1640106471573996277u }, { 11361262242960250625u, 2050133089467495346u }, + { 11712474920277544544u, 1281333180917184591u }, { 10028907631919542777u, 1601666476146480739u }, + { 7924448521472040567u, 2002083095183100924u }, { 14176152362774801162u, 1251301934489438077u }, + { 3885132398186337741u, 1564127418111797597u }, { 9468101516160310080u, 1955159272639746996u }, + { 15140935484454969608u, 1221974545399841872u }, { 479425281859160394u, 1527468181749802341u }, + { 5210967620751338397u, 1909335227187252926u }, { 17091912818251750210u, 1193334516992033078u }, + { 12141518985959911954u, 1491668146240041348u }, { 15176898732449889943u, 1864585182800051685u }, + { 11791404716994875166u, 1165365739250032303u }, { 10127569877816206054u, 1456707174062540379u }, + { 8047776328842869663u, 1820883967578175474u }, { 836348374198811271u, 2276104959472719343u }, + { 7440246761515338900u, 1422565599670449589u }, { 13911994470321561530u, 1778206999588061986u }, + { 8166621051047176104u, 2222758749485077483u }, { 2798295147690791113u, 1389224218428173427u }, + { 17332926989895652603u, 1736530273035216783u }, { 17054472718942177850u, 2170662841294020979u }, + { 8353202440125167204u, 1356664275808763112u }, { 10441503050156459005u, 1695830344760953890u }, + { 3828506775840797949u, 2119787930951192363u }, { 86973725686804766u, 1324867456844495227u }, + { 13943775212390669669u, 1656084321055619033u }, { 3594660960206173375u, 2070105401319523792u }, + { 2246663100128858359u, 1293815875824702370u }, { 12031700912015848757u, 1617269844780877962u }, + { 5816254103165035138u, 2021587305976097453u }, { 5941001823691840913u, 1263492066235060908u }, + { 7426252279614801142u, 1579365082793826135u }, { 4671129331091113523u, 1974206353492282669u }, + { 5225298841145639904u, 1233878970932676668u }, { 6531623551432049880u, 1542348713665845835u }, + { 3552843420862674446u, 1927935892082307294u }, { 16055585193321335241u, 1204959932551442058u }, + { 10846109454796893243u, 1506199915689302573u }, { 18169322836923504458u, 1882749894611628216u }, + { 11355826773077190286u, 1176718684132267635u }, { 9583097447919099954u, 1470898355165334544u }, + { 11978871809898874942u, 1838622943956668180u }, { 14973589762373593678u, 2298278679945835225u }, + { 2440964573842414192u, 1436424174966147016u }, { 3051205717303017741u, 1795530218707683770u }, + { 13037379183483547984u, 2244412773384604712u }, { 8148361989677217490u, 1402757983365377945u }, + { 14797138505523909766u, 1753447479206722431u }, { 13884737113477499304u, 2191809349008403039u }, + { 15595489723564518921u, 1369880843130251899u }, { 14882676136028260747u, 1712351053912814874u }, + { 9379973133180550126u, 2140438817391018593u }, { 17391698254306313589u, 1337774260869386620u }, + { 3292878744173340370u, 1672217826086733276u }, { 4116098430216675462u, 2090272282608416595u }, + { 266718509671728212u, 1306420176630260372u }, { 333398137089660265u, 1633025220787825465u }, + { 5028433689789463235u, 2041281525984781831u }, { 10060300083759496378u, 1275800953740488644u }, + { 12575375104699370472u, 1594751192175610805u }, { 1884160825592049379u, 1993438990219513507u }, + { 17318501580490888525u, 1245899368887195941u }, { 7813068920331446945u, 1557374211108994927u }, + { 5154650131986920777u, 1946717763886243659u }, { 915813323278131534u, 1216698602428902287u }, + { 14979824709379828129u, 1520873253036127858u }, { 9501408849870009354u, 1901091566295159823u }, + { 12855909558809837702u, 1188182228934474889u }, { 2234828893230133415u, 1485227786168093612u }, + { 2793536116537666769u, 1856534732710117015u }, { 8663489100477123587u, 1160334207943823134u }, + { 1605989338741628675u, 1450417759929778918u }, { 11230858710281811652u, 1813022199912223647u }, + { 9426887369424876662u, 2266277749890279559u }, { 12809333633531629769u, 1416423593681424724u }, + { 16011667041914537212u, 1770529492101780905u }, { 6179525747111007803u, 2213161865127226132u }, + { 13085575628799155685u, 1383226165704516332u }, { 16356969535998944606u, 1729032707130645415u }, + { 15834525901571292854u, 2161290883913306769u }, { 2979049660840976177u, 1350806802445816731u }, + { 17558870131333383934u, 1688508503057270913u }, { 8113529608884566205u, 2110635628821588642u }, + { 9682642023980241782u, 1319147268013492901u }, { 16714988548402690132u, 1648934085016866126u }, + { 11670363648648586857u, 2061167606271082658u }, { 11905663298832754689u, 1288229753919426661u }, + { 1047021068258779650u, 1610287192399283327u }, { 15143834390605638274u, 2012858990499104158u }, + { 4853210475701136017u, 1258036869061940099u }, { 1454827076199032118u, 1572546086327425124u }, + { 1818533845248790147u, 1965682607909281405u }, { 3442426662494187794u, 1228551629943300878u }, + { 13526405364972510550u, 1535689537429126097u }, { 3072948650933474476u, 1919611921786407622u }, + { 15755650962115585259u, 1199757451116504763u }, { 15082877684217093670u, 1499696813895630954u }, + { 9630225068416591280u, 1874621017369538693u }, { 8324733676974063502u, 1171638135855961683u }, + { 5794231077790191473u, 1464547669819952104u }, { 7242788847237739342u, 1830684587274940130u }, + { 18276858095901949986u, 2288355734093675162u }, { 16034722328366106645u, 1430222333808546976u }, + { 1596658836748081690u, 1787777917260683721u }, { 6607509564362490017u, 2234722396575854651u }, + { 1823850468512862308u, 1396701497859909157u }, { 6891499104068465790u, 1745876872324886446u }, + { 17837745916940358045u, 2182346090406108057u }, { 4231062170446641922u, 1363966306503817536u }, + { 5288827713058302403u, 1704957883129771920u }, { 6611034641322878003u, 2131197353912214900u }, + { 13355268687681574560u, 1331998346195134312u }, { 16694085859601968200u, 1664997932743917890u }, + { 11644235287647684442u, 2081247415929897363u }, { 4971804045566108824u, 1300779634956185852u }, + { 6214755056957636030u, 1625974543695232315u }, { 3156757802769657134u, 2032468179619040394u }, + { 6584659645158423613u, 1270292612261900246u }, { 17454196593302805324u, 1587865765327375307u }, + { 17206059723201118751u, 1984832206659219134u }, { 6142101308573311315u, 1240520129162011959u }, + { 3065940617289251240u, 1550650161452514949u }, { 8444111790038951954u, 1938312701815643686u }, + { 665883850346957067u, 1211445438634777304u }, { 832354812933696334u, 1514306798293471630u }, + { 10263815553021896226u, 1892883497866839537u }, { 17944099766707154901u, 1183052186166774710u }, + { 13206752671529167818u, 1478815232708468388u }, { 16508440839411459773u, 1848519040885585485u }, + { 12623618533845856310u, 1155324400553490928u }, { 15779523167307320387u, 1444155500691863660u }, + { 1277659885424598868u, 1805194375864829576u }, { 1597074856780748586u, 2256492969831036970u }, + { 5609857803915355770u, 1410308106144398106u }, { 16235694291748970521u, 1762885132680497632u }, + { 1847873790976661535u, 2203606415850622041u }, { 12684136165428883219u, 1377254009906638775u }, + { 11243484188358716120u, 1721567512383298469u }, { 219297180166231438u, 2151959390479123087u }, + { 7054589765244976505u, 1344974619049451929u }, { 13429923224983608535u, 1681218273811814911u }, + { 12175718012802122765u, 2101522842264768639u }, { 14527352785642408584u, 1313451776415480399u }, + { 13547504963625622826u, 1641814720519350499u }, { 12322695186104640628u, 2052268400649188124u }, + { 16925056528170176201u, 1282667750405742577u }, { 7321262604930556539u, 1603334688007178222u }, + { 18374950293017971482u, 2004168360008972777u }, { 4566814905495150320u, 1252605225005607986u }, + { 14931890668723713708u, 1565756531257009982u }, { 9441491299049866327u, 1957195664071262478u }, + { 1289246043478778550u, 1223247290044539049u }, { 6223243572775861092u, 1529059112555673811u }, + { 3167368447542438461u, 1911323890694592264u }, { 1979605279714024038u, 1194577431684120165u }, + { 7086192618069917952u, 1493221789605150206u }, { 18081112809442173248u, 1866527237006437757u }, + { 13606538515115052232u, 1166579523129023598u }, { 7784801107039039482u, 1458224403911279498u }, + { 507629346944023544u, 1822780504889099373u }, { 5246222702107417334u, 2278475631111374216u }, + { 3278889188817135834u, 1424047269444608885u }, { 8710297504448807696u, 1780059086805761106u } +}; + +// IEEE 754 double precision constants +#define DOUBLE_MANTISSA_BITS 52 +#define DOUBLE_EXPONENT_BITS 11 +#define DOUBLE_EXPONENT_BIAS 1023 + +// Helper: floor(log2(value)) using ryu_leading_zeros64 +static inline uint32_t floor_log2(const uint64_t value) { + return 63 - ryu_leading_zeros64(value); +} + +// Helper: log2(5^e) approximation +static inline int32_t log2pow5(const int32_t e) { + return (int32_t) ((((uint32_t) e) * 1217359) >> 19); +} + +// Helper: ceil(log2(5^e)) +static inline int32_t ceil_log2pow5(const int32_t e) { + return log2pow5(e) + 1; +} + +// Helper: max of two int32 +static inline int32_t max32(int32_t a, int32_t b) { + return a < b ? b : a; +} + +// Helper: convert uint64 bits to double +static inline double int64Bits2Double(uint64_t bits) { + double f; + memcpy(&f, &bits, sizeof(double)); + return f; +} + +// Check if value is multiple of 2^p +static inline bool multipleOfPowerOf2(const uint64_t value, const uint32_t p) { + return (value & ((1ull << p) - 1)) == 0; +} + +// Count how many times value is divisible by 5 +// Uses modular inverse to avoid expensive division +static inline uint32_t pow5Factor(uint64_t value) { + const uint64_t m_inv_5 = 14757395258967641293u; // 5 * m_inv_5 = 1 (mod 2^64) + const uint64_t n_div_5 = 3689348814741910323u; // 2^64 / 5 + uint32_t count = 0; + for (;;) { + value *= m_inv_5; + if (value > n_div_5) + break; + ++count; + } + return count; +} + +// Check if value is multiple of 5^p +// Optimized: uses modular inverse instead of division +static inline bool multipleOfPowerOf5(const uint64_t value, const uint32_t p) { + return pow5Factor(value) >= p; +} + +// 128-bit multiplication with shift +// This is the core operation for converting decimal to binary +#if defined(__SIZEOF_INT128__) +// Use native 128-bit integers if available (GCC/Clang) +static inline uint64_t mulShift64(const uint64_t m, const uint64_t* const mul, const int32_t j) { + const unsigned __int128 b0 = ((unsigned __int128) m) * mul[0]; + const unsigned __int128 b2 = ((unsigned __int128) m) * mul[1]; + return (uint64_t) (((b0 >> 64) + b2) >> (j - 64)); +} +#else +// Fallback for systems without 128-bit integers +static inline uint64_t umul128(const uint64_t a, const uint64_t b, uint64_t* const productHi) { + const uint32_t aLo = (uint32_t)a; + const uint32_t aHi = (uint32_t)(a >> 32); + const uint32_t bLo = (uint32_t)b; + const uint32_t bHi = (uint32_t)(b >> 32); + + const uint64_t b00 = (uint64_t)aLo * bLo; + const uint64_t b01 = (uint64_t)aLo * bHi; + const uint64_t b10 = (uint64_t)aHi * bLo; + const uint64_t b11 = (uint64_t)aHi * bHi; + + const uint32_t b00Lo = (uint32_t)b00; + const uint32_t b00Hi = (uint32_t)(b00 >> 32); + + const uint64_t mid1 = b10 + b00Hi; + const uint32_t mid1Lo = (uint32_t)(mid1); + const uint32_t mid1Hi = (uint32_t)(mid1 >> 32); + + const uint64_t mid2 = b01 + mid1Lo; + const uint32_t mid2Lo = (uint32_t)(mid2); + const uint32_t mid2Hi = (uint32_t)(mid2 >> 32); + + const uint64_t pHi = b11 + mid1Hi + mid2Hi; + const uint64_t pLo = ((uint64_t)mid2Lo << 32) | b00Lo; + + *productHi = pHi; + return pLo; +} + +static inline uint64_t shiftright128(const uint64_t lo, const uint64_t hi, const uint32_t dist) { + return (hi << (64 - dist)) | (lo >> dist); +} + +static inline uint64_t mulShift64(const uint64_t m, const uint64_t* const mul, const int32_t j) { + uint64_t high1; + const uint64_t low1 = umul128(m, mul[1], &high1); + uint64_t high0; + umul128(m, mul[0], &high0); + const uint64_t sum = high0 + low1; + if (sum < high0) { + ++high1; + } + return shiftright128(sum, high1, j - 64); +} +#endif + +// Main conversion function: decimal mantissa+exponent to IEEE 754 double +// Optimized for JSON parsing with fast paths for edge cases +static inline double ryu_s2d_from_parts(uint64_t m10, int m10digits, int32_t e10, bool signedM) { + // Fast path: handle zero explicitly (e.g., "0.0", "0e0") + if (m10 == 0) { + return int64Bits2Double(((uint64_t) signedM) << 63); + } + + // Fast path: handle overflow/underflow early + if (m10digits + e10 <= -324) { + // Underflow to zero + return int64Bits2Double(((uint64_t) signedM) << 63); + } + + if (m10digits + e10 >= 310) { + // Overflow to infinity + return int64Bits2Double((((uint64_t) signedM) << 63) | 0x7ff0000000000000ULL); + } + + // Convert decimal to binary: m10 * 10^e10 = m2 * 2^e2 + int32_t e2; + uint64_t m2; + bool trailingZeros; + + if (e10 >= 0) { + // Positive exponent: multiply by 5^e10 and adjust binary exponent + e2 = floor_log2(m10) + e10 + log2pow5(e10) - (DOUBLE_MANTISSA_BITS + 1); + int j = e2 - e10 - ceil_log2pow5(e10) + DOUBLE_POW5_BITCOUNT; + m2 = mulShift64(m10, DOUBLE_POW5_SPLIT[e10], j); + trailingZeros = e2 < e10 || (e2 - e10 < 64 && multipleOfPowerOf2(m10, e2 - e10)); + } else { + // Negative exponent: divide by 5^(-e10) + e2 = floor_log2(m10) + e10 - ceil_log2pow5(-e10) - (DOUBLE_MANTISSA_BITS + 1); + int j = e2 - e10 + ceil_log2pow5(-e10) - 1 + DOUBLE_POW5_INV_BITCOUNT; + m2 = mulShift64(m10, DOUBLE_POW5_INV_SPLIT[-e10], j); + trailingZeros = multipleOfPowerOf5(m10, -e10); + } + + // Compute IEEE 754 exponent + uint32_t ieee_e2 = (uint32_t) max32(0, e2 + DOUBLE_EXPONENT_BIAS + floor_log2(m2)); + + if (ieee_e2 > 0x7fe) { + // Overflow to infinity + return int64Bits2Double((((uint64_t) signedM) << 63) | 0x7ff0000000000000ULL); + } + + // Compute shift amount for rounding + int32_t shift = (ieee_e2 == 0 ? 1 : ieee_e2) - e2 - DOUBLE_EXPONENT_BIAS - DOUBLE_MANTISSA_BITS; + + // IEEE 754 round-to-even (banker's rounding) + trailingZeros &= (m2 & ((1ull << (shift - 1)) - 1)) == 0; + uint64_t lastRemovedBit = (m2 >> (shift - 1)) & 1; + bool roundUp = (lastRemovedBit != 0) && (!trailingZeros || (((m2 >> shift) & 1) != 0)); + + uint64_t ieee_m2 = (m2 >> shift) + roundUp; + ieee_m2 &= (1ull << DOUBLE_MANTISSA_BITS) - 1; + + if (ieee_m2 == 0 && roundUp) { + ieee_e2++; + } + + // Pack sign, exponent, and mantissa into IEEE 754 format + // Match original Ryu: group sign+exponent, then shift and add mantissa + uint64_t ieee = (((((uint64_t) signedM) << DOUBLE_EXPONENT_BITS) | (uint64_t)ieee_e2) << DOUBLE_MANTISSA_BITS) | ieee_m2; + return int64Bits2Double(ieee); +} + +#endif // RYU_H diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_ryu_fallback_test.rb new file mode 100644 index 000000000..59ba76d39 --- /dev/null +++ b/test/json/json_ryu_fallback_test.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true +require_relative 'test_helper' +begin + require 'bigdecimal' +rescue LoadError +end + +class JSONRyuFallbackTest < Test::Unit::TestCase + include JSON + + # Test that numbers with more than 17 significant digits fall back to rb_cstr_to_dbl + def test_more_than_17_significant_digits + # These numbers have > 17 significant digits and should use fallback path + # They should still parse correctly, just not via the Ryu optimization + + test_cases = [ + # input, expected (rounded to double precision) + ["1.23456789012345678901234567890", 1.2345678901234567], + ["123456789012345678.901234567890", 1.2345678901234568e+17], + ["0.123456789012345678901234567890", 0.12345678901234568], + ["9999999999999999999999999999.9", 1.0e+28], + # Edge case: exactly 18 digits + ["123456789012345678", 123456789012345680.0], + # Many fractional digits + ["0.12345678901234567890123456789", 0.12345678901234568], + ] + + test_cases.each do |input, expected| + result = JSON.parse(input) + assert_in_delta(expected, result, 1e-10, + "Failed to parse #{input} correctly (>17 digits, fallback path)") + end + end + + # Test decimal_class option forces fallback + def test_decimal_class_option + input = "3.141" + + # Without decimal_class: uses Ryu, returns Float + result_float = JSON.parse(input) + assert_instance_of(Float, result_float) + assert_equal(3.141, result_float) + + # With decimal_class: uses fallback, returns BigDecimal + result_bigdecimal = JSON.parse(input, decimal_class: BigDecimal) + assert_instance_of(BigDecimal, result_bigdecimal) + assert_equal(BigDecimal("3.141"), result_bigdecimal) + end if defined?(::BigDecimal) + + # Test that numbers with <= 17 digits use Ryu optimization + def test_ryu_optimization_used_for_normal_numbers + test_cases = [ + ["3.141", 3.141], + ["1.23456789012345e100", 1.23456789012345e100], + ["0.00000000000001", 1.0e-14], + ["123456789012345.67", 123456789012345.67], + ["-1.7976931348623157e+308", -1.7976931348623157e+308], + ["2.2250738585072014e-308", 2.2250738585072014e-308], + # Exactly 17 significant digits + ["12345678901234567", 12345678901234567.0], + ["1.2345678901234567", 1.2345678901234567], + ] + + test_cases.each do |input, expected| + result = JSON.parse(input) + assert_in_delta(expected, result, expected.abs * 1e-15, + "Failed to parse #{input} correctly (<=17 digits, Ryu path)") + end + end + + # Test edge cases at the boundary (17 digits) + def test_seventeen_digit_boundary + # Exactly 17 significant digits should use Ryu + input_17 = "12345678901234567.0" # Force it to be a float with .0 + result = JSON.parse(input_17) + assert_in_delta(12345678901234567.0, result, 1e-10) + + # 18 significant digits should use fallback + input_18 = "123456789012345678.0" + result = JSON.parse(input_18) + # Note: This will be rounded to double precision + assert_in_delta(123456789012345680.0, result, 1e-10) + end + + # Test that leading zeros don't count toward the 17-digit limit + def test_leading_zeros_dont_count + test_cases = [ + ["0.00012345678901234567", 0.00012345678901234567], # 17 significant digits + ["0.000000000000001234567890123456789", 1.234567890123457e-15], # >17 significant + ] + + test_cases.each do |input, expected| + result = JSON.parse(input) + assert_in_delta(expected, result, expected.abs * 1e-10, + "Failed to parse #{input} correctly") + end + end + + # Test that Ryu handles special values correctly + def test_special_double_values + test_cases = [ + ["1.7976931348623157e+308", Float::MAX], # Largest finite double + ["2.2250738585072014e-308", Float::MIN], # Smallest normalized double + ] + + test_cases.each do |input, expected| + result = JSON.parse(input) + assert_in_delta(expected, result, expected.abs * 1e-10, + "Failed to parse #{input} correctly") + end + + # Test zero separately + result_pos_zero = JSON.parse("0.0") + assert_equal(0.0, result_pos_zero) + + # Note: JSON.parse doesn't preserve -0.0 vs +0.0 distinction in standard mode + result_neg_zero = JSON.parse("-0.0") + assert_equal(0.0, result_neg_zero.abs) + end + + # Test subnormal numbers that caused precision issues before fallback was added + # These are extreme edge cases discovered by fuzzing (4 in 6 billion numbers tested) + def test_subnormal_edge_cases_round_trip + # These subnormal numbers (~1e-310) had 1 ULP rounding errors in original Ryu + # They now use rb_cstr_to_dbl fallback for exact precision + test_cases = [ + "-3.2652630314355e-310", + "3.9701623107025e-310", + "-3.6607772435415e-310", + "2.9714076801985e-310", + ] + + test_cases.each do |input| + # Parse the number + result = JSON.parse(input) + + # Should be bit-identical + assert_equal(result, JSON.parse(result.to_s), + "Subnormal #{input} failed round-trip test") + + # Should be bit-identical + assert_equal(result, JSON.parse(JSON.dump(result)), + "Subnormal #{input} failed round-trip test") + + # Verify the value is in the expected subnormal range + assert(result.abs < 2.225e-308, + "#{input} should be subnormal (< 2.225e-308)") + end + end + + # Test invalid numbers are properly rejected + def test_invalid_numbers_rejected + invalid_cases = [ + "-", + ".", + "-.", + "-.e10", + "1.2.3", + "1e", + "1e+", + ] + + invalid_cases.each do |input| + assert_raise(JSON::ParserError, "Should reject invalid number: #{input}") do + JSON.parse(input) + end + end + end +end From 53e90f30d0b9db3c5d3b36573d70e21c8eab091b Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 31 Oct 2025 08:48:14 +0900 Subject: [PATCH 180/506] [DOC] Fix typos in LEGAL --- LEGAL | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LEGAL b/LEGAL index 126ccc000..dfc7189b0 100644 --- a/LEGAL +++ b/LEGAL @@ -8,11 +8,11 @@ license (see the file COPYING) or public-domain except some files mentioned below. ext/json/ext/vendor/fpconv.h:: - This files is adapted from https://github.com/night-shift/fpconv + This file is adapted from https://github.com/night-shift/fpconv It is licensed under Boost Software License 1.0. ext/json/ext/vendor/jeaiii-ltoa.h:: - This files is adapted from https://github.com/jeaiii/itoa + This file is adapted from https://github.com/jeaiii/itoa It is licensed under the MIT License ext/json/ext/vendor/ryu.h:: From 37d1ee468b4e7fc81a7b036e76bb3a55fd4fa25d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 31 Oct 2025 09:52:30 +0900 Subject: [PATCH 181/506] Update depend files - Recently added ryu.h is missing in parser/depend. - Remove duplicate dependencies on the sources. --- ext/json/ext/generator/depend | 11 +++++++---- ext/json/ext/parser/depend | 8 ++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 14241d7f0..365c75afa 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1,4 +1,7 @@ -generator.o: generator.c $(srcdir)/../fbuffer/fbuffer.h -generator.o: generator.c $(srcdir)/../vendor/fpconv.c -generator.o: generator.c $(srcdir)/../vendor/jeaiii-ltoa.h -generator.o: generator.c $(srcdir)/../simd/simd.h +# AUTOGENERATED DEPENDENCIES START +generator.o: $(srcdir)/../fbuffer/fbuffer.h +generator.o: $(srcdir)/../simd/simd.h +generator.o: $(srcdir)/../vendor/fpconv.c +generator.o: $(srcdir)/../vendor/jeaiii-ltoa.h +generator.o: generator.c +# AUTOGENERATED DEPENDENCIES END diff --git a/ext/json/ext/parser/depend b/ext/json/ext/parser/depend index a1926b7d6..bb73bfa1e 100644 --- a/ext/json/ext/parser/depend +++ b/ext/json/ext/parser/depend @@ -1,2 +1,6 @@ -parser.o: parser.c $(srcdir)/../fbuffer/fbuffer.h -parser.o: parser.c $(srcdir)/../simd/simd.h +# AUTOGENERATED DEPENDENCIES START +parser.o: $(srcdir)/../fbuffer/fbuffer.h +parser.o: $(srcdir)/../simd/simd.h +parser.o: $(srcdir)/../vendor/ryu.h +parser.o: parser.c +# AUTOGENERATED DEPENDENCIES END From 9b7b648ecd2ab96a3b022ef5a3e5ebfe8faed9ea Mon Sep 17 00:00:00 2001 From: Kazuki Yamaguchi Date: Fri, 31 Oct 2025 23:52:23 +0900 Subject: [PATCH 182/506] Fix memory leak when exception is raised during JSON generation part 2 Commit 44df509dc2de fixed it for StandardError, but other exceptions and jumps are also possible. Use rb_ensure() to release FBuffer instead of rb_rescue(). A reproducer: o = Object.new def o.to_json(a) = throw :a a = ["make heap allocation"*100, o] 10.times do 100_000.times do catch(:a) { JSON(a) } end puts `ps -o rss= -p #{$$}` end --- ext/json/ext/fbuffer/fbuffer.h | 5 +---- ext/json/ext/generator/generator.c | 18 +++++------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index d5fd8ac6d..7d57a87b1 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -283,13 +283,10 @@ static VALUE fbuffer_finalize(FBuffer *fb) { if (fb->io) { fbuffer_flush(fb); - fbuffer_free(fb); rb_io_flush(fb->io); return fb->io; } else { - VALUE result = rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb)); - fbuffer_free(fb); - return result; + return rb_utf8_str_new(FBUFFER_PTR(fb), FBUFFER_LEN(fb)); } } diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d04c8a907..72155abe5 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1444,16 +1444,14 @@ static VALUE generate_json_try(VALUE d) data->func(data->buffer, data, data->obj); - return Qnil; + return fbuffer_finalize(data->buffer); } -static VALUE generate_json_rescue(VALUE d, VALUE exc) +static VALUE generate_json_ensure(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; fbuffer_free(data->buffer); - rb_exc_raise(exc); - return Qundef; } @@ -1474,9 +1472,7 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, .obj = obj, .func = func }; - rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); - - return fbuffer_finalize(&buffer); + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } /* call-seq: @@ -1522,9 +1518,7 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) .obj = obj, .func = generate_json }; - rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); - - return fbuffer_finalize(&buffer); + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } static VALUE cState_initialize(int argc, VALUE *argv, VALUE self) @@ -2030,9 +2024,7 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) .obj = obj, .func = generate_json, }; - rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); - - return fbuffer_finalize(&buffer); + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } /* From 2681b23b876094db58cdef5cc3559a43dd60f88b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 09:24:18 +0100 Subject: [PATCH 183/506] parser.c: Extract `json_parse_number` --- ext/json/ext/parser/parser.c | 184 ++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 89 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index ca8f50153..565f8e020 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1022,6 +1022,95 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig return Qfalse; } +static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start) +{ + bool integer = true; + + // Variables for Ryu optimization - extract digits during parsing + uint64_t mantissa = 0; + int mantissa_digits = 0; + int32_t exponent = 0; + int decimal_point_pos = -1; + + const char first_digit = *state->cursor; + + // Parse integer part and extract mantissa digits + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + mantissa = mantissa * 10 + (*state->cursor - '0'); + mantissa_digits++; + state->cursor++; + } + + if (RB_UNLIKELY(first_digit == '0' && mantissa_digits > 1 || negative && mantissa_digits == 0)) { + raise_parse_error_at("invalid number: %s", state, start); + } + + // Parse fractional part + if ((state->cursor < state->end) && (*state->cursor == '.')) { + integer = false; + decimal_point_pos = mantissa_digits; // Remember position of decimal point + state->cursor++; + + if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { + raise_parse_error_at("invalid number: %s", state, start); + } + + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + mantissa = mantissa * 10 + (*state->cursor - '0'); + mantissa_digits++; + state->cursor++; + } + } + + // Parse exponent + if ((state->cursor < state->end) && ((*state->cursor == 'e') || (*state->cursor == 'E'))) { + integer = false; + state->cursor++; + + bool negative_exponent = false; + if ((state->cursor < state->end) && ((*state->cursor == '-') || (*state->cursor == '+'))) { + negative_exponent = (*state->cursor == '-'); + state->cursor++; + } + + if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { + raise_parse_error_at("invalid number: %s", state, start); + } + + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + exponent = exponent * 10 + (*state->cursor - '0'); + state->cursor++; + } + + if (negative_exponent) { + exponent = -exponent; + } + } + + if (integer) { + return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor); + } + + // Adjust exponent based on decimal point position + if (decimal_point_pos >= 0) { + exponent -= (mantissa_digits - decimal_point_pos); + } + + return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor); +} + +static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config) +{ + return json_parse_number(state, config, false, state->cursor); +} + +static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config) +{ + const char *start = state->cursor; + state->cursor++; + return json_parse_number(state, config, true, start); +} + static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { json_eat_whitespace(state); @@ -1072,7 +1161,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); break; - case '-': + case '-': { // Note: memcmp with a small power of two compile to an integer comparison if ((state->end - state->cursor >= 9) && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { if (config->allow_nan) { @@ -1082,95 +1171,12 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); } } - // Fallthrough - case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - bool integer = true; - - // Variables for Ryu optimization - extract digits during parsing - uint64_t mantissa = 0; - int mantissa_digits = 0; - int32_t exponent = 0; - bool negative = false; - int decimal_point_pos = -1; - - // /\A-?(0|[1-9]\d*)(\.\d+)?([Ee][-+]?\d+)?/ - const char *start = state->cursor; - - // Handle optional negative sign - if (*state->cursor == '-') { - negative = true; - state->cursor++; - if (state->cursor >= state->end || !rb_isdigit(*state->cursor)) { - raise_parse_error_at("invalid number: %s", state, start); - } - } - - // Parse integer part and extract mantissa digits - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - mantissa = mantissa * 10 + (*state->cursor - '0'); - mantissa_digits++; - state->cursor++; - } - - if (RB_UNLIKELY(start[0] == '0' && mantissa_digits > 1)) { - raise_parse_error_at("invalid number: %s", state, start); - } else if (RB_UNLIKELY(mantissa_digits > 1 && negative && start[1] == '0')) { - raise_parse_error_at("invalid number: %s", state, start); - } - - // Parse fractional part - if ((state->cursor < state->end) && (*state->cursor == '.')) { - integer = false; - decimal_point_pos = mantissa_digits; // Remember position of decimal point - state->cursor++; - - if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { - raise_parse_error_at("invalid number: %s", state, start); - } - - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - mantissa = mantissa * 10 + (*state->cursor - '0'); - mantissa_digits++; - state->cursor++; - } - } - - // Parse exponent - if ((state->cursor < state->end) && ((*state->cursor == 'e') || (*state->cursor == 'E'))) { - integer = false; - state->cursor++; - - bool negative_exponent = false; - if ((state->cursor < state->end) && ((*state->cursor == '-') || (*state->cursor == '+'))) { - negative_exponent = (*state->cursor == '-'); - state->cursor++; - } - - if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { - raise_parse_error_at("invalid number: %s", state, start); - } - - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - exponent = exponent * 10 + (*state->cursor - '0'); - state->cursor++; - } - - if (negative_exponent) { - exponent = -exponent; - } - } - - if (integer) { - return json_push_value(state, config, json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor)); - } - - // Adjust exponent based on decimal point position - if (decimal_point_pos >= 0) { - exponent -= (mantissa_digits - decimal_point_pos); - } - - return json_push_value(state, config, json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor)); + return json_push_value(state, config, json_parse_negative_number(state, config)); + break; } + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + return json_push_value(state, config, json_parse_positive_number(state, config)); + break; case '"': { // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} return json_parse_string(state, config, false); From 1bf405ecc697c6cc7ce3092b33c29a2a279df331 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 09:42:24 +0100 Subject: [PATCH 184/506] parser.c: Extract json_parse_digits --- ext/json/ext/parser/parser.c | 49 +++++++++++++++++------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 565f8e020..d69cc28a9 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1022,24 +1022,28 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig return Qfalse; } +static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulator) +{ + const char *start = state->cursor; + while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { + *accumulator = *accumulator * 10 + (*state->cursor - '0'); + state->cursor++; + } + return (int)(state->cursor - start); +} + static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start) { bool integer = true; + const char first_digit = *state->cursor; // Variables for Ryu optimization - extract digits during parsing - uint64_t mantissa = 0; - int mantissa_digits = 0; int32_t exponent = 0; int decimal_point_pos = -1; - - const char first_digit = *state->cursor; + uint64_t mantissa = 0; // Parse integer part and extract mantissa digits - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - mantissa = mantissa * 10 + (*state->cursor - '0'); - mantissa_digits++; - state->cursor++; - } + int mantissa_digits = json_parse_digits(state, &mantissa); if (RB_UNLIKELY(first_digit == '0' && mantissa_digits > 1 || negative && mantissa_digits == 0)) { raise_parse_error_at("invalid number: %s", state, start); @@ -1051,19 +1055,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig decimal_point_pos = mantissa_digits; // Remember position of decimal point state->cursor++; - if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { - raise_parse_error_at("invalid number: %s", state, start); - } + int fractional_digits = json_parse_digits(state, &mantissa); + mantissa_digits += fractional_digits; - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - mantissa = mantissa * 10 + (*state->cursor - '0'); - mantissa_digits++; - state->cursor++; + if (RB_UNLIKELY(!fractional_digits)) { + raise_parse_error_at("invalid number: %s", state, start); } } // Parse exponent - if ((state->cursor < state->end) && ((*state->cursor == 'e') || (*state->cursor == 'E'))) { + if ((state->cursor < state->end) && ((rb_tolower(*state->cursor) == 'e'))) { integer = false; state->cursor++; @@ -1073,18 +1074,14 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig state->cursor++; } - if (state->cursor == state->end || !rb_isdigit(*state->cursor)) { - raise_parse_error_at("invalid number: %s", state, start); - } + uint64_t abs_exponent = 0; + int exponent_digits = json_parse_digits(state, &abs_exponent); - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - exponent = exponent * 10 + (*state->cursor - '0'); - state->cursor++; + if (RB_UNLIKELY(!exponent_digits)) { + raise_parse_error_at("invalid number: %s", state, start); } - if (negative_exponent) { - exponent = -exponent; - } + exponent = negative_exponent ? -((int32_t)abs_exponent) : ((int32_t)abs_exponent); } if (integer) { From 8b394072254f9b92a38da86f63b1ab03330462db Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 10:27:15 +0100 Subject: [PATCH 185/506] parser.c: Introduce `peek()` and `eos()` helpers Encapsulate pointer arithmetic to reduce possibility of mistakes. --- ext/json/ext/parser/parser.c | 154 +++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 69 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d69cc28a9..90460f3a2 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -406,6 +406,18 @@ typedef struct JSON_ParserStateStruct { int current_nesting; } JSON_ParserState; +static inline bool eos(JSON_ParserState *state) { + return state->cursor >= state->end; +} + +static inline char peek(JSON_ParserState *state) +{ + if (RB_UNLIKELY(eos(state))) { + return 0; + } + return *state->cursor; +} + static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out) { const char *cursor = state->cursor; @@ -571,7 +583,7 @@ json_eat_comments(JSON_ParserState *state) raise_parse_error_at("unexpected end of input, expected closing '*/'", state, state->end); } else { state->cursor++; - if (state->cursor < state->end && *state->cursor == '/') { + if (peek(state) == '/') { state->cursor++; break; } @@ -591,11 +603,12 @@ json_eat_comments(JSON_ParserState *state) static inline void json_eat_whitespace(JSON_ParserState *state) { - while (state->cursor < state->end && RB_UNLIKELY(whitespace[(unsigned char)*state->cursor])) { - if (RB_LIKELY(*state->cursor != '/')) { - state->cursor++; - } else { + unsigned char cursor; + while (RB_UNLIKELY(whitespace[cursor = (unsigned char)peek(state)])) { + if (RB_UNLIKELY(cursor == '/')) { json_eat_comments(state); + } else { + state->cursor++; } } } @@ -980,7 +993,7 @@ static inline bool FORCE_INLINE string_scan(JSON_ParserState *state) #endif /* HAVE_SIMD_NEON or HAVE_SIMD_SSE2 */ #endif /* HAVE_SIMD */ - while (state->cursor < state->end) { + while (!eos(state)) { if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) { return 1; } @@ -1025,8 +1038,10 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulator) { const char *start = state->cursor; - while ((state->cursor < state->end) && rb_isdigit(*state->cursor)) { - *accumulator = *accumulator * 10 + (*state->cursor - '0'); + char next_char; + + while (rb_isdigit(next_char = peek(state))) { + *accumulator = *accumulator * 10 + (next_char - '0'); state->cursor++; } return (int)(state->cursor - start); @@ -1050,7 +1065,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } // Parse fractional part - if ((state->cursor < state->end) && (*state->cursor == '.')) { + if (peek(state) == '.') { integer = false; decimal_point_pos = mantissa_digits; // Remember position of decimal point state->cursor++; @@ -1064,13 +1079,14 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } // Parse exponent - if ((state->cursor < state->end) && ((rb_tolower(*state->cursor) == 'e'))) { + if (rb_tolower(peek(state)) == 'e') { integer = false; state->cursor++; bool negative_exponent = false; - if ((state->cursor < state->end) && ((*state->cursor == '-') || (*state->cursor == '+'))) { - negative_exponent = (*state->cursor == '-'); + const char next_char = peek(state); + if (next_char == '-' || next_char == '+') { + negative_exponent = next_char == '-'; state->cursor++; } @@ -1111,11 +1127,8 @@ static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_Par static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { json_eat_whitespace(state); - if (state->cursor >= state->end) { - raise_parse_error("unexpected end of input", state); - } - switch (*state->cursor) { + switch (peek(state)) { case 'n': if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "null", 4) == 0)) { state->cursor += 4; @@ -1184,7 +1197,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_eat_whitespace(state); long stack_head = state->stack->head; - if ((state->cursor < state->end) && (*state->cursor == ']')) { + if (peek(state) == ']') { state->cursor++; return json_push_value(state, config, json_decode_array(state, config, 0)); } else { @@ -1199,26 +1212,26 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) while (true) { json_eat_whitespace(state); - if (state->cursor < state->end) { - if (*state->cursor == ']') { - state->cursor++; - long count = state->stack->head - stack_head; - state->current_nesting--; - state->in_array--; - return json_push_value(state, config, json_decode_array(state, config, count)); - } + const char next_char = peek(state); - if (*state->cursor == ',') { - state->cursor++; - if (config->allow_trailing_comma) { - json_eat_whitespace(state); - if ((state->cursor < state->end) && (*state->cursor == ']')) { - continue; - } + if (RB_LIKELY(next_char == ',')) { + state->cursor++; + if (config->allow_trailing_comma) { + json_eat_whitespace(state); + if (peek(state) == ']') { + continue; } - json_parse_any(state, config); - continue; } + json_parse_any(state, config); + continue; + } + + if (next_char == ']') { + state->cursor++; + long count = state->stack->head - stack_head; + state->current_nesting--; + state->in_array--; + return json_push_value(state, config, json_decode_array(state, config, count)); } raise_parse_error("expected ',' or ']' after array value", state); @@ -1232,7 +1245,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_eat_whitespace(state); long stack_head = state->stack->head; - if ((state->cursor < state->end) && (*state->cursor == '}')) { + if (peek(state) == '}') { state->cursor++; return json_push_value(state, config, json_decode_object(state, config, 0)); } else { @@ -1241,13 +1254,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); } - if (*state->cursor != '"') { + if (peek(state) != '"') { raise_parse_error("expected object key, got %s", state); } json_parse_string(state, config, true); json_eat_whitespace(state); - if ((state->cursor >= state->end) || (*state->cursor != ':')) { + if (peek(state) != ':') { raise_parse_error("expected ':' after object key", state); } state->cursor++; @@ -1258,46 +1271,45 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) while (true) { json_eat_whitespace(state); - if (state->cursor < state->end) { - if (*state->cursor == '}') { - state->cursor++; - state->current_nesting--; - size_t count = state->stack->head - stack_head; + const char next_char = peek(state); + if (next_char == '}') { + state->cursor++; + state->current_nesting--; + size_t count = state->stack->head - stack_head; - // Temporary rewind cursor in case an error is raised - const char *final_cursor = state->cursor; - state->cursor = object_start_cursor; - VALUE object = json_decode_object(state, config, count); - state->cursor = final_cursor; + // Temporary rewind cursor in case an error is raised + const char *final_cursor = state->cursor; + state->cursor = object_start_cursor; + VALUE object = json_decode_object(state, config, count); + state->cursor = final_cursor; - return json_push_value(state, config, object); - } + return json_push_value(state, config, object); + } - if (*state->cursor == ',') { - state->cursor++; - json_eat_whitespace(state); + if (next_char == ',') { + state->cursor++; + json_eat_whitespace(state); - if (config->allow_trailing_comma) { - if ((state->cursor < state->end) && (*state->cursor == '}')) { - continue; - } + if (config->allow_trailing_comma) { + if (peek(state) == '}') { + continue; } + } - if (*state->cursor != '"') { - raise_parse_error("expected object key, got: %s", state); - } - json_parse_string(state, config, true); + if (RB_UNLIKELY(peek(state) != '"')) { + raise_parse_error("expected object key, got: %s", state); + } + json_parse_string(state, config, true); - json_eat_whitespace(state); - if ((state->cursor >= state->end) || (*state->cursor != ':')) { - raise_parse_error("expected ':' after object key, got: %s", state); - } - state->cursor++; + json_eat_whitespace(state); + if (RB_UNLIKELY(peek(state) != ':')) { + raise_parse_error("expected ':' after object key, got: %s", state); + } + state->cursor++; - json_parse_any(state, config); + json_parse_any(state, config); - continue; - } + continue; } raise_parse_error("expected ',' or '}' after object value, got: %s", state); @@ -1305,6 +1317,10 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; } + case 0: + raise_parse_error("unexpected end of input", state); + break; + default: raise_parse_error("unexpected character: %s", state); break; @@ -1316,7 +1332,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) static void json_ensure_eof(JSON_ParserState *state) { json_eat_whitespace(state); - if (state->cursor != state->end) { + if (!eos(state)) { raise_parse_error("unexpected token at end of stream %s", state); } } From f0150e2944933d077919171835df7df2f01f3add Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 31 Oct 2025 08:31:11 +0100 Subject: [PATCH 186/506] Add test coverage for T_BIGNUM parsing --- test/json/json_parser_test.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 9d387cb80..30188c4eb 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -131,6 +131,12 @@ def test_parse_numbers capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) } end + def test_parse_bignum + bignum = Integer('1234567890' * 10) + assert_equal(bignum, JSON.parse(bignum.to_s)) + assert_equal(bignum.to_f, JSON.parse(bignum.to_s + ".0")) + end + def test_parse_bigdecimals assert_equal(BigDecimal, JSON.parse('{"foo": 9.01234567890123456789}', decimal_class: BigDecimal)["foo"].class) assert_equal(BigDecimal("0.901234567890123456789E1"),JSON.parse('{"foo": 9.01234567890123456789}', decimal_class: BigDecimal)["foo"] ) From 11f4e7b7be1582b351c6f0f5d40e41f27c7bb64a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 10:54:03 +0100 Subject: [PATCH 187/506] parser.c: Introduce `rest()` helper --- ext/json/ext/parser/parser.c | 74 +++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 90460f3a2..3bd654dc3 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -406,6 +406,10 @@ typedef struct JSON_ParserStateStruct { int current_nesting; } JSON_ParserState; +static inline ssize_t rest(JSON_ParserState *state) { + return state->end - state->cursor; +} + static inline bool eos(JSON_ParserState *state) { return state->cursor >= state->end; } @@ -564,39 +568,39 @@ static const bool whitespace[256] = { static void json_eat_comments(JSON_ParserState *state) { - if (state->cursor + 1 < state->end) { - switch (state->cursor[1]) { - case '/': { - state->cursor = memchr(state->cursor, '\n', state->end - state->cursor); - if (!state->cursor) { - state->cursor = state->end; - } else { - state->cursor++; - } - break; + const char *start = state->cursor; + state->cursor++; + + switch (peek(state)) { + case '/': { + state->cursor = memchr(state->cursor, '\n', state->end - state->cursor); + if (!state->cursor) { + state->cursor = state->end; + } else { + state->cursor++; } - case '*': { - state->cursor += 2; - while (true) { - state->cursor = memchr(state->cursor, '*', state->end - state->cursor); - if (!state->cursor) { - raise_parse_error_at("unexpected end of input, expected closing '*/'", state, state->end); - } else { - state->cursor++; - if (peek(state) == '/') { - state->cursor++; - break; - } - } + break; + } + case '*': { + state->cursor++; + + while (true) { + const char *next_match = memchr(state->cursor, '*', state->end - state->cursor); + if (!next_match) { + raise_parse_error_at("unterminated comment, expected closing '*/'", state, start); + } + + state->cursor = next_match + 1; + if (peek(state) == '/') { + state->cursor++; + break; } - break; } - default: - raise_parse_error("unexpected token %s", state); - break; + break; } - } else { - raise_parse_error("unexpected token %s", state); + default: + raise_parse_error_at("unexpected token %s", state, start); + break; } } @@ -1130,7 +1134,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) switch (peek(state)) { case 'n': - if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "null", 4) == 0)) { + if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) { state->cursor += 4; return json_push_value(state, config, Qnil); } @@ -1138,7 +1142,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); break; case 't': - if ((state->end - state->cursor >= 4) && (memcmp(state->cursor, "true", 4) == 0)) { + if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) { state->cursor += 4; return json_push_value(state, config, Qtrue); } @@ -1147,7 +1151,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; case 'f': // Note: memcmp with a small power of two compile to an integer comparison - if ((state->end - state->cursor >= 5) && (memcmp(state->cursor + 1, "alse", 4) == 0)) { + if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) { state->cursor += 5; return json_push_value(state, config, Qfalse); } @@ -1156,7 +1160,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; case 'N': // Note: memcmp with a small power of two compile to an integer comparison - if (config->allow_nan && (state->end - state->cursor >= 3) && (memcmp(state->cursor + 1, "aN", 2) == 0)) { + if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) { state->cursor += 3; return json_push_value(state, config, CNaN); } @@ -1164,7 +1168,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); break; case 'I': - if (config->allow_nan && (state->end - state->cursor >= 8) && (memcmp(state->cursor, "Infinity", 8) == 0)) { + if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) { state->cursor += 8; return json_push_value(state, config, CInfinity); } @@ -1173,7 +1177,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) break; case '-': { // Note: memcmp with a small power of two compile to an integer comparison - if ((state->end - state->cursor >= 9) && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { + if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { if (config->allow_nan) { state->cursor += 9; return json_push_value(state, config, CMinusInfinity); From 6348ff08913a07ae529453c91ff49499c2ebe716 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 11:06:32 +0100 Subject: [PATCH 188/506] Use SWAR for parsing integers on little endian machines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/ruby/json/pull/878 ``` == Parsing float parsing (2251051 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 23.000 i/100ms Calculating ------------------------------------- after 214.382 (± 0.5%) i/s (4.66 ms/i) - 1.081k in 5.042555s Comparison: before: 189.5 i/s after: 214.4 i/s - 1.13x faster ``` Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 52 +++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 3bd654dc3..e591ca2c5 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1039,11 +1039,61 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig return Qfalse; } +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +// From: https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/ +// Additional References: +// https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ +// http://0x80.pl/notesen/2014-10-12-parsing-decimal-numbers-part-1-swar.html +static inline uint64_t decode_8digits_unrolled(uint64_t val) { + const uint64_t mask = 0x000000FF000000FF; + const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) + const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) + val -= 0x3030303030303030; + val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; + val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; + return val; +} + +static inline uint64_t decode_4digits_unrolled(uint32_t val) { + const uint32_t mask = 0x000000FF; + const uint32_t mul1 = 100; + val -= 0x30303030; + val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; + val = ((val & mask) * mul1) + (((val >> 16) & mask)); + return val; +} +#endif + static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulator) { const char *start = state->cursor; - char next_char; +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + while (rest(state) >= 8) { + uint64_t next_8bytes; + memcpy(&next_8bytes, state->cursor, sizeof(uint64_t)); + + // From: https://github.com/simdjson/simdjson/blob/32b301893c13d058095a07d9868edaaa42ee07aa/include/simdjson/generic/numberparsing.h#L333 + // Branchless version of: http://0x80.pl/articles/swar-digits-validate.html + uint64_t match = (next_8bytes & 0xF0F0F0F0F0F0F0F0) | (((next_8bytes + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4); + + if (match == 0x3333333333333333) { // 8 consecutive digits + *accumulator = (*accumulator * 100000000) + decode_8digits_unrolled(next_8bytes); + state->cursor += 8; + continue; + } + + if ((match & 0xFFFFFFFF) == 0x33333333) { // 4 consecutive digits + *accumulator = (*accumulator * 10000) + decode_4digits_unrolled((uint32_t)next_8bytes); + state->cursor += 4; + break; + } + + break; + } +#endif + + char next_char; while (rb_isdigit(next_char = peek(state))) { *accumulator = *accumulator * 10 + (next_char - '0'); state->cursor++; From b3fd7b26be63c76711dcd70df79453fa0175cd9d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 12:15:04 +0100 Subject: [PATCH 189/506] parser.c: Use SWAR to skip consecutive spaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/ruby/json/pull/881 If we encounter a newline, it is likely that the document is pretty printed, hence that the newline is followed by multiple spaces. In such case we can use SWAR to count up to eight consecutive spaces at once. ``` == Parsing activitypub.json (58160 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 1.118k i/100ms Calculating ------------------------------------- after 11.223k (± 0.7%) i/s (89.10 μs/i) - 57.018k in 5.080522s Comparison: before: 10834.4 i/s after: 11223.4 i/s - 1.04x faster == Parsing twitter.json (567916 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 118.000 i/100ms Calculating ------------------------------------- after 1.188k (± 1.0%) i/s (841.62 μs/i) - 6.018k in 5.065355s Comparison: before: 1094.8 i/s after: 1188.2 i/s - 1.09x faster == Parsing citm_catalog.json (1727030 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 58.000 i/100ms Calculating ------------------------------------- after 570.506 (± 3.7%) i/s (1.75 ms/i) - 2.900k in 5.091529s Comparison: before: 419.6 i/s after: 570.5 i/s - 1.36x faster == Parsing float parsing (2251051 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 22.000 i/100ms Calculating ------------------------------------- after 212.010 (± 1.9%) i/s (4.72 ms/i) - 1.078k in 5.086885s Comparison: before: 189.4 i/s after: 212.0 i/s - 1.12x faster ``` Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 46 +++++++++++++++++++++++++----------- ext/json/ext/simd/simd.h | 3 +-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e591ca2c5..01234b5a8 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -557,14 +557,6 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p static const rb_data_type_t JSON_ParserConfig_type; -static const bool whitespace[256] = { - [' '] = 1, - ['\t'] = 1, - ['\n'] = 1, - ['\r'] = 1, - ['/'] = 1, -}; - static void json_eat_comments(JSON_ParserState *state) { @@ -607,12 +599,38 @@ json_eat_comments(JSON_ParserState *state) static inline void json_eat_whitespace(JSON_ParserState *state) { - unsigned char cursor; - while (RB_UNLIKELY(whitespace[cursor = (unsigned char)peek(state)])) { - if (RB_UNLIKELY(cursor == '/')) { - json_eat_comments(state); - } else { - state->cursor++; + while (true) { + switch (peek(state)) { + case ' ': + state->cursor++; + break; + case '\n': + state->cursor++; + + // Heuristic: if we see a newline, there is likely consecutive spaces after it. +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + while (rest(state) > 8) { + uint64_t chunk; + memcpy(&chunk, state->cursor, sizeof(uint64_t)); + size_t consecutive_spaces = trailing_zeros64(chunk ^ 0x2020202020202020) / CHAR_BIT; + + state->cursor += consecutive_spaces; + if (consecutive_spaces != 8) { + break; + } + } +#endif + break; + case '\t': + case '\r': + state->cursor++; + break; + case '/': + json_eat_comments(state); + break; + + default: + return; } } } diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 3abbdb020..2aa6c3d04 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -4,8 +4,6 @@ typedef enum { SIMD_SSE2 } SIMD_Implementation; -#ifdef JSON_ENABLE_SIMD - #ifdef __clang__ # if __has_builtin(__builtin_ctzll) # define HAVE_BUILTIN_CTZLL 1 @@ -54,6 +52,7 @@ static inline int trailing_zeros(int input) #define FORCE_INLINE #endif +#ifdef JSON_ENABLE_SIMD #define SIMD_MINIMUM_THRESHOLD 6 From ded62a5122699d0870a0c7901424b00b185975f1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 13:05:18 +0100 Subject: [PATCH 190/506] parser.c: Appease GCC warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` ../../../../../../ext/json/ext/parser/parser.c:1142:40: warning: suggest parentheses around ‘&&’ within ‘||’ [-Wparentheses] 1142 | if (RB_UNLIKELY(first_digit == '0' && mantissa_digits > 1 || negative && mantissa_digits == 0)) { | ~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~ ``` --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 01234b5a8..555652f42 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1132,7 +1132,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig // Parse integer part and extract mantissa digits int mantissa_digits = json_parse_digits(state, &mantissa); - if (RB_UNLIKELY(first_digit == '0' && mantissa_digits > 1 || negative && mantissa_digits == 0)) { + if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) { raise_parse_error_at("invalid number: %s", state, start); } From 3ea744ad67680c641949175f0bcce26efb6c1e35 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 14:57:33 +0100 Subject: [PATCH 191/506] Enable JSON_DEBUG for parser/extconf.rb --- ext/json/ext/parser/extconf.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index de5d5758b..dc1c8952c 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'mkmf' +$defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby From 82b030f2941c4aa084e374fe0e956ff7732828f2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 14:57:33 +0100 Subject: [PATCH 192/506] Enable JSON_DEBUG for parser/extconf.rb --- ext/json/ext/parser/parser.c | 24 +++++++++++++++++------- ext/json/ext/simd/simd.h | 12 ++++++++++++ test/json/json_parser_test.rb | 4 ++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 555652f42..8206716d7 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -612,12 +612,14 @@ json_eat_whitespace(JSON_ParserState *state) while (rest(state) > 8) { uint64_t chunk; memcpy(&chunk, state->cursor, sizeof(uint64_t)); - size_t consecutive_spaces = trailing_zeros64(chunk ^ 0x2020202020202020) / CHAR_BIT; + if (chunk == 0x2020202020202020) { + state->cursor += 8; + continue; + } + uint32_t consecutive_spaces = trailing_zeros64(chunk ^ 0x2020202020202020) / CHAR_BIT; state->cursor += consecutive_spaces; - if (consecutive_spaces != 8) { - break; - } + break; } #endif break; @@ -1101,13 +1103,21 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat continue; } - if ((match & 0xFFFFFFFF) == 0x33333333) { // 4 consecutive digits + uint32_t consecutive_digits = trailing_zeros64(match ^ 0x3333333333333333) / CHAR_BIT; + + if (consecutive_digits >= 4) { *accumulator = (*accumulator * 10000) + decode_4digits_unrolled((uint32_t)next_8bytes); state->cursor += 4; - break; + consecutive_digits -= 4; + } + + while (consecutive_digits) { + *accumulator = *accumulator * 10 + (*state->cursor - '0'); + consecutive_digits--; + state->cursor++; } - break; + return (int)(state->cursor - start); } #endif diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 2aa6c3d04..0abe4fad6 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -1,3 +1,7 @@ +#ifdef JSON_DEBUG +#include +#endif + typedef enum { SIMD_NONE, SIMD_NEON, @@ -18,6 +22,10 @@ typedef enum { static inline uint32_t trailing_zeros64(uint64_t input) { +#ifdef JSON_DEBUG + assert(input > 0); // __builtin_ctz(0) is undefined behavior +#endif + #if HAVE_BUILTIN_CTZLL return __builtin_ctzll(input); #else @@ -33,6 +41,10 @@ static inline uint32_t trailing_zeros64(uint64_t input) static inline int trailing_zeros(int input) { +#ifdef JSON_DEBUG + assert(input > 0); // __builtin_ctz(0) is undefined behavior +#endif + #if HAVE_BUILTIN_CTZLL return __builtin_ctz(input); #else diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 30188c4eb..bab16a6fc 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -804,6 +804,10 @@ def test_parse_leading_slash end end + def test_parse_whitespace_after_newline + assert_equal [], JSON.parse("[\n#{' ' * (8 + 8 + 4 + 3)}]") + end + private def assert_equal_float(expected, actual, delta = 1e-2) From ee41f1534753f13c126dbd988ad3912a4b870d76 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 17:12:37 +0100 Subject: [PATCH 193/506] Update changelog --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 858792f52..82278fbbe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,9 @@ ### Unreleased -* Optimized floating point number parsing by integrating the ryu algorithm. +* Optimized floating point number parsing by integrating the ryu algorithm (thanks to Josef Šimánek). +* Optimized numbers parsing using SWAR (thanks to Scott Myron). +* Optimized parsing of pretty printed documents using SWAR (thanks to Scott Myron). ### 2025-10-25 (2.15.2) From 9bb1dc67d025ba7b7820801f3e32f0f031fd2df0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 1 Nov 2025 17:14:50 +0100 Subject: [PATCH 194/506] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 82278fbbe..d8915b132 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix GeneratorError messages to be UTF-8 encoded. +* Fix memory leak when `Exception` is raised, or `throw` is used during JSON generation. * Optimized floating point number parsing by integrating the ryu algorithm (thanks to Josef Šimánek). * Optimized numbers parsing using SWAR (thanks to Scott Myron). * Optimized parsing of pretty printed documents using SWAR (thanks to Scott Myron). From b1b16c416f2bb48bdbd30ce3e94a18544775c62c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 30 Oct 2025 10:29:33 +0100 Subject: [PATCH 195/506] Invoke `as_json` callback for strings with invalid encoding Fix: https://github.com/ruby/json/issues/873 This allow users to encode binary strings if they so wish. e.g. they can use Base64 or similar, or chose to replace invalid characters with something else. --- CHANGES.md | 1 + README.md | 18 +- ext/json/ext/generator/generator.c | 237 ++++++++++++++++----------- java/src/json/ext/Generator.java | 17 +- java/src/json/ext/StringEncoder.java | 11 ++ lib/json/truffle_ruby/generator.rb | 34 +++- test/json/json_coder_test.rb | 65 ++++++++ 7 files changed, 279 insertions(+), 104 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d8915b132..6ad2da7c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* `JSON::Coder` now also yields to the block when encountering strings with invalid encoding. * Fix GeneratorError messages to be UTF-8 encoded. * Fix memory leak when `Exception` is raised, or `throw` is used during JSON generation. * Optimized floating point number parsing by integrating the ryu algorithm (thanks to Josef Šimánek). diff --git a/README.md b/README.md index eed71ba43..fee99305e 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,23 @@ puts MyApp::API_JSON_CODER.dump(Time.now.utc) # => "2025-01-21T08:41:44.286Z" The provided block is called for all objects that don't have a native JSON equivalent, and must return a Ruby object that has a native JSON equivalent. -It is also called for objects that do have a JSON equivalent, but are used as Hash keys, for instance `{ 1 => 2}`. +It is also called for objects that do have a JSON equivalent, but are used as Hash keys, for instance `{ 1 => 2}`, +as well as for strings that aren't valid UTF-8: + +```ruby +coder = JSON::Combining.new do |object, is_object_key| + case object + when String + if !string.valid_encoding? || string.encoding != Encoding::UTF_8 + Base64.encode64(string) + else + string + end + else + object + end +end +``` ## Combining JSON fragments diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 72155abe5..024a85727 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -996,13 +996,12 @@ static inline VALUE vstate_get(struct generate_json_data *data) return data->vstate; } -struct hash_foreach_arg { - VALUE hash; - struct generate_json_data *data; - int first_key_type; - bool first; - bool mixed_keys_encountered; -}; +static VALUE +json_call_as_json(JSON_Generator_State *state, VALUE object, VALUE is_key) +{ + VALUE proc_args[2] = {object, is_key}; + return rb_proc_call_with_block(state->as_json, 2, proc_args, Qnil); +} static VALUE convert_string_subclass(VALUE key) @@ -1019,6 +1018,129 @@ convert_string_subclass(VALUE key) return key_to_s; } +static bool enc_utf8_compatible_p(int enc_idx) +{ + if (enc_idx == usascii_encindex) return true; + if (enc_idx == utf8_encindex) return true; + return false; +} + +static VALUE encode_json_string_try(VALUE str) +{ + return rb_funcall(str, i_encode, 1, Encoding_UTF_8); +} + +static VALUE encode_json_string_rescue(VALUE str, VALUE exception) +{ + raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0)); + return Qundef; +} + +static inline bool valid_json_string_p(VALUE str) +{ + int coderange = rb_enc_str_coderange(str); + + if (RB_LIKELY(coderange == ENC_CODERANGE_7BIT)) { + return true; + } + + if (RB_LIKELY(coderange == ENC_CODERANGE_VALID)) { + return enc_utf8_compatible_p(RB_ENCODING_GET_INLINED(str)); + } + + return false; +} + +static inline VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) +{ + if (RB_LIKELY(valid_json_string_p(str))) { + return str; + } + + if (!as_json_called && data->state->strict && RTEST(data->state->as_json)) { + VALUE coerced_str = json_call_as_json(data->state, str, Qfalse); + if (coerced_str != str) { + if (RB_TYPE_P(coerced_str, T_STRING)) { + if (!valid_json_string_p(coerced_str)) { + raise_generator_error(str, "source sequence is illegal/malformed utf-8"); + } + } else { + // as_json could return another type than T_STRING + if (is_key) { + raise_generator_error(coerced_str, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(coerced_str)); + } + } + + return coerced_str; + } + } + + if (RB_ENCODING_GET_INLINED(str) == binary_encindex) { + VALUE utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex); + switch (rb_enc_str_coderange(utf8_string)) { + case ENC_CODERANGE_7BIT: + return utf8_string; + case ENC_CODERANGE_VALID: + // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work. + // TODO: Raise in 3.0.0 + rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0"); + return utf8_string; + break; + } + } + + return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); +} + +static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +{ + fbuffer_append_char(buffer, '"'); + + long len; + search_state search; + search.buffer = buffer; + RSTRING_GETMEM(obj, search.ptr, len); + search.cursor = search.ptr; + search.end = search.ptr + len; + +#ifdef HAVE_SIMD + search.matches_mask = 0; + search.has_matches = false; + search.chunk_base = NULL; +#endif /* HAVE_SIMD */ + + switch (rb_enc_str_coderange(obj)) { + case ENC_CODERANGE_7BIT: + case ENC_CODERANGE_VALID: + if (RB_UNLIKELY(data->state->ascii_only)) { + convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table); + } else if (RB_UNLIKELY(data->state->script_safe)) { + convert_UTF8_to_script_safe_JSON(&search); + } else { + convert_UTF8_to_JSON(&search); + } + break; + default: + raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); + break; + } + fbuffer_append_char(buffer, '"'); +} + +static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +{ + obj = ensure_valid_encoding(data, obj, false, false); + raw_generate_json_string(buffer, data, obj); +} + +struct hash_foreach_arg { + VALUE hash; + struct generate_json_data *data; + int first_key_type; + bool first; + bool mixed_keys_encountered; +}; + NOINLINE() static void json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) @@ -1035,13 +1157,6 @@ json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) } } -static VALUE -json_call_as_json(JSON_Generator_State *state, VALUE object, VALUE is_key) -{ - VALUE proc_args[2] = {object, is_key}; - return rb_proc_call_with_block(state->as_json, 2, proc_args, Qnil); -} - static int json_object_i(VALUE key, VALUE val, VALUE _arg) { @@ -1107,8 +1222,10 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) break; } + key_to_s = ensure_valid_encoding(data, key_to_s, as_json_called, true); + if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) { - generate_json_string(buffer, data, key_to_s); + raw_generate_json_string(buffer, data, key_to_s); } else { generate_json(buffer, data, key_to_s); } @@ -1191,85 +1308,6 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data fbuffer_append_char(buffer, ']'); } -static inline int enc_utf8_compatible_p(int enc_idx) -{ - if (enc_idx == usascii_encindex) return 1; - if (enc_idx == utf8_encindex) return 1; - return 0; -} - -static VALUE encode_json_string_try(VALUE str) -{ - return rb_funcall(str, i_encode, 1, Encoding_UTF_8); -} - -static VALUE encode_json_string_rescue(VALUE str, VALUE exception) -{ - raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0)); - return Qundef; -} - -static inline VALUE ensure_valid_encoding(VALUE str) -{ - int encindex = RB_ENCODING_GET(str); - VALUE utf8_string; - if (RB_UNLIKELY(!enc_utf8_compatible_p(encindex))) { - if (encindex == binary_encindex) { - utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex); - switch (rb_enc_str_coderange(utf8_string)) { - case ENC_CODERANGE_7BIT: - return utf8_string; - case ENC_CODERANGE_VALID: - // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work. - // TODO: Raise in 3.0.0 - rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0"); - return utf8_string; - break; - } - } - - str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); - } - return str; -} - -static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) -{ - obj = ensure_valid_encoding(obj); - - fbuffer_append_char(buffer, '"'); - - long len; - search_state search; - search.buffer = buffer; - RSTRING_GETMEM(obj, search.ptr, len); - search.cursor = search.ptr; - search.end = search.ptr + len; - -#ifdef HAVE_SIMD - search.matches_mask = 0; - search.has_matches = false; - search.chunk_base = NULL; -#endif /* HAVE_SIMD */ - - switch (rb_enc_str_coderange(obj)) { - case ENC_CODERANGE_7BIT: - case ENC_CODERANGE_VALID: - if (RB_UNLIKELY(data->state->ascii_only)) { - convert_UTF8_to_ASCII_only_JSON(&search, data->state->script_safe ? script_safe_escape_table : ascii_only_escape_table); - } else if (RB_UNLIKELY(data->state->script_safe)) { - convert_UTF8_to_script_safe_JSON(&search); - } else { - convert_UTF8_to_JSON(&search); - } - break; - default: - raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); - break; - } - fbuffer_append_char(buffer, '"'); -} - static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { VALUE tmp; @@ -1408,7 +1446,16 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALU break; case T_STRING: if (klass != rb_cString) goto general; - generate_json_string(buffer, data, obj); + + if (RB_LIKELY(valid_json_string_p(obj))) { + raw_generate_json_string(buffer, data, obj); + } else if (as_json_called) { + raise_generator_error(obj, "source sequence is illegal/malformed utf-8"); + } else { + obj = ensure_valid_encoding(data, obj, false, false); + as_json_called = true; + goto start; + } break; case T_SYMBOL: generate_json_symbol(buffer, data, obj); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index be0e82579..78281dbd4 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -606,6 +606,8 @@ private static IRubyObject castKey(ThreadContext context, IRubyObject key) { } private static void processEntry(ThreadContext context, Session session, OutputStream buffer, RubyHash.RubyHashEntry entry, boolean firstPair, ByteList objectNl, byte[] indent, ByteList spaceBefore, ByteList space) { + StringEncoder encoder = session.getStringEncoder(context); + IRubyObject key = (IRubyObject) entry.getKey(); IRubyObject value = (IRubyObject) entry.getValue(); @@ -619,7 +621,7 @@ private static void processEntry(ThreadContext context, Session session, OutputS Ruby runtime = context.runtime; IRubyObject keyStr = castKey(context, key); - if (keyStr == null || !(keyStr instanceof RubyString)) { + if (keyStr == null || !(keyStr instanceof RubyString) || !encoder.hasValidEncoding((RubyString)keyStr)) { GeneratorState state = session.getState(context); if (state.strict()) { if (state.getAsJSON() != null) { @@ -664,6 +666,19 @@ int guessSize(ThreadContext context, Session session, RubyString object) { @Override void generate(ThreadContext context, Session session, RubyString object, OutputStream buffer) throws IOException { + GeneratorState state = session.getState(context); + StringEncoder encoder = session.getStringEncoder(context); + + if (state.strict() && !encoder.hasValidEncoding(object) && state.getAsJSON() != null) { + IRubyObject value = state.getAsJSON().call(context, object, context.getRuntime().getFalse()); + if (value instanceof RubyString) { + object = (RubyString)value; + } else { + Handler handler = getHandlerFor(context.runtime, value); + handler.generate(context, session, value, buffer); + return; + } + } generateString(context, session, object, buffer); } } diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index b874ad785..4eea11c62 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -208,6 +208,17 @@ void generate(ThreadContext context, RubyString object, OutputStream buffer) thr append('"'); } + static boolean hasValidEncoding(RubyString str) { + switch (str.scanForCodeRange()) { + case StringSupport.CR_7BIT: + return true; + case StringSupport.CR_VALID: + return str.getEncoding() == UTF8Encoding.INSTANCE || str.getEncoding() == USASCIIEncoding.INSTANCE; + default: + return false; + } + } + static RubyString ensureValidEncoding(ThreadContext context, RubyString str) { Encoding encoding = str.getEncoding(); diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 703211cbc..feed4c795 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -55,6 +55,11 @@ def self.native_key?(key) # :nodoc: (Symbol === key || String === key) end + def self.valid_encoding?(string) # :nodoc: + return false unless string.encoding == ::Encoding::UTF_8 || string.encoding == ::Encoding::US_ASCII + string.is_a?(Symbol) || string.valid_encoding? + end + # Convert a UTF8 encoded Ruby string _string_ to a JSON string, encoded with # UTF16 big endian characters as \u????, and return it. def self.utf8_to_json(string, script_safe = false) # :nodoc: @@ -521,13 +526,17 @@ def json_transform(state) end result << state.indent * depth if indent - if state.strict? && !Generator.native_key?(key) - if state.as_json + if state.strict? + if state.as_json && (!Generator.native_key?(key) || !Generator.valid_encoding?(key)) key = state.as_json.call(key, true) end unless Generator.native_key?(key) - raise GeneratorError.new("#{key.class} not allowed as object key in JSON", value) + raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) + end + + unless Generator.valid_encoding?(key) + raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) end end @@ -674,14 +683,25 @@ module String # \u????. def to_json(state = nil, *args) state = State.from_state(state) - if encoding == ::Encoding::UTF_8 - unless valid_encoding? + string = self + + if state.strict? && state.as_json + unless Generator.valid_encoding?(string) + string = state.as_json.call(string, false) + unless string.is_a?(::String) + return string.to_json(state, *args) + end + end + end + + if string.encoding == ::Encoding::UTF_8 + unless string.valid_encoding? raise GeneratorError.new("source sequence is illegal/malformed utf-8", self) end - string = self else - string = encode(::Encoding::UTF_8) + string = string.encode(::Encoding::UTF_8) end + if state.ascii_only? %("#{JSON::TruffleRuby::Generator.utf8_to_json_ascii(string, state.script_safe)}") else diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index fb9d7b30a..83b89a3b1 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -67,6 +67,71 @@ def test_json_coder_dump_NaN_or_Infinity_loop assert_include error.message, "NaN not allowed in JSON" end + def test_json_coder_string_invalid_encoding + calls = 0 + coder = JSON::Coder.new do |object, is_key| + calls += 1 + object + end + + error = assert_raise JSON::GeneratorError do + coder.dump("\xFF") + end + assert_equal "source sequence is illegal/malformed utf-8", error.message + assert_equal 1, calls + + error = assert_raise JSON::GeneratorError do + coder.dump({ "\xFF" => 1 }) + end + assert_equal "source sequence is illegal/malformed utf-8", error.message + assert_equal 2, calls + + calls = 0 + coder = JSON::Coder.new do |object, is_key| + calls += 1 + object.dup + end + + error = assert_raise JSON::GeneratorError do + coder.dump("\xFF") + end + assert_equal "source sequence is illegal/malformed utf-8", error.message + assert_equal 1, calls + + error = assert_raise JSON::GeneratorError do + coder.dump({ "\xFF" => 1 }) + end + assert_equal "source sequence is illegal/malformed utf-8", error.message + assert_equal 2, calls + + calls = 0 + coder = JSON::Coder.new do |object, is_key| + calls += 1 + object.bytes + end + + assert_equal "[255]", coder.dump("\xFF") + assert_equal 1, calls + + error = assert_raise JSON::GeneratorError do + coder.dump({ "\xFF" => 1 }) + end + assert_equal "Array not allowed as object key in JSON", error.message + assert_equal 2, calls + + calls = 0 + coder = JSON::Coder.new do |object, is_key| + calls += 1 + [object].pack("m") + end + + assert_equal '"/w==\\n"', coder.dump("\xFF") + assert_equal 1, calls + + assert_equal '{"/w==\\n":1}', coder.dump({ "\xFF" => 1 }) + assert_equal 2, calls + end + def test_nesting_recovery coder = JSON::Coder.new ary = [] From 2b8f853589a5898bbfddb5286d59f191ddd66ef9 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 09:36:31 +0100 Subject: [PATCH 196/506] Avoid power_assert 3.0 on Ruby 3.0 and older Ref: https://github.com/test-unit/test-unit/pull/329 --- Gemfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Gemfile b/Gemfile index c02e4d494..85e8f54eb 100644 --- a/Gemfile +++ b/Gemfile @@ -11,6 +11,11 @@ group :development do gem "test-unit-ruby-core" gem "all_images", "~> 0" unless RUBY_PLATFORM =~ /java/ gem "simplecov", require: false + + if RUBY_VERSION < "3.1." + # Ref: https://github.com/test-unit/test-unit/pull/329 + gem 'power_assert', '< 3.0.0' + end end group :release do From 1576ea7d4784b4eee6dba4b31c5802432f618fe2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 10:07:28 +0100 Subject: [PATCH 197/506] Centralize macro definitions --- ext/json/ext/fbuffer/fbuffer.h | 50 +----------------- ext/json/ext/generator/depend | 1 + ext/json/ext/generator/generator.c | 34 ++++-------- ext/json/ext/json.h | 85 ++++++++++++++++++++++++++++++ ext/json/ext/parser/depend | 1 + ext/json/ext/parser/parser.c | 49 +---------------- ext/json/ext/simd/simd.h | 28 +++------- 7 files changed, 108 insertions(+), 140 deletions(-) create mode 100644 ext/json/ext/json.h diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 7d57a87b1..ecba61465 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -1,55 +1,9 @@ #ifndef _FBUFFER_H_ #define _FBUFFER_H_ -#include "ruby.h" -#include "ruby/encoding.h" +#include "../json.h" #include "../vendor/jeaiii-ltoa.h" -/* shims */ -/* This is the fallback definition from Ruby 3.4 */ - -#ifndef RBIMPL_STDBOOL_H -#if defined(__cplusplus) -# if defined(HAVE_STDBOOL_H) && (__cplusplus >= 201103L) -# include -# endif -#elif defined(HAVE_STDBOOL_H) -# include -#elif !defined(HAVE__BOOL) -typedef unsigned char _Bool; -# define bool _Bool -# define true ((_Bool)+1) -# define false ((_Bool)+0) -# define __bool_true_false_are_defined -#endif -#endif - -#ifndef NOINLINE -#if defined(__has_attribute) && __has_attribute(noinline) -#define NOINLINE() __attribute__((noinline)) -#else -#define NOINLINE() -#endif -#endif - -#ifndef RB_UNLIKELY -#define RB_UNLIKELY(expr) expr -#endif - -#ifndef RB_LIKELY -#define RB_LIKELY(expr) expr -#endif - -#ifndef MAYBE_UNUSED -# define MAYBE_UNUSED(x) x -#endif - -#ifdef RUBY_DEBUG -#ifndef JSON_DEBUG -#define JSON_DEBUG RUBY_DEBUG -#endif -#endif - enum fbuffer_type { FBUFFER_HEAP_ALLOCATED = 0, FBUFFER_STACK_ALLOCATED = 1, @@ -290,4 +244,4 @@ static VALUE fbuffer_finalize(FBuffer *fb) } } -#endif +#endif // _FBUFFER_H_ diff --git a/ext/json/ext/generator/depend b/ext/json/ext/generator/depend index 365c75afa..43fd7d33a 100644 --- a/ext/json/ext/generator/depend +++ b/ext/json/ext/generator/depend @@ -1,5 +1,6 @@ # AUTOGENERATED DEPENDENCIES START generator.o: $(srcdir)/../fbuffer/fbuffer.h +generator.o: $(srcdir)/../json.h generator.o: $(srcdir)/../simd/simd.h generator.o: $(srcdir)/../vendor/fpconv.c generator.o: $(srcdir)/../vendor/jeaiii-ltoa.h diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 024a85727..56aa636ef 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1,4 +1,4 @@ -#include "ruby.h" +#include "../json.h" #include "../fbuffer/fbuffer.h" #include "../vendor/fpconv.c" @@ -36,10 +36,6 @@ typedef struct JSON_Generator_StateStruct { bool strict; } JSON_Generator_State; -#ifndef RB_UNLIKELY -#define RB_UNLIKELY(cond) (cond) -#endif - static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8; static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode; @@ -85,10 +81,7 @@ static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *d static int usascii_encindex, utf8_encindex, binary_encindex; -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_generator_error_str(VALUE invalid_object, VALUE str) +NORETURN(static void) raise_generator_error_str(VALUE invalid_object, VALUE str) { rb_enc_associate_index(str, utf8_encindex); VALUE exc = rb_exc_new_str(eGeneratorError, str); @@ -96,13 +89,10 @@ static void raise_generator_error_str(VALUE invalid_object, VALUE str) rb_exc_raise(exc); } -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif #ifdef RBIMPL_ATTR_FORMAT RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 2, 3) #endif -static void raise_generator_error(VALUE invalid_object, const char *fmt, ...) +NORETURN(static void) raise_generator_error(VALUE invalid_object, const char *fmt, ...) { va_list args; va_start(args, fmt); @@ -137,13 +127,7 @@ typedef struct _search_state { #endif /* HAVE_SIMD */ } search_state; -#if (defined(__GNUC__ ) || defined(__clang__)) -#define FORCE_INLINE __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - -static inline FORCE_INLINE void search_flush(search_state *search) +static ALWAYS_INLINE() void search_flush(search_state *search) { // Do not remove this conditional without profiling, specifically escape-heavy text. // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush). @@ -186,7 +170,7 @@ static inline unsigned char search_escape_basic(search_state *search) return 0; } -static inline FORCE_INLINE void escape_UTF8_char_basic(search_state *search) +static ALWAYS_INLINE() void escape_UTF8_char_basic(search_state *search) { const unsigned char ch = (unsigned char)*search->ptr; switch (ch) { @@ -273,7 +257,7 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) #ifdef HAVE_SIMD -static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) +static ALWAYS_INLINE() char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) { // Flush the buffer so everything up until the last 'len' characters are unflushed. search_flush(search); @@ -296,7 +280,7 @@ static inline FORCE_INLINE char *copy_remaining_bytes(search_state *search, unsi #ifdef HAVE_SIMD_NEON -static inline FORCE_INLINE unsigned char neon_next_match(search_state *search) +static ALWAYS_INLINE() unsigned char neon_next_match(search_state *search) { uint64_t mask = search->matches_mask; uint32_t index = trailing_zeros64(mask) >> 2; @@ -410,7 +394,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) #ifdef HAVE_SIMD_SSE2 -static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search) +static ALWAYS_INLINE() unsigned char sse2_next_match(search_state *search) { int mask = search->matches_mask; int index = trailing_zeros(mask); @@ -434,7 +418,7 @@ static inline FORCE_INLINE unsigned char sse2_next_match(search_state *search) #define TARGET_SSE2 #endif -static inline TARGET_SSE2 FORCE_INLINE unsigned char search_escape_basic_sse2(search_state *search) +static inline TARGET_SSE2 ALWAYS_INLINE() unsigned char search_escape_basic_sse2(search_state *search) { if (RB_UNLIKELY(search->has_matches)) { // There are more matches if search->matches_mask > 0. diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h new file mode 100644 index 000000000..873440527 --- /dev/null +++ b/ext/json/ext/json.h @@ -0,0 +1,85 @@ +#ifndef _JSON_H_ +#define _JSON_H_ + +#include "ruby.h" +#include "ruby/encoding.h" + +#if defined(RUBY_DEBUG) && RUBY_DEBUG +# define JSON_ASSERT RUBY_ASSERT +#else +# ifdef JSON_DEBUG +# include +# define JSON_ASSERT(x) assert(x) +# else +# define JSON_ASSERT(x) +# endif +#endif + +/* shims */ + +#if SIZEOF_UINT64_T == SIZEOF_LONG_LONG +# define INT64T2NUM(x) LL2NUM(x) +# define UINT64T2NUM(x) ULL2NUM(x) +#elif SIZEOF_UINT64_T == SIZEOF_LONG +# define INT64T2NUM(x) LONG2NUM(x) +# define UINT64T2NUM(x) ULONG2NUM(x) +#else +# error No uint64_t conversion +#endif + +/* This is the fallback definition from Ruby 3.4 */ +#ifndef RBIMPL_STDBOOL_H +#if defined(__cplusplus) +# if defined(HAVE_STDBOOL_H) && (__cplusplus >= 201103L) +# include +# endif +#elif defined(HAVE_STDBOOL_H) +# include +#elif !defined(HAVE__BOOL) +typedef unsigned char _Bool; +# define bool _Bool +# define true ((_Bool)+1) +# define false ((_Bool)+0) +# define __bool_true_false_are_defined +#endif +#endif + +#ifndef NORETURN +#define NORETURN(x) x +#endif + +#ifndef NOINLINE +#if defined(__has_attribute) && __has_attribute(noinline) +#define NOINLINE(x) __attribute__((noinline)) x +#else +#define NOINLINE(x) x +#endif +#endif + +#ifndef ALWAYS_INLINE +#if defined(__has_attribute) && __has_attribute(always_inline) +#define ALWAYS_INLINE(x) inline __attribute__((always_inline)) x +#else +#define ALWAYS_INLINE(x) inline x +#endif +#endif + +#ifndef RB_UNLIKELY +#define RB_UNLIKELY(expr) expr +#endif + +#ifndef RB_LIKELY +#define RB_LIKELY(expr) expr +#endif + +#ifndef MAYBE_UNUSED +# define MAYBE_UNUSED(x) x +#endif + +#ifdef RUBY_DEBUG +#ifndef JSON_DEBUG +#define JSON_DEBUG RUBY_DEBUG +#endif +#endif + +#endif // _JSON_H_ \ No newline at end of file diff --git a/ext/json/ext/parser/depend b/ext/json/ext/parser/depend index bb73bfa1e..5d75ef248 100644 --- a/ext/json/ext/parser/depend +++ b/ext/json/ext/parser/depend @@ -1,5 +1,6 @@ # AUTOGENERATED DEPENDENCIES START parser.o: $(srcdir)/../fbuffer/fbuffer.h +parser.o: $(srcdir)/../json.h parser.o: $(srcdir)/../simd/simd.h parser.o: $(srcdir)/../vendor/ryu.h parser.o: parser.c diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 8206716d7..d9b578f24 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1,46 +1,7 @@ -#include "ruby.h" -#include "ruby/encoding.h" +#include "../json.h" #include "../vendor/ryu.h" - -/* shims */ -/* This is the fallback definition from Ruby 3.4 */ - -#ifndef RBIMPL_STDBOOL_H -#if defined(__cplusplus) -# if defined(HAVE_STDBOOL_H) && (__cplusplus >= 201103L) -# include -# endif -#elif defined(HAVE_STDBOOL_H) -# include -#elif !defined(HAVE__BOOL) -typedef unsigned char _Bool; -# define bool _Bool -# define true ((_Bool)+1) -# define false ((_Bool)+0) -# define __bool_true_false_are_defined -#endif -#endif - -#if SIZEOF_UINT64_T == SIZEOF_LONG_LONG -# define INT64T2NUM(x) LL2NUM(x) -# define UINT64T2NUM(x) ULL2NUM(x) -#elif SIZEOF_UINT64_T == SIZEOF_LONG -# define INT64T2NUM(x) LONG2NUM(x) -# define UINT64T2NUM(x) ULONG2NUM(x) -#else -# error No uint64_t conversion -#endif - #include "../simd/simd.h" -#ifndef RB_UNLIKELY -#define RB_UNLIKELY(expr) expr -#endif - -#ifndef RB_LIKELY -#define RB_LIKELY(expr) expr -#endif - static VALUE mJSON, eNestingError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; @@ -985,17 +946,11 @@ static const bool string_scan_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -#if (defined(__GNUC__ ) || defined(__clang__)) -#define FORCE_INLINE __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - #ifdef HAVE_SIMD static SIMD_Implementation simd_impl = SIMD_NONE; #endif /* HAVE_SIMD */ -static inline bool FORCE_INLINE string_scan(JSON_ParserState *state) +static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) { #ifdef HAVE_SIMD #if defined(HAVE_SIMD_NEON) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 0abe4fad6..194baee51 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -1,6 +1,4 @@ -#ifdef JSON_DEBUG -#include -#endif +#include "../json.h" typedef enum { SIMD_NONE, @@ -22,9 +20,7 @@ typedef enum { static inline uint32_t trailing_zeros64(uint64_t input) { -#ifdef JSON_DEBUG - assert(input > 0); // __builtin_ctz(0) is undefined behavior -#endif + JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior #if HAVE_BUILTIN_CTZLL return __builtin_ctzll(input); @@ -41,9 +37,7 @@ static inline uint32_t trailing_zeros64(uint64_t input) static inline int trailing_zeros(int input) { -#ifdef JSON_DEBUG - assert(input > 0); // __builtin_ctz(0) is undefined behavior -#endif + JSON_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior #if HAVE_BUILTIN_CTZLL return __builtin_ctz(input); @@ -58,12 +52,6 @@ static inline int trailing_zeros(int input) #endif } -#if (defined(__GNUC__ ) || defined(__clang__)) -#define FORCE_INLINE __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - #ifdef JSON_ENABLE_SIMD #define SIMD_MINIMUM_THRESHOLD 6 @@ -81,14 +69,14 @@ static inline SIMD_Implementation find_simd_implementation(void) #define HAVE_SIMD_NEON 1 // See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon -static inline FORCE_INLINE uint64_t neon_match_mask(uint8x16_t matches) +static ALWAYS_INLINE() uint64_t neon_match_mask(uint8x16_t matches) { const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4); const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); return mask & 0x8888888888888888ull; } -static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr) +static ALWAYS_INLINE() uint64_t compute_chunk_mask_neon(const char *ptr) { uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); @@ -101,7 +89,7 @@ static inline FORCE_INLINE uint64_t compute_chunk_mask_neon(const char *ptr) return neon_match_mask(needs_escape); } -static inline FORCE_INLINE int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) +static ALWAYS_INLINE() int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) { while (*ptr + sizeof(uint8x16_t) <= end) { uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); @@ -148,7 +136,7 @@ static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) #define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) #define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) -static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *ptr) +static inline TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr) { __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 @@ -159,7 +147,7 @@ static inline TARGET_SSE2 FORCE_INLINE int compute_chunk_mask_sse2(const char *p return _mm_movemask_epi8(needs_escape); } -static inline TARGET_SSE2 FORCE_INLINE int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) +static inline TARGET_SSE2 ALWAYS_INLINE() int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) { while (*ptr + sizeof(__m128i) <= end) { int chunk_mask = compute_chunk_mask_sse2(*ptr); From b8cdf3282d7c61c0cc49344ae391b55a42641105 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 10:46:20 +0100 Subject: [PATCH 198/506] parser.c: Skip checking for escape sequences in `rstring_cache_fetch` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caller already know if the string contains escape sequences so this check is redundant. Also stop calling `rstring_cache_fetch` from `json_string_unescape` as we know it won't match anyways. ``` == Parsing twitter.json (567916 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 122.000 i/100ms Calculating ------------------------------------- after 1.226k (± 0.3%) i/s (815.85 μs/i) - 6.222k in 5.076282s Comparison: before: 1206.2 i/s after: 1225.7 i/s - 1.02x faster ``` Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 25 ------------------------- test/json/json_parser_test.rb | 12 ++++++++++++ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d9b578f24..68ff6d7ac 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -122,12 +122,6 @@ static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const lon } } - if (RB_UNLIKELY(memchr(str, '\\', length))) { - // We assume the overwhelming majority of names don't need to be escaped. - // But if they do, we have to fallback to the slow path. - return Qfalse; - } - VALUE rstring = build_interned_string(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { @@ -174,12 +168,6 @@ static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const lon } } - if (RB_UNLIKELY(memchr(str, '\\', length))) { - // We assume the overwhelming majority of names don't need to be escaped. - // But if they do, we have to fallback to the slow path. - return Qfalse; - } - VALUE rsymbol = build_symbol(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { @@ -652,19 +640,6 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c int unescape_len; char buf[4]; - if (is_name && state->in_array) { - VALUE cached_key; - if (RB_UNLIKELY(symbolize)) { - cached_key = rsymbol_cache_fetch(&state->name_cache, string, bufferSize); - } else { - cached_key = rstring_cache_fetch(&state->name_cache, string, bufferSize); - } - - if (RB_LIKELY(cached_key)) { - return cached_key; - } - } - VALUE result = rb_str_buf_new(bufferSize); rb_enc_associate_index(result, utf8_encindex); buffer = RSTRING_PTR(result); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index bab16a6fc..6315c3e66 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -344,6 +344,18 @@ def test_parse_big_integers assert_equal orig, parse(json5) end + def test_parse_escaped_key + doc = { + "test\r1" => 1, + "entries" => [ + "test\t2" => 2, + "test\n3" => 3, + ] + } + + assert_equal doc, parse(JSON.generate(doc)) + end + def test_parse_duplicate_key expected = {"a" => 2} expected_sym = {a: 2} From 31453b8e953a923c089accba3552d4e19aae3573 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 11:10:56 +0100 Subject: [PATCH 199/506] parser.c: simplify sorted insert loop in rstring_cache_fetch Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 68ff6d7ac..caa4f9fa0 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -105,17 +105,15 @@ static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const lon int low = 0; int high = cache->length - 1; - int mid = 0; - int last_cmp = 0; while (low <= high) { - mid = (high + low) >> 1; + int mid = (high + low) >> 1; VALUE entry = cache->entries[mid]; - last_cmp = rstring_cache_cmp(str, length, entry); + int cmp = rstring_cache_cmp(str, length, entry); - if (last_cmp == 0) { + if (cmp == 0) { return entry; - } else if (last_cmp > 0) { + } else if (cmp > 0) { low = mid + 1; } else { high = mid - 1; @@ -125,11 +123,7 @@ static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const lon VALUE rstring = build_interned_string(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { - if (last_cmp > 0) { - mid += 1; - } - - rvalue_cache_insert_at(cache, mid, rstring); + rvalue_cache_insert_at(cache, low, rstring); } return rstring; } @@ -151,17 +145,15 @@ static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const lon int low = 0; int high = cache->length - 1; - int mid = 0; - int last_cmp = 0; while (low <= high) { - mid = (high + low) >> 1; + int mid = (high + low) >> 1; VALUE entry = cache->entries[mid]; - last_cmp = rstring_cache_cmp(str, length, rb_sym2str(entry)); + int cmp = rstring_cache_cmp(str, length, rb_sym2str(entry)); - if (last_cmp == 0) { + if (cmp == 0) { return entry; - } else if (last_cmp > 0) { + } else if (cmp > 0) { low = mid + 1; } else { high = mid - 1; @@ -171,11 +163,7 @@ static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const lon VALUE rsymbol = build_symbol(str, length); if (cache->length < JSON_RVALUE_CACHE_CAPA) { - if (last_cmp > 0) { - mid += 1; - } - - rvalue_cache_insert_at(cache, mid, rsymbol); + rvalue_cache_insert_at(cache, low, rsymbol); } return rsymbol; } From ac580458e0a2b8377e2e6a4b64c126e0d1eee2fb Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 11:32:58 +0100 Subject: [PATCH 200/506] parser.c: Extract `json_string_cacheable_p` We can share that logic between the two functions. --- ext/json/ext/parser/parser.c | 37 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index caa4f9fa0..1e83dad91 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -90,19 +90,6 @@ static inline int rstring_cache_cmp(const char *str, const long length, VALUE rs static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) { - if (RB_UNLIKELY(length > JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH)) { - // Common names aren't likely to be very long. So we just don't - // cache names above an arbitrary threshold. - return Qfalse; - } - - if (RB_UNLIKELY(!rb_isalpha((unsigned char)str[0]))) { - // Simple heuristic, if the first character isn't a letter, - // we're much less likely to see this string again. - // We mostly want to cache strings that are likely to be repeated. - return Qfalse; - } - int low = 0; int high = cache->length - 1; @@ -130,19 +117,6 @@ static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const lon static VALUE rsymbol_cache_fetch(rvalue_cache *cache, const char *str, const long length) { - if (RB_UNLIKELY(length > JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH)) { - // Common names aren't likely to be very long. So we just don't - // cache names above an arbitrary threshold. - return Qfalse; - } - - if (RB_UNLIKELY(!rb_isalpha((unsigned char)str[0]))) { - // Simple heuristic, if the first character isn't a letter, - // we're much less likely to see this string again. - // We mostly want to cache strings that are likely to be repeated. - return Qfalse; - } - int low = 0; int high = cache->length - 1; @@ -600,11 +574,20 @@ static inline VALUE build_string(const char *start, const char *end, bool intern return result; } +static inline bool json_string_cacheable_p(const char *string, size_t length) +{ + // We mostly want to cache strings that are likely to be repeated. + // Simple heuristics: + // - Common names aren't likely to be very long. So we just don't cache names above an arbitrary threshold. + // - If the first character isn't a letter, we're much less likely to see this string again. + return length <= JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH && rb_isalpha(string[0]); +} + static inline VALUE json_string_fastpath(JSON_ParserState *state, const char *string, const char *stringEnd, bool is_name, bool intern, bool symbolize) { size_t bufferSize = stringEnd - string; - if (is_name && state->in_array) { + if (is_name && state->in_array && RB_LIKELY(json_string_cacheable_p(string, bufferSize))) { VALUE cached_key; if (RB_UNLIKELY(symbolize)) { cached_key = rsymbol_cache_fetch(&state->name_cache, string, bufferSize); From 21284ea6490e8c2fb077d90c17f587c2479edaee Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 11:45:43 +0100 Subject: [PATCH 201/506] parser.c: use `rb_str_to_interned_str` over `rb_funcall` --- ext/json/ext/parser/extconf.rb | 1 + ext/json/ext/parser/parser.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index dc1c8952c..cda385767 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -3,6 +3,7 @@ $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 +have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby have_func("strnlen", "string.h") # Missing on Solaris 10 diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1e83dad91..25eeb89e7 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -16,7 +16,7 @@ static int utf8_encindex; #ifndef HAVE_RB_HASH_BULK_INSERT // For TruffleRuby -void +static void rb_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash) { long index = 0; @@ -33,6 +33,12 @@ rb_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash) #define rb_hash_new_capa(n) rb_hash_new() #endif +#ifndef HAVE_RB_STR_TO_INTERNED_STR +static VALUE rb_str_to_interned_str(VALUE str) +{ + return rb_funcall(rb_str_freeze(str), i_uminus, 0); +} +#endif /* name cache */ @@ -703,7 +709,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c if (symbolize) { result = rb_str_intern(result); } else if (intern) { - result = rb_funcall(rb_str_freeze(result), i_uminus, 0); + result = rb_str_to_interned_str(result); } return result; @@ -1310,6 +1316,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } raise_parse_error("unreachable: %s", state); + return Qundef; } static void json_ensure_eof(JSON_ParserState *state) From 043880f6abd966d515b223df2a401fc5f48feb40 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 14:10:02 +0100 Subject: [PATCH 202/506] parser.c: Always inline `json_eat_whitespace` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` == Parsing activitypub.json (58160 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 1.174k i/100ms Calculating ------------------------------------- after 11.756k (± 0.9%) i/s (85.06 μs/i) - 59.874k in 5.093438s Comparison: before: 11078.6 i/s after: 11756.1 i/s - 1.06x faster == Parsing twitter.json (567916 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 130.000 i/100ms Calculating ------------------------------------- after 1.340k (± 0.3%) i/s (746.06 μs/i) - 6.760k in 5.043432s Comparison: before: 1191.1 i/s after: 1340.4 i/s - 1.13x faster == Parsing citm_catalog.json (1727030 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 68.000 i/100ms Calculating ------------------------------------- after 689.451 (± 1.6%) i/s (1.45 ms/i) - 3.468k in 5.031470s Comparison: before: 630.3 i/s after: 689.5 i/s - 1.09x faster == Parsing float parsing (2251051 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 27.000 i/100ms Calculating ------------------------------------- after 248.265 (± 0.8%) i/s (4.03 ms/i) - 1.242k in 5.003185s Comparison: before: 232.7 i/s after: 248.3 i/s - 1.07x faster ``` Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 25eeb89e7..20754249e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -513,7 +513,7 @@ json_eat_comments(JSON_ParserState *state) } } -static inline void +static ALWAYS_INLINE() void json_eat_whitespace(JSON_ParserState *state) { while (true) { From 591510392a6d5f794470b45576ca006f58a74f6d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 16:55:42 +0100 Subject: [PATCH 203/506] Fix duplicate 'inline' declaration specifier Followup: https://github.com/ruby/json/pull/889 --- ext/json/ext/generator/generator.c | 2 +- ext/json/ext/simd/simd.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 56aa636ef..8930213b9 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -418,7 +418,7 @@ static ALWAYS_INLINE() unsigned char sse2_next_match(search_state *search) #define TARGET_SSE2 #endif -static inline TARGET_SSE2 ALWAYS_INLINE() unsigned char search_escape_basic_sse2(search_state *search) +static TARGET_SSE2 ALWAYS_INLINE() unsigned char search_escape_basic_sse2(search_state *search) { if (RB_UNLIKELY(search->has_matches)) { // There are more matches if search->matches_mask > 0. diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 194baee51..a4f917fd0 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -136,7 +136,7 @@ static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) #define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) #define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) -static inline TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr) +static TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr) { __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 @@ -147,7 +147,7 @@ static inline TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char return _mm_movemask_epi8(needs_escape); } -static inline TARGET_SSE2 ALWAYS_INLINE() int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) +static TARGET_SSE2 ALWAYS_INLINE() int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) { while (*ptr + sizeof(__m128i) <= end) { int chunk_mask = compute_chunk_mask_sse2(*ptr); From 3bc1787bd413953bbfad237d06082a0c905ef945 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 3 Nov 2025 18:02:10 +0100 Subject: [PATCH 204/506] ext/json/ext/json.h: Add missing newline at end of file --- ext/json/ext/json.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 873440527..20e7101de 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -82,4 +82,4 @@ typedef unsigned char _Bool; #endif #endif -#endif // _JSON_H_ \ No newline at end of file +#endif // _JSON_H_ From ddad00b7469c04f58fd29779ba84bf75a8283515 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 4 Nov 2025 09:03:35 +0100 Subject: [PATCH 205/506] Extract `JSON_CPU_LITTLE_ENDIAN_64BITS` definition Only apply these definitions on 64 bits archs, as it's unclear if they have performance benefits or compatibility issues on 32bit archs. --- ext/json/ext/json.h | 7 +++++++ ext/json/ext/parser/parser.c | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 20e7101de..01fe0cd03 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -3,6 +3,7 @@ #include "ruby.h" #include "ruby/encoding.h" +#include #if defined(RUBY_DEBUG) && RUBY_DEBUG # define JSON_ASSERT RUBY_ASSERT @@ -82,4 +83,10 @@ typedef unsigned char _Bool; #endif #endif +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && INTPTR_MAX == INT64_MAX +#define JSON_CPU_LITTLE_ENDIAN_64BITS 1 +#else +#define JSON_CPU_LITTLE_ENDIAN_64BITS 0 +#endif + #endif // _JSON_H_ diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 20754249e..0f4d31097 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -323,7 +323,7 @@ typedef struct JSON_ParserStateStruct { int current_nesting; } JSON_ParserState; -static inline ssize_t rest(JSON_ParserState *state) { +static inline size_t rest(JSON_ParserState *state) { return state->end - state->cursor; } @@ -525,7 +525,7 @@ json_eat_whitespace(JSON_ParserState *state) state->cursor++; // Heuristic: if we see a newline, there is likely consecutive spaces after it. -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#if JSON_CPU_LITTLE_ENDIAN_64BITS while (rest(state) > 8) { uint64_t chunk; memcpy(&chunk, state->cursor, sizeof(uint64_t)); @@ -966,7 +966,7 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig return Qfalse; } -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#if JSON_CPU_LITTLE_ENDIAN_64BITS // From: https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/ // Additional References: // https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ @@ -995,8 +995,8 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat { const char *start = state->cursor; -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - while (rest(state) >= 8) { +#if JSON_CPU_LITTLE_ENDIAN_64BITS + while (rest(state) >= sizeof(uint64_t)) { uint64_t next_8bytes; memcpy(&next_8bytes, state->cursor, sizeof(uint64_t)); From a67d1a1af481693304dfa894901f8a8b186c1221 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 4 Nov 2025 09:08:05 +0100 Subject: [PATCH 206/506] Micro-optimize `rstring_cache_fetch` Closes: https://github.com/ruby/json/pull/888 - Mark it as `inline`. - Use `RSTRING_GETMEM`, instead of `RSTRING_LEN` and `RSTRING_PTR`. - Use an inlinable version of `memcmp`. ``` == Parsing activitypub.json (58160 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Comparison: before: 11766.6 i/s after: 12272.1 i/s - 1.04x faster == Parsing twitter.json (567916 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Comparison: before: 1333.2 i/s after: 1422.0 i/s - 1.07x faster == Parsing citm_catalog.json (1727030 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Comparison: before: 656.3 i/s after: 673.1 i/s - 1.03x faster == Parsing float parsing (2251051 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Comparison: before: 276.8 i/s after: 276.4 i/s - same-ish: difference falls within error ``` Co-Authored-By: Scott Myron --- ext/json/ext/parser/parser.c | 43 ++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 0f4d31097..3d88a399e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -84,17 +84,52 @@ static void rvalue_cache_insert_at(rvalue_cache *cache, int index, VALUE rstring cache->entries[index] = rstring; } -static inline int rstring_cache_cmp(const char *str, const long length, VALUE rstring) +#if JSON_CPU_LITTLE_ENDIAN_64BITS && defined(__has_builtin) && __has_builtin(__builtin_bswap64) +static ALWAYS_INLINE() int rstring_cache_memcmp(const char *str, const char *rptr, const long length) { - long rstring_length = RSTRING_LEN(rstring); + // The libc memcmp has numerous complex optimizations, but in this particular case, + // we know the string is small (JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH), so being able to + // inline a simpler memcmp outperforms calling the libc version. + long i = 0; + + for (; i + 8 <= length; i += 8) { + uint64_t a, b; + memcpy(&a, str + i, 8); + memcpy(&b, rptr + i, 8); + if (a != b) { + a = __builtin_bswap64(a); + b = __builtin_bswap64(b); + return (a < b) ? -1 : 1; + } + } + + for (; i < length; i++) { + if (str[i] != rptr[i]) { + return (str[i] < rptr[i]) ? -1 : 1; + } + } + + return 0; +} +#else +#define rstring_cache_memcmp memcmp +#endif + +static ALWAYS_INLINE() int rstring_cache_cmp(const char *str, const long length, VALUE rstring) +{ + const char *rstring_ptr; + long rstring_length; + + RSTRING_GETMEM(rstring, rstring_ptr, rstring_length); + if (length == rstring_length) { - return memcmp(str, RSTRING_PTR(rstring), length); + return rstring_cache_memcmp(str, rstring_ptr, length); } else { return (int)(length - rstring_length); } } -static VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) +static ALWAYS_INLINE() VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) { int low = 0; int high = cache->length - 1; From fce1c7e84a72d03fe5824f0b1197195189397b68 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 4 Nov 2025 12:06:46 +0100 Subject: [PATCH 207/506] Tentative fix for RHEL8 compiler ``` parser.c:87:77: error: missing binary operator before token "(" #if JSON_CPU_LITTLE_ENDIAN_64BITS && defined(__has_builtin) && __has_builtin(__builtin_bswap64) ``` --- ext/json/ext/parser/parser.c | 9 ++++++--- ext/json/ext/simd/simd.h | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 3d88a399e..9df04ce00 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -84,7 +84,11 @@ static void rvalue_cache_insert_at(rvalue_cache *cache, int index, VALUE rstring cache->entries[index] = rstring; } -#if JSON_CPU_LITTLE_ENDIAN_64BITS && defined(__has_builtin) && __has_builtin(__builtin_bswap64) +#define rstring_cache_memcmp memcmp + +#if JSON_CPU_LITTLE_ENDIAN_64BITS +#if __has_builtin(__builtin_bswap64) +#undef rstring_cache_memcmp static ALWAYS_INLINE() int rstring_cache_memcmp(const char *str, const char *rptr, const long length) { // The libc memcmp has numerous complex optimizations, but in this particular case, @@ -111,8 +115,7 @@ static ALWAYS_INLINE() int rstring_cache_memcmp(const char *str, const char *rpt return 0; } -#else -#define rstring_cache_memcmp memcmp +#endif #endif static ALWAYS_INLINE() int rstring_cache_cmp(const char *str, const long length, VALUE rstring) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index a4f917fd0..c9e3b3ec7 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -6,6 +6,10 @@ typedef enum { SIMD_SSE2 } SIMD_Implementation; +#ifndef __has_builtin // Optional of course. + #define __has_builtin(x) 0 // Compatibility with non-clang compilers. +#endif + #ifdef __clang__ # if __has_builtin(__builtin_ctzll) # define HAVE_BUILTIN_CTZLL 1 From 826cb2a4f4a384562ad809af1d1dcb4a2f0e4f5c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 7 Nov 2025 09:45:06 +0100 Subject: [PATCH 208/506] Get rid of JSON.deep_const_get (private API) --- lib/json/common.rb | 19 +++++++------------ test/json/json_common_interface_test.rb | 5 ----- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 60178c2a5..f22d911f5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -71,8 +71,13 @@ def create_additions_proc(opts) end when object_class if opts[:create_additions] != false - if class_name = object[JSON.create_id] - klass = JSON.deep_const_get(class_name) + if class_path = object[JSON.create_id] + klass = begin + Object.const_get(class_path) + rescue NameError => e + raise ArgumentError, "can't get const #{class_path}: #{e}" + end + if klass.respond_to?(:json_creatable?) ? klass.json_creatable? : klass.respond_to?(:json_create) create_additions_warning if create_additions.nil? object = klass.json_create(object) @@ -147,16 +152,6 @@ def parser=(parser) # :nodoc: const_set :Parser, parser end - # Return the constant located at _path_. The format of _path_ has to be - # either ::A::B::C or A::B::C. In any case, A has to be located at the top - # level (absolute namespace path?). If there doesn't exist a constant at - # the given path, an ArgumentError is raised. - def deep_const_get(path) # :nodoc: - Object.const_get(path) - rescue NameError => e - raise ArgumentError, "can't get const #{path}: #{e}" - end - # Set the module _generator_ to be used by JSON. def generator=(generator) # :nodoc: old, $VERBOSE = $VERBOSE, nil diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 077d7e153..37fa43957 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -68,11 +68,6 @@ def test_create_id JSON.create_id = 'json_class' end - def test_deep_const_get - assert_raise(ArgumentError) { JSON.deep_const_get('Nix::Da') } - assert_equal File::SEPARATOR, JSON.deep_const_get('File::SEPARATOR') - end - def test_parse assert_equal [ 1, 2, 3, ], JSON.parse('[ 1, 2, 3 ]') end From e9fbc8937f21041962146201c97f9e65a7ce5af2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 7 Nov 2025 09:51:34 +0100 Subject: [PATCH 209/506] Deprecate `JSON::State#[]` and `JSON::State#[]=` This prevent from freezing and sharing state instances. If you needs some sort of arguments or extra state to the generator methods, consider using `JSON::Coder` instead. --- CHANGES.md | 1 + lib/json/ext/generator/state.rb | 4 +++ lib/json/truffle_ruby/generator.rb | 4 +++ test/json/json_generator_test.rb | 50 +++++++++++++++++------------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6ad2da7c6..17b03a75e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Deprecate `JSON::State#[]` and `JSON::State#[]=`. Consider using `JSON::Coder` instead. * `JSON::Coder` now also yields to the block when encountering strings with invalid encoding. * Fix GeneratorError messages to be UTF-8 encoded. * Fix memory leak when `Exception` is raised, or `throw` is used during JSON generation. diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index 1f56e6c68..ce5c185ca 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -75,6 +75,8 @@ def to_h # # Returns the value returned by method +name+. def [](name) + ::JSON.deprecation_warning("JSON::State#[] is deprecated and will be removed in json 3.0.0") + if respond_to?(name) __send__(name) else @@ -87,6 +89,8 @@ def [](name) # # Sets the attribute name to value. def []=(name, value) + ::JSON.deprecation_warning("JSON::State#[]= is deprecated and will be removed in json 3.0.0") + if respond_to?(name_writer = "#{name}=") __send__ name_writer, value else diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index feed4c795..dd79bab82 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -439,6 +439,8 @@ def generate_new(obj, anIO = nil) # :nodoc: # Return the value returned by method +name+. def [](name) + ::JSON.deprecation_warning("JSON::State#[] is deprecated and will be removed in json 3.0.0") + if respond_to?(name) __send__(name) else @@ -448,6 +450,8 @@ def [](name) end def []=(name, value) + ::JSON.deprecation_warning("JSON::State#[]= is deprecated and will be removed in json 3.0.0") + if respond_to?(name_writer = "#{name}=") __send__ name_writer, value else diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index a6950f888..1b3c702c9 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -183,7 +183,9 @@ def test_states assert_equal('{"1":2}', json) s = JSON.state.new assert s.check_circular? - assert s[:check_circular?] + assert_deprecated_warning(/JSON::State/) do + assert s[:check_circular?] + end h = { 1=>2 } h[3] = h assert_raise(JSON::NestingError) { generate(h) } @@ -193,7 +195,9 @@ def test_states a << a assert_raise(JSON::NestingError) { generate(a, s) } assert s.check_circular? - assert s[:check_circular?] + assert_deprecated_warning(/JSON::State/) do + assert s[:check_circular?] + end end def test_falsy_state @@ -375,28 +379,32 @@ def to_s end def test_hash_likeness_set_symbol - state = JSON.state.new - assert_equal nil, state[:foo] - assert_equal nil.class, state[:foo].class - assert_equal nil, state['foo'] - state[:foo] = :bar - assert_equal :bar, state[:foo] - assert_equal :bar, state['foo'] - state_hash = state.to_hash - assert_kind_of Hash, state_hash - assert_equal :bar, state_hash[:foo] + assert_deprecated_warning(/JSON::State/) do + state = JSON.state.new + assert_equal nil, state[:foo] + assert_equal nil.class, state[:foo].class + assert_equal nil, state['foo'] + state[:foo] = :bar + assert_equal :bar, state[:foo] + assert_equal :bar, state['foo'] + state_hash = state.to_hash + assert_kind_of Hash, state_hash + assert_equal :bar, state_hash[:foo] + end end def test_hash_likeness_set_string - state = JSON.state.new - assert_equal nil, state[:foo] - assert_equal nil, state['foo'] - state['foo'] = :bar - assert_equal :bar, state[:foo] - assert_equal :bar, state['foo'] - state_hash = state.to_hash - assert_kind_of Hash, state_hash - assert_equal :bar, state_hash[:foo] + assert_deprecated_warning(/JSON::State/) do + state = JSON.state.new + assert_equal nil, state[:foo] + assert_equal nil, state['foo'] + state['foo'] = :bar + assert_equal :bar, state[:foo] + assert_equal :bar, state['foo'] + state_hash = state.to_hash + assert_kind_of Hash, state_hash + assert_equal :bar, state_hash[:foo] + end end def test_json_state_to_h_roundtrip From 5a12067f8878d94739d70e0085b91c88fd9e31d6 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 7 Nov 2025 10:04:43 +0100 Subject: [PATCH 210/506] Release 2.16.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 17b03a75e..83fbaa097 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-11-07 (2.16.0) + * Deprecate `JSON::State#[]` and `JSON::State#[]=`. Consider using `JSON::Coder` instead. * `JSON::Coder` now also yields to the block when encountering strings with invalid encoding. * Fix GeneratorError messages to be UTF-8 encoded. diff --git a/lib/json/version.rb b/lib/json/version.rb index dc01ba290..cc25a0453 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.15.2' + VERSION = '2.16.0' end From 45deb203c81212cb2acc683f29725e5ed2fbcedb Mon Sep 17 00:00:00 2001 From: "NARUSE, Yui" Date: Mon, 17 Nov 2025 21:38:51 +0900 Subject: [PATCH 211/506] strnlen is not used now --- ext/json/ext/parser/extconf.rb | 1 - ext/json/ext/parser/parser.c | 9 --------- 2 files changed, 10 deletions(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index cda385767..079b37382 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -6,7 +6,6 @@ have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby -have_func("strnlen", "string.h") # Missing on Solaris 10 append_cflags("-std=c99") diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 9df04ce00..2bf3ae0eb 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -296,15 +296,6 @@ static void rvalue_stack_eagerly_release(VALUE handle) } } - -#ifndef HAVE_STRNLEN -static size_t strnlen(const char *s, size_t maxlen) -{ - char *p; - return ((p = memchr(s, '\0', maxlen)) ? p - s : maxlen); -} -#endif - static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) { int len = 1; From 4f1adb10d303fcfc3b9a8906e28bab3fccccb789 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Mon, 17 Nov 2025 21:08:29 -0600 Subject: [PATCH 212/506] Use #if instead of #ifdef when checking for JSON_DEBUG so debugging code is not generated when JSON_DEBUG=0. --- ext/json/ext/fbuffer/fbuffer.h | 12 ++++++------ ext/json/ext/generator/extconf.rb | 2 +- ext/json/ext/parser/extconf.rb | 2 +- ext/json/ext/vendor/fpconv.c | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index ecba61465..752d153b3 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -14,7 +14,7 @@ typedef struct FBufferStruct { unsigned long initial_length; unsigned long len; unsigned long capa; -#ifdef JSON_DEBUG +#if JSON_DEBUG unsigned long requested; #endif char *ptr; @@ -45,14 +45,14 @@ static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char * fb->ptr = stack_buffer; fb->capa = stack_buffer_size; } -#ifdef JSON_DEBUG +#if JSON_DEBUG fb->requested = 0; #endif } static inline void fbuffer_consumed(FBuffer *fb, unsigned long consumed) { -#ifdef JSON_DEBUG +#if JSON_DEBUG if (consumed > fb->requested) { rb_bug("fbuffer: Out of bound write"); } @@ -122,7 +122,7 @@ static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested) static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested) { -#ifdef JSON_DEBUG +#if JSON_DEBUG fb->requested = requested; #endif @@ -148,7 +148,7 @@ static inline void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long /* Appends a character into a buffer. The buffer needs to have sufficient capacity, via fbuffer_inc_capa(...). */ static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) { -#ifdef JSON_DEBUG +#if JSON_DEBUG if (fb->requested < 1) { rb_bug("fbuffer: unreserved write"); } @@ -174,7 +174,7 @@ static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) fbuffer_inc_capa(fb, repeat * len); while (repeat) { -#ifdef JSON_DEBUG +#if JSON_DEBUG fb->requested = len; #endif fbuffer_append_reserved(fb, newstr, len); diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index fb9afd07f..ee1bbeaba 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -6,7 +6,7 @@ else append_cflags("-std=c99") $defs << "-DJSON_GENERATOR" - $defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] + $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" if enable_config('generator-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) load __dir__ + "/../simd/conf.rb" diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 079b37382..2440e66d8 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'mkmf' -$defs << "-DJSON_DEBUG" if ENV["JSON_DEBUG"] +$defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index e91c7889c..4888ab4b8 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -29,7 +29,7 @@ #include #include -#ifdef JSON_DEBUG +#if JSON_DEBUG #include #endif @@ -472,7 +472,7 @@ static int fpconv_dtoa(double d, char dest[28]) int ndigits = grisu2(d, digits, &K); str_len += emit_digits(digits, ndigits, dest + str_len, K, neg); -#ifdef JSON_DEBUG +#if JSON_DEBUG assert(str_len <= 32); #endif From ab5efca015167684d696980b6cec7848551ca07f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 18 Nov 2025 22:45:08 +0000 Subject: [PATCH 213/506] parser.c: Remove unued JSON_ParserStruct.parsing_name --- ext/json/ext/parser/parser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2bf3ae0eb..62ee1f24e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -336,7 +336,6 @@ typedef struct JSON_ParserStruct { int max_nesting; bool allow_nan; bool allow_trailing_comma; - bool parsing_name; bool symbolize_names; bool freeze; } JSON_ParserConfig; From 9364d0c761dc74e6bce146e4c965c01aa3e646b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 20 Nov 2025 16:41:45 +0100 Subject: [PATCH 214/506] Remove unused symbols --- ext/json/ext/generator/generator.c | 6 +----- ext/json/ext/parser/parser.c | 7 +------ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 8930213b9..fb8424dd8 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -38,7 +38,7 @@ typedef struct JSON_Generator_StateStruct { static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8; -static ID i_to_s, i_to_json, i_new, i_pack, i_unpack, i_create_id, i_extend, i_encode; +static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json; @@ -2173,10 +2173,6 @@ void Init_generator(void) i_to_s = rb_intern("to_s"); i_to_json = rb_intern("to_json"); i_new = rb_intern("new"); - i_pack = rb_intern("pack"); - i_unpack = rb_intern("unpack"); - i_create_id = rb_intern("create_id"); - i_extend = rb_intern("extend"); i_encode = rb_intern("encode"); sym_indent = ID2SYM(rb_intern("indent")); diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 62ee1f24e..a1d0265a9 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -5,8 +5,7 @@ static VALUE mJSON, eNestingError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; -static ID i_chr, i_aset, i_aref, - i_leftshift, i_new, i_try_convert, i_uminus, i_encode; +static ID i_new, i_try_convert, i_uminus, i_encode; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key; @@ -1612,10 +1611,6 @@ void Init_parser(void) sym_decimal_class = ID2SYM(rb_intern("decimal_class")); sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); - i_chr = rb_intern("chr"); - i_aset = rb_intern("[]="); - i_aref = rb_intern("[]"); - i_leftshift = rb_intern("<<"); i_new = rb_intern("new"); i_try_convert = rb_intern("try_convert"); i_uminus = rb_intern("-@"); From 3d90aaf568c49711619096758a0e1b3557fadf11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 03:12:13 +0000 Subject: [PATCH 215/506] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- .github/workflows/sync-ruby.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8c08f196..3f85fded2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: - { os: windows-latest, ruby: jruby } steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 @@ -68,7 +68,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 diff --git a/.github/workflows/sync-ruby.yml b/.github/workflows/sync-ruby.yml index 29c434491..78e94eb9e 100644 --- a/.github/workflows/sync-ruby.yml +++ b/.github/workflows/sync-ruby.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository_owner == 'ruby' }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Create GitHub App token id: app-token From 58d60d6b769b9214d77053cd01fc1a3540f030b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Tue, 18 Nov 2025 11:44:06 +0100 Subject: [PATCH 216/506] Ractor-shareable JSON::Coder --- ext/json/ext/generator/generator.c | 14 +++++++ ext/json/ext/parser/parser.c | 8 +++- java/src/json/ext/GeneratorState.java | 13 ++++++ java/src/json/ext/ParserConfig.java | 1 + lib/json/common.rb | 2 +- test/json/json_generator_test.rb | 14 +++++++ test/json/json_parser_test.rb | 7 ++++ test/json/ractor_test.rb | 59 +++++++++++++++++++++++++++ 8 files changed, 116 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index fb8424dd8..2fb00ee27 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1630,6 +1630,7 @@ static VALUE string_config(VALUE config) */ static VALUE cState_indent_set(VALUE self, VALUE indent) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->indent, string_config(indent)); return Qnil; @@ -1655,6 +1656,7 @@ static VALUE cState_space(VALUE self) */ static VALUE cState_space_set(VALUE self, VALUE space) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->space, string_config(space)); return Qnil; @@ -1678,6 +1680,7 @@ static VALUE cState_space_before(VALUE self) */ static VALUE cState_space_before_set(VALUE self, VALUE space_before) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->space_before, string_config(space_before)); return Qnil; @@ -1703,6 +1706,7 @@ static VALUE cState_object_nl(VALUE self) */ static VALUE cState_object_nl_set(VALUE self, VALUE object_nl) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->object_nl, string_config(object_nl)); return Qnil; @@ -1726,6 +1730,7 @@ static VALUE cState_array_nl(VALUE self) */ static VALUE cState_array_nl_set(VALUE self, VALUE array_nl) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->array_nl, string_config(array_nl)); return Qnil; @@ -1749,6 +1754,7 @@ static VALUE cState_as_json(VALUE self) */ static VALUE cState_as_json_set(VALUE self, VALUE as_json) { + rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->as_json, rb_convert_type(as_json, T_DATA, "Proc", "to_proc")); return Qnil; @@ -1791,6 +1797,7 @@ static long long_config(VALUE num) */ static VALUE cState_max_nesting_set(VALUE self, VALUE depth) { + rb_check_frozen(self); GET_STATE(self); state->max_nesting = long_config(depth); return Qnil; @@ -1816,6 +1823,7 @@ static VALUE cState_script_safe(VALUE self) */ static VALUE cState_script_safe_set(VALUE self, VALUE enable) { + rb_check_frozen(self); GET_STATE(self); state->script_safe = RTEST(enable); return Qnil; @@ -1847,6 +1855,7 @@ static VALUE cState_strict(VALUE self) */ static VALUE cState_strict_set(VALUE self, VALUE enable) { + rb_check_frozen(self); GET_STATE(self); state->strict = RTEST(enable); return Qnil; @@ -1871,6 +1880,7 @@ static VALUE cState_allow_nan_p(VALUE self) */ static VALUE cState_allow_nan_set(VALUE self, VALUE enable) { + rb_check_frozen(self); GET_STATE(self); state->allow_nan = RTEST(enable); return Qnil; @@ -1895,6 +1905,7 @@ static VALUE cState_ascii_only_p(VALUE self) */ static VALUE cState_ascii_only_set(VALUE self, VALUE enable) { + rb_check_frozen(self); GET_STATE(self); state->ascii_only = RTEST(enable); return Qnil; @@ -1932,6 +1943,7 @@ static VALUE cState_depth(VALUE self) */ static VALUE cState_depth_set(VALUE self, VALUE depth) { + rb_check_frozen(self); GET_STATE(self); state->depth = long_config(depth); return Qnil; @@ -1965,6 +1977,7 @@ static void buffer_initial_length_set(JSON_Generator_State *state, VALUE buffer_ */ static VALUE cState_buffer_initial_length_set(VALUE self, VALUE buffer_initial_length) { + rb_check_frozen(self); GET_STATE(self); buffer_initial_length_set(state, buffer_initial_length); return Qnil; @@ -2031,6 +2044,7 @@ static void configure_state(JSON_Generator_State *state, VALUE vstate, VALUE con static VALUE cState_configure(VALUE self, VALUE opts) { + rb_check_frozen(self); GET_STATE(self); configure_state(state, self, opts); return self; diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a1d0265a9..e23945d0e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1466,6 +1466,7 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts) */ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) { + rb_check_frozen(self); GET_PARSER_CONFIG; parser_config_init(config, opts); @@ -1553,6 +1554,11 @@ static size_t JSON_ParserConfig_memsize(const void *ptr) return sizeof(JSON_ParserConfig); } +#ifndef HAVE_RB_EXT_RACTOR_SAFE +# undef RUBY_TYPED_FROZEN_SHAREABLE +# define RUBY_TYPED_FROZEN_SHAREABLE 0 +#endif + static const rb_data_type_t JSON_ParserConfig_type = { "JSON::Ext::Parser/ParserConfig", { @@ -1561,7 +1567,7 @@ static const rb_data_type_t JSON_ParserConfig_type = { JSON_ParserConfig_memsize, }, 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, + RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE, }; static VALUE cJSON_parser_s_allocate(VALUE klass) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index fd1478b0e..19d6a1341 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -283,6 +283,7 @@ public IRubyObject op_aref(ThreadContext context, IRubyObject vName) { @JRubyMethod(name="[]=") public IRubyObject op_aset(ThreadContext context, IRubyObject vName, IRubyObject value) { + checkFrozen(); String name = vName.asJavaString(); String nameWriter = name + "="; if (getMetaClass().isMethodBound(nameWriter, true)) { @@ -304,6 +305,7 @@ public RubyString indent_get(ThreadContext context) { @JRubyMethod(name="indent=") public IRubyObject indent_set(ThreadContext context, IRubyObject indent) { + checkFrozen(); this.indent = prepareByteList(context, indent); return indent; } @@ -319,6 +321,7 @@ public RubyString space_get(ThreadContext context) { @JRubyMethod(name="space=") public IRubyObject space_set(ThreadContext context, IRubyObject space) { + checkFrozen(); this.space = prepareByteList(context, space); return space; } @@ -335,6 +338,7 @@ public RubyString space_before_get(ThreadContext context) { @JRubyMethod(name="space_before=") public IRubyObject space_before_set(ThreadContext context, IRubyObject spaceBefore) { + checkFrozen(); this.spaceBefore = prepareByteList(context, spaceBefore); return spaceBefore; } @@ -351,6 +355,7 @@ public RubyString object_nl_get(ThreadContext context) { @JRubyMethod(name="object_nl=") public IRubyObject object_nl_set(ThreadContext context, IRubyObject objectNl) { + checkFrozen(); this.objectNl = prepareByteList(context, objectNl); return objectNl; } @@ -367,6 +372,7 @@ public RubyString array_nl_get(ThreadContext context) { @JRubyMethod(name="array_nl=") public IRubyObject array_nl_set(ThreadContext context, IRubyObject arrayNl) { + checkFrozen(); this.arrayNl = prepareByteList(context, arrayNl); return arrayNl; } @@ -382,6 +388,7 @@ public IRubyObject as_json_get(ThreadContext context) { @JRubyMethod(name="as_json=") public IRubyObject as_json_set(ThreadContext context, IRubyObject asJSON) { + checkFrozen(); if (asJSON.isNil() || asJSON == context.getRuntime().getFalse()) { this.asJSON = null; } else { @@ -402,6 +409,7 @@ public RubyInteger max_nesting_get(ThreadContext context) { @JRubyMethod(name="max_nesting=") public IRubyObject max_nesting_set(IRubyObject max_nesting) { + checkFrozen(); maxNesting = RubyNumeric.fix2int(max_nesting); return max_nesting; } @@ -420,6 +428,7 @@ public RubyBoolean script_safe_get(ThreadContext context) { @JRubyMethod(name="script_safe=", alias="escape_slash=") public IRubyObject script_safe_set(IRubyObject script_safe) { + checkFrozen(); scriptSafe = script_safe.isTrue(); return script_safe.getRuntime().newBoolean(scriptSafe); } @@ -443,6 +452,7 @@ public RubyBoolean strict_get(ThreadContext context) { @JRubyMethod(name="strict=") public IRubyObject strict_set(IRubyObject isStrict) { + checkFrozen(); strict = isStrict.isTrue(); return isStrict.getRuntime().newBoolean(strict); } @@ -472,6 +482,7 @@ public RubyInteger buffer_initial_length_get(ThreadContext context) { @JRubyMethod(name="buffer_initial_length=") public IRubyObject buffer_initial_length_set(IRubyObject buffer_initial_length) { + checkFrozen(); int newLength = RubyNumeric.fix2int(buffer_initial_length); if (newLength > 0) bufferInitialLength = newLength; return buffer_initial_length; @@ -488,6 +499,7 @@ public RubyInteger depth_get(ThreadContext context) { @JRubyMethod(name="depth=") public IRubyObject depth_set(IRubyObject vDepth) { + checkFrozen(); depth = RubyNumeric.fix2int(vDepth); return vDepth; } @@ -532,6 +544,7 @@ public boolean getDeprecateDuplicateKey() { */ @JRubyMethod(visibility=Visibility.PRIVATE) public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { + checkFrozen(); OptionsReader opts = new OptionsReader(context, vOpts); ByteList indent = opts.getString("indent"); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 2bd03babc..02bd7e5e2 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -172,6 +172,7 @@ public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, IRuby @JRubyMethod(visibility = Visibility.PRIVATE) public IRubyObject initialize(ThreadContext context, IRubyObject options) { + checkFrozen(); Ruby runtime = context.runtime; OptionsReader opts = new OptionsReader(context, options); diff --git a/lib/json/common.rb b/lib/json/common.rb index f22d911f5..233b8c7e6 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1048,7 +1048,7 @@ def initialize(options = nil, &as_json) options[:as_json] = as_json if as_json @state = State.new(options).freeze - @parser_config = Ext::Parser::Config.new(ParserOptions.prepare(options)) + @parser_config = Ext::Parser::Config.new(ParserOptions.prepare(options)).freeze end # call-seq: diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1b3c702c9..54a2ec614 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -901,4 +901,18 @@ def test_generate_duplicate_keys_disallowed end assert_equal %(detected duplicate key "foo" in #{hash.inspect}), error.message end + + def test_frozen + state = JSON::State.new.freeze + assert_raise(FrozenError) do + state.configure(max_nesting: 1) + end + setters = state.methods.grep(/\w=$/) + assert_not_empty setters + setters.each do |setter| + assert_raise(FrozenError) do + state.send(setter, 1) + end + end + end end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 6315c3e66..544a8b564 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -820,6 +820,13 @@ def test_parse_whitespace_after_newline assert_equal [], JSON.parse("[\n#{' ' * (8 + 8 + 4 + 3)}]") end + def test_frozen + parser_config = JSON::Parser::Config.new({}).freeze + assert_raise FrozenError do + parser_config.send(:initialize, {}) + end + end + private def assert_equal_float(expected, actual, delta = 1e-2) diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index 53e1099ce..e53c405a7 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -52,4 +52,63 @@ def test_generate _, status = Process.waitpid2(pid) assert_predicate status, :success? end + + def test_coder + coder = JSON::Coder.new.freeze + assert Ractor.shareable?(coder) + pid = fork do + Warning[:experimental] = false + r = Ractor.new(coder) do |coder| + json = coder.dump({ + 'a' => 2, + 'b' => 3.141, + 'c' => 'c', + 'd' => [ 1, "b", 3.14 ], + 'e' => { 'foo' => 'bar' }, + 'g' => "\"\0\037", + 'h' => 1000.0, + 'i' => 0.001 + }) + coder.load(json) + end + expected_json = JSON.parse('{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},' + + '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}') + actual_json = r.value + + if expected_json == actual_json + exit 0 + else + puts "Expected:" + puts expected_json + puts "Actual:" + puts actual_json + puts + exit 1 + end + end + _, status = Process.waitpid2(pid) + assert_predicate status, :success? + end + + class NonNative + def initialize(value) + @value = value + end + end + + def test_coder_proc + block = Ractor.shareable_proc { |value| value.as_json } + coder = JSON::Coder.new(&block).freeze + assert Ractor.shareable?(coder) + + pid = fork do + Warning[:experimental] = false + assert_equal [{}], Ractor.new(coder) { |coder| + coder.load('[{}]') + }.value + end + + _, status = Process.waitpid2(pid) + assert_predicate status, :success? + end if Ractor.respond_to?(:shareable_proc) end if defined?(Ractor) && Process.respond_to?(:fork) From 305d3832dbc6adde025e867be1cc1d4d0b7e0cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 21 Nov 2025 11:53:17 +0100 Subject: [PATCH 217/506] Skip test failing with JRuby in CI --- test/json/json_parser_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 544a8b564..a5b476361 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -822,6 +822,7 @@ def test_parse_whitespace_after_newline def test_frozen parser_config = JSON::Parser::Config.new({}).freeze + omit "JRuby failure in CI" if RUBY_ENGINE == "jruby" assert_raise FrozenError do parser_config.send(:initialize, {}) end From 2a4ebe82508aac68c54e60a2924a953041495aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 21 Nov 2025 12:48:21 +0100 Subject: [PATCH 218/506] Move RUBY_TYPED_FROZEN_SHAREABLE macro to json.h --- ext/json/ext/generator/generator.c | 5 ----- ext/json/ext/json.h | 5 +++++ ext/json/ext/parser/parser.c | 5 ----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 2fb00ee27..8d04bef53 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -926,11 +926,6 @@ static size_t State_memsize(const void *ptr) return sizeof(JSON_Generator_State); } -#ifndef HAVE_RB_EXT_RACTOR_SAFE -# undef RUBY_TYPED_FROZEN_SHAREABLE -# define RUBY_TYPED_FROZEN_SHAREABLE 0 -#endif - static const rb_data_type_t JSON_Generator_State_type = { "JSON/Generator/State", { diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 01fe0cd03..28efa04c2 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -45,6 +45,11 @@ typedef unsigned char _Bool; #endif #endif +#ifndef HAVE_RB_EXT_RACTOR_SAFE +# undef RUBY_TYPED_FROZEN_SHAREABLE +# define RUBY_TYPED_FROZEN_SHAREABLE 0 +#endif + #ifndef NORETURN #define NORETURN(x) x #endif diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e23945d0e..2dcd1012b 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1554,11 +1554,6 @@ static size_t JSON_ParserConfig_memsize(const void *ptr) return sizeof(JSON_ParserConfig); } -#ifndef HAVE_RB_EXT_RACTOR_SAFE -# undef RUBY_TYPED_FROZEN_SHAREABLE -# define RUBY_TYPED_FROZEN_SHAREABLE 0 -#endif - static const rb_data_type_t JSON_ParserConfig_type = { "JSON::Ext::Parser/ParserConfig", { From 256cad5def35f8a93e8a391dbb7c4d9132e08fd2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 22 Nov 2025 10:32:16 +0100 Subject: [PATCH 219/506] Use booleans in string_scan --- ext/json/ext/parser/parser.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2dcd1012b..cb22648db 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -937,7 +937,7 @@ static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) uint64_t mask = 0; if (string_scan_simd_neon(&state->cursor, state->end, &mask)) { state->cursor += trailing_zeros64(mask) >> 2; - return 1; + return true; } #elif defined(HAVE_SIMD_SSE2) @@ -945,7 +945,7 @@ static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) int mask = 0; if (string_scan_simd_sse2(&state->cursor, state->end, &mask)) { state->cursor += trailing_zeros(mask); - return 1; + return true; } } #endif /* HAVE_SIMD_NEON or HAVE_SIMD_SSE2 */ @@ -953,11 +953,11 @@ static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) while (!eos(state)) { if (RB_UNLIKELY(string_scan_table[(unsigned char)*state->cursor])) { - return 1; + return true; } state->cursor++; } - return 0; + return false; } static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) From 3b473ffd705ef7775d87b2cd871f1b641ce4e67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Sat, 22 Nov 2025 12:24:34 +0100 Subject: [PATCH 220/506] Don't emit deprecation warnings in State#to_h in TruffleRuby $ ruby -Ilib -rjson -W:deprecated -e JSON::State.new.to_h |& uniq -c 13 -e:1: warning: JSON::State#[] is deprecated and will be removed in json 3.0.0 --- lib/json/truffle_ruby/generator.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index dd79bab82..71a909dd6 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -312,8 +312,8 @@ def allow_duplicate_key? # :nodoc: def to_h result = {} instance_variables.each do |iv| - iv = iv.to_s[1..-1] - result[iv.to_sym] = self[iv] + key = iv.to_s[1..-1] + result[key.to_sym] = instance_variable_get(iv) end if result[:allow_duplicate_key].nil? From f7f8f552ed513aa54d04903d127ca3469defd7db Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 22 Nov 2025 12:59:42 +0100 Subject: [PATCH 221/506] Fix the parser to not accept invalid escapes Only `"\/bfnrtu` are valid after a backslash. --- CHANGES.md | 3 ++ ext/json/ext/parser/parser.c | 53 ++++++++++--------- java/src/json/ext/StringDecoder.java | 21 +++++++- .../fixtures/{pass15.json => fail15.json} | 0 .../fixtures/{pass16.json => fail16.json} | 0 .../fixtures/{pass17.json => fail17.json} | 0 .../fixtures/{pass26.json => fail26.json} | 0 test/json/json_fixtures_test.rb | 2 + test/json/json_parser_test.rb | 4 +- 9 files changed, 53 insertions(+), 30 deletions(-) rename test/json/fixtures/{pass15.json => fail15.json} (100%) rename test/json/fixtures/{pass16.json => fail16.json} (100%) rename test/json/fixtures/{pass17.json => fail17.json} (100%) rename test/json/fixtures/{pass26.json => fail26.json} (100%) diff --git a/CHANGES.md b/CHANGES.md index 83fbaa097..3ebbe1909 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +* Fixed the parser to no longer ignore invalid escapes in strings. + Only `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t` and `\u` are valid JSON escapes. + ### 2025-11-07 (2.16.0) * Deprecate `JSON::State#[]` and `JSON::State#[]=`. Consider using `JSON::Coder` instead. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index cb22648db..2f584a51a 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -639,44 +639,43 @@ static inline VALUE json_string_fastpath(JSON_ParserState *state, const char *st static VALUE json_string_unescape(JSON_ParserState *state, const char *string, const char *stringEnd, bool is_name, bool intern, bool symbolize) { size_t bufferSize = stringEnd - string; - const char *p = string, *pe = string, *unescape, *bufferStart; + const char *p = string, *pe = string, *bufferStart; char *buffer; - int unescape_len; - char buf[4]; VALUE result = rb_str_buf_new(bufferSize); rb_enc_associate_index(result, utf8_encindex); buffer = RSTRING_PTR(result); bufferStart = buffer; +#define APPEND_CHAR(chr) *buffer++ = chr; p = ++pe; + while (pe < stringEnd && (pe = memchr(pe, '\\', stringEnd - pe))) { - unescape = (char *) "?"; - unescape_len = 1; if (pe > p) { MEMCPY(buffer, p, char, pe - p); buffer += pe - p; } switch (*++pe) { + case '"': + case '/': + p = pe; // nothing to unescape just need to skip the backslash + break; + case '\\': + APPEND_CHAR('\\'); + break; case 'n': - unescape = (char *) "\n"; + APPEND_CHAR('\n'); break; case 'r': - unescape = (char *) "\r"; + APPEND_CHAR('\r'); break; case 't': - unescape = (char *) "\t"; - break; - case '"': - unescape = (char *) "\""; - break; - case '\\': - unescape = (char *) "\\"; + APPEND_CHAR('\t'); break; case 'b': - unescape = (char *) "\b"; + APPEND_CHAR('\b'); break; case 'f': - unescape = (char *) "\f"; + APPEND_CHAR('\f'); break; case 'u': if (pe > stringEnd - 5) { @@ -714,18 +713,23 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c break; } } - unescape_len = convert_UTF32_to_UTF8(buf, ch); - unescape = buf; + + char buf[4]; + int unescape_len = convert_UTF32_to_UTF8(buf, ch); + MEMCPY(buffer, buf, char, unescape_len); + buffer += unescape_len; + p = ++pe; } break; default: - p = pe; - continue; + if ((unsigned char)*pe < 0x20) { + raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); + } + raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); + break; } - MEMCPY(buffer, unescape, char, unescape_len); - buffer += unescape_len; - p = ++pe; } +#undef APPEND_CHAR if (stringEnd > p) { MEMCPY(buffer, p, char, stringEnd - p); @@ -976,9 +980,6 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig case '\\': { state->cursor++; escaped = true; - if ((unsigned char)*state->cursor < 0x20) { - raise_parse_error("invalid ASCII control character in string: %s", state); - } break; } default: diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index bf616c1e9..03c78514a 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -69,6 +69,15 @@ private void handleEscapeSequence(ThreadContext context) throws IOException { case 't': append('\t'); break; + case '/': + append('/'); + break; + case '"': + append('"'); + break; + case '\\': + append('\\'); + break; case 'u': ensureMin(context, 4); int cp = readHex(context); @@ -81,8 +90,8 @@ private void handleEscapeSequence(ThreadContext context) throws IOException { writeUtf8Char(cp); } break; - default: // '\\', '"', '/'... - quoteStart(); + default: + throw invalidEscape(context); } } @@ -174,4 +183,12 @@ protected RaiseException invalidUtf8(ThreadContext context) { return Utils.newException(context, Utils.M_PARSER_ERROR, context.runtime.newString(message)); } + + protected RaiseException invalidEscape(ThreadContext context) { + ByteList message = new ByteList( + ByteList.plain("invalid escape character in string: ")); + message.append(src, charStart, srcEnd - charStart); + return Utils.newException(context, Utils.M_PARSER_ERROR, + context.runtime.newString(message)); + } } diff --git a/test/json/fixtures/pass15.json b/test/json/fixtures/fail15.json similarity index 100% rename from test/json/fixtures/pass15.json rename to test/json/fixtures/fail15.json diff --git a/test/json/fixtures/pass16.json b/test/json/fixtures/fail16.json similarity index 100% rename from test/json/fixtures/pass16.json rename to test/json/fixtures/fail16.json diff --git a/test/json/fixtures/pass17.json b/test/json/fixtures/fail17.json similarity index 100% rename from test/json/fixtures/pass17.json rename to test/json/fixtures/fail17.json diff --git a/test/json/fixtures/pass26.json b/test/json/fixtures/fail26.json similarity index 100% rename from test/json/fixtures/pass26.json rename to test/json/fixtures/fail26.json diff --git a/test/json/json_fixtures_test.rb b/test/json/json_fixtures_test.rb index c153ebef7..c0d103793 100644 --- a/test/json/json_fixtures_test.rb +++ b/test/json/json_fixtures_test.rb @@ -10,6 +10,8 @@ class JSONFixturesTest < Test::Unit::TestCase source = File.read(f) define_method("test_#{name}") do assert JSON.parse(source), "Did not pass for fixture '#{File.basename(f)}': #{source.inspect}" + rescue JSON::ParserError + raise "#{File.basename(f)} parsing failure" end end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index a5b476361..3e662bda3 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -510,8 +510,8 @@ def test_backslash data = ['"'] assert_equal data, parse(json) # - json = '["\\\'"]' - data = ["'"] + json = '["\\/"]' + data = ["/"] assert_equal data, parse(json) json = '["\/"]' From 4f4551f993bf1642eb8e0a18152306c5b25b18ea Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 22 Nov 2025 14:13:30 +0100 Subject: [PATCH 222/506] parser.c: Record escape positions while parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can then pass them to the decoder to save having to parse the string again. ``` == Parsing activitypub.json (58160 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 1.275k i/100ms Calculating ------------------------------------- after 12.774k (± 0.8%) i/s (78.29 μs/i) - 65.025k in 5.090834s Comparison: before: 12314.3 i/s after: 12773.8 i/s - 1.04x faster == Parsing twitter.json (567916 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 143.000 i/100ms Calculating ------------------------------------- after 1.441k (± 0.2%) i/s (693.86 μs/i) - 7.293k in 5.060345s Comparison: before: 1430.1 i/s after: 1441.2 i/s - 1.01x faster == Parsing citm_catalog.json (1727030 bytes) ruby 3.4.6 (2025-09-16 revision dbd83256b1) +YJIT +PRISM [arm64-darwin24] Warming up -------------------------------------- after 69.000 i/100ms Calculating ------------------------------------- after 695.919 (± 0.4%) i/s (1.44 ms/i) - 3.519k in 5.056691s Comparison: before: 687.8 i/s after: 695.9 i/s - 1.01x faster ``` --- ext/json/ext/parser/parser.c | 88 ++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2f584a51a..0216eef98 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -616,8 +616,10 @@ static inline bool json_string_cacheable_p(const char *string, size_t length) return length <= JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH && rb_isalpha(string[0]); } -static inline VALUE json_string_fastpath(JSON_ParserState *state, const char *string, const char *stringEnd, bool is_name, bool intern, bool symbolize) +static inline VALUE json_string_fastpath(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name) { + bool intern = is_name || config->freeze; + bool symbolize = is_name && config->symbolize_names; size_t bufferSize = stringEnd - string; if (is_name && state->in_array && RB_LIKELY(json_string_cacheable_p(string, bufferSize))) { @@ -636,8 +638,33 @@ static inline VALUE json_string_fastpath(JSON_ParserState *state, const char *st return build_string(string, stringEnd, intern, symbolize); } -static VALUE json_string_unescape(JSON_ParserState *state, const char *string, const char *stringEnd, bool is_name, bool intern, bool symbolize) +#define JSON_MAX_UNESCAPE_POSITIONS 16 +typedef struct _json_unescape_positions { + long size; + const char **positions; + bool has_more; +} JSON_UnescapePositions; + +static inline const char *json_next_backslash(const char *pe, const char *stringEnd, JSON_UnescapePositions *positions) +{ + while (positions->size) { + positions->size--; + const char *next_position = positions->positions[0]; + positions->positions++; + return next_position; + } + + if (positions->has_more) { + return memchr(pe, '\\', stringEnd - pe); + } + + return NULL; +} + +static NOINLINE() VALUE json_string_unescape(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name, JSON_UnescapePositions *positions) { + bool intern = is_name || config->freeze; + bool symbolize = is_name && config->symbolize_names; size_t bufferSize = stringEnd - string; const char *p = string, *pe = string, *bufferStart; char *buffer; @@ -649,7 +676,7 @@ static VALUE json_string_unescape(JSON_ParserState *state, const char *string, c #define APPEND_CHAR(chr) *buffer++ = chr; p = ++pe; - while (pe < stringEnd && (pe = memchr(pe, '\\', stringEnd - pe))) { + while (pe < stringEnd && (pe = json_next_backslash(pe, stringEnd, positions))) { if (pe > p) { MEMCPY(buffer, p, char, pe - p); buffer += pe - p; @@ -893,20 +920,6 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi return object; } -static inline VALUE json_decode_string(JSON_ParserState *state, JSON_ParserConfig *config, const char *start, const char *end, bool escaped, bool is_name) -{ - VALUE string; - bool intern = is_name || config->freeze; - bool symbolize = is_name && config->symbolize_names; - if (escaped) { - string = json_string_unescape(state, start, end, is_name, intern, symbolize); - } else { - string = json_string_fastpath(state, start, end, is_name, intern, symbolize); - } - - return string; -} - static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig *config, VALUE value) { if (RB_UNLIKELY(config->on_load_proc)) { @@ -964,22 +977,30 @@ static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) return false; } -static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) +static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name, const char *start) { - state->cursor++; - const char *start = state->cursor; - bool escaped = false; + const char *backslashes[JSON_MAX_UNESCAPE_POSITIONS]; + JSON_UnescapePositions positions = { + .size = 0, + .positions = backslashes, + .has_more = false, + }; - while (RB_UNLIKELY(string_scan(state))) { + do { switch (*state->cursor) { case '"': { - VALUE string = json_decode_string(state, config, start, state->cursor, escaped, is_name); + VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions); state->cursor++; return json_push_value(state, config, string); } case '\\': { + if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) { + backslashes[positions.size] = state->cursor; + positions.size++; + } else { + positions.has_more = true; + } state->cursor++; - escaped = true; break; } default: @@ -988,12 +1009,29 @@ static inline VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig } state->cursor++; - } + } while (string_scan(state)); raise_parse_error("unexpected end of input, expected closing \"", state); return Qfalse; } +static ALWAYS_INLINE() VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) +{ + state->cursor++; + const char *start = state->cursor; + + if (RB_UNLIKELY(!string_scan(state))) { + raise_parse_error("unexpected end of input, expected closing \"", state); + } + + if (RB_LIKELY(*state->cursor == '"')) { + VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name); + state->cursor++; + return json_push_value(state, config, string); + } + return json_parse_escaped_string(state, config, is_name, start); +} + #if JSON_CPU_LITTLE_ENDIAN_64BITS // From: https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/ // Additional References: From 9d32cf461835fdb03c7385616849b3cefe9c20ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Sun, 23 Nov 2025 11:36:58 +0100 Subject: [PATCH 223/506] Test that depth of unfrozen State does not change --- test/json/json_generator_test.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 54a2ec614..f1fb72ee2 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -915,4 +915,13 @@ def test_frozen end end end + + # The case when the State is frozen is tested in JSONCoderTest#test_nesting_recovery + def test_nesting_recovery + state = JSON::State.new + ary = [] + ary << ary + assert_raise(JSON::NestingError) { state.generate_new(ary) } + assert_equal '{"a":1}', state.generate({ a: 1 }) + end end From d02e40324ae71228700ced5514aa6409e6e31193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 24 Nov 2025 15:40:06 +0100 Subject: [PATCH 224/506] Test depth --- test/json/json_generator_test.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f1fb72ee2..0931668f3 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -282,6 +282,16 @@ def test_allow_nan end def test_depth + pretty = { object_nl: "\n", array_nl: "\n", space: " ", indent: " " } + state = JSON.state.new(**pretty) + assert_equal %({\n "foo": 42\n}), JSON.generate({ foo: 42 }, pretty) + assert_equal %({\n "foo": 42\n}), state.generate(foo: 42) + state.depth = 1 + assert_equal %({\n "foo": 42\n }), JSON.generate({ foo: 42 }, pretty.merge(depth: 1)) + assert_equal %({\n "foo": 42\n }), state.generate(foo: 42) + end + + def test_depth_nesting_error ary = []; ary << ary assert_raise(JSON::NestingError) { generate(ary) } assert_raise(JSON::NestingError) { JSON.pretty_generate(ary) } From ac0a980668d9e990808f50286c60847b323cce71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Wed, 26 Nov 2025 11:03:33 +0100 Subject: [PATCH 225/506] Test to_json using State#depth --- test/json/json_generator_test.rb | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 0931668f3..c01ed678f 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -92,6 +92,46 @@ def test_dump_strict assert_equal '"World"', "World".to_json(strict: true) end + def test_state_depth_to_json + depth = Object.new + def depth.to_json(state) + JSON::State.from_state(state).depth.to_s + end + + assert_equal "0", JSON.generate(depth) + assert_equal "[1]", JSON.generate([depth]) + assert_equal %({"depth":1}), JSON.generate(depth: depth) + assert_equal "[[2]]", JSON.generate([[depth]]) + assert_equal %([{"depth":2}]), JSON.generate([{depth: depth}]) + + state = JSON::State.new + assert_equal "0", state.generate(depth) + assert_equal "[1]", state.generate([depth]) + assert_equal %({"depth":1}), state.generate(depth: depth) + assert_equal "[[2]]", state.generate([[depth]]) + assert_equal %([{"depth":2}]), state.generate([{depth: depth}]) + end + + def test_state_depth_to_json_recursive + recur = Object.new + def recur.to_json(state = nil, *) + state = JSON::State.from_state(state) + if state.depth < 3 + state.generate([state.depth, self]) + else + state.generate([state.depth]) + end + end + + assert_raise(NestingError) { JSON.generate(recur, max_nesting: 3) } + assert_equal "[0,[1,[2,[3]]]]", JSON.generate(recur, max_nesting: 4) + + state = JSON::State.new(max_nesting: 3) + assert_raise(NestingError) { state.generate(recur) } + state.max_nesting = 4 + assert_equal "[0,[1,[2,[3]]]]", JSON.generate(recur, max_nesting: 4) + end + def test_generate_pretty json = pretty_generate({}) assert_equal('{}', json) From 5abd43490714460e1ec1fa382f82a09485eed563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Sat, 22 Nov 2025 12:30:44 +0100 Subject: [PATCH 226/506] Add depth to struct generate_json_data Instead of incrementing JSON_Generator_State::depth, we now increment generate_json_data::depth, and only copied at the end. --- ext/json/ext/generator/generator.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 8d04bef53..7125f4437 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -60,6 +60,7 @@ struct generate_json_data { JSON_Generator_State *state; VALUE obj; generator_func func; + long depth; }; static VALUE cState_from_state_s(VALUE self, VALUE opts); @@ -972,6 +973,8 @@ static inline VALUE vstate_get(struct generate_json_data *data) if (RB_UNLIKELY(!data->vstate)) { vstate_spill(data); } + GET_STATE(data->vstate); + state->depth = data->depth; return data->vstate; } @@ -1145,7 +1148,7 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) FBuffer *buffer = data->buffer; JSON_Generator_State *state = data->state; - long depth = state->depth; + long depth = data->depth; int key_type = rb_type(key); if (arg->first) { @@ -1219,9 +1222,9 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) static inline long increase_depth(struct generate_json_data *data) { JSON_Generator_State *state = data->state; - long depth = ++state->depth; + long depth = ++data->depth; if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) { - rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", --state->depth); + rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", --data->depth); } return depth; } @@ -1232,7 +1235,7 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat if (RHASH_SIZE(obj) == 0) { fbuffer_append(buffer, "{}", 2); - --data->state->depth; + --data->depth; return; } @@ -1245,7 +1248,7 @@ static void generate_json_object(FBuffer *buffer, struct generate_json_data *dat }; rb_hash_foreach(obj, json_object_i, (VALUE)&arg); - depth = --data->state->depth; + depth = --data->depth; if (RB_UNLIKELY(data->state->object_nl)) { fbuffer_append_str(buffer, data->state->object_nl); if (RB_UNLIKELY(data->state->indent)) { @@ -1261,7 +1264,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data if (RARRAY_LEN(obj) == 0) { fbuffer_append(buffer, "[]", 2); - --data->state->depth; + --data->depth; return; } @@ -1277,7 +1280,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data } generate_json(buffer, data, RARRAY_AREF(obj, i)); } - data->state->depth = --depth; + data->depth = --depth; if (RB_UNLIKELY(data->state->array_nl)) { fbuffer_append_str(buffer, data->state->array_nl); if (RB_UNLIKELY(data->state->indent)) { @@ -1358,7 +1361,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data if (casted_obj != obj) { increase_depth(data); generate_json(buffer, data, casted_obj); - data->state->depth--; + data->depth--; return; } } @@ -1477,6 +1480,7 @@ static VALUE generate_json_ensure(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; fbuffer_free(data->buffer); + data->state->depth = data->depth; return Qundef; } @@ -1495,6 +1499,7 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, .buffer = &buffer, .vstate = self, .state = state, + .depth = state->depth, .obj = obj, .func = func }; @@ -1541,6 +1546,7 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) .buffer = &buffer, .vstate = Qfalse, .state = &new_state, + .depth = new_state.depth, .obj = obj, .func = generate_json }; @@ -2061,6 +2067,7 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) .buffer = &buffer, .vstate = Qfalse, .state = &state, + .depth = state.depth, .obj = obj, .func = generate_json, }; From 873b29ea34488f3bca7d8a4266bbdf228f913c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Sat, 22 Nov 2025 13:59:16 +0100 Subject: [PATCH 227/506] Don't write depth to JSON_Generator_State in some cases For `JSON.generate` and `JSON::State#generate_new`, don't copy generate_json_data::depth to JSON_Generator_State::depth. In `JSON.generate`, the JSON_Generator_State is on the stack and discarded anyway. In `JSON::State#generate_new`, we copy the struct to avoid mutating the original one. --- ext/json/ext/generator/generator.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 7125f4437..8f9080625 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1476,7 +1476,8 @@ static VALUE generate_json_try(VALUE d) return fbuffer_finalize(data->buffer); } -static VALUE generate_json_ensure(VALUE d) +// Preserves the deprecated behavior of State#depth being set. +static VALUE generate_json_ensure_deprecated(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; fbuffer_free(data->buffer); @@ -1485,6 +1486,14 @@ static VALUE generate_json_ensure(VALUE d) return Qundef; } +static VALUE generate_json_ensure(VALUE d) +{ + struct generate_json_data *data = (struct generate_json_data *)d; + fbuffer_free(data->buffer); + + return Qundef; +} + static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io) { GET_STATE(self); @@ -1503,7 +1512,7 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, .obj = obj, .func = func }; - return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure_deprecated, (VALUE)&data); } /* call-seq: From d7964f8892d7c8483c8c9aea612b32bc7b6281d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Sat, 22 Nov 2025 14:57:30 +0100 Subject: [PATCH 228/506] Don't copy JSON_Generator_State in generate_new Now that the state isn't mutated in generate_new, we no longer need to copy the struct, we can just use it. --- ext/json/ext/generator/generator.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 8f9080625..32a9b4851 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1539,12 +1539,6 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) GET_STATE(self); - JSON_Generator_State new_state; - MEMCPY(&new_state, state, JSON_Generator_State, 1); - - // FIXME: depth shouldn't be part of JSON_Generator_State, as that prevents it from being used concurrently. - new_state.depth = 0; - char stack_buffer[FBUFFER_STACK_SIZE]; FBuffer buffer = { .io = RTEST(io) ? io : Qfalse, @@ -1554,8 +1548,8 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) struct generate_json_data data = { .buffer = &buffer, .vstate = Qfalse, - .state = &new_state, - .depth = new_state.depth, + .state = state, + .depth = 0, .obj = obj, .func = generate_json }; From f92024a43eeca5fa8978110907fbdf842a7ebafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Tue, 25 Nov 2025 17:37:34 +0100 Subject: [PATCH 229/506] Stop testing on macos-13 These runners are closing down: https://github.blog/changelog/2025-09-19-github-actions-macos-13-runner-image-is-closing-down/ --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f85fded2..4a79be455 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: matrix: os: - ubuntu-latest - - macos-14 + - macos-latest - windows-latest ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }} env: @@ -32,7 +32,7 @@ jobs: - { os: ubuntu-24.04-arm, ruby: 3.4 } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } - - { os: macos-13, ruby: 3.4 } + - { os: macos-14, ruby: 3.4 } - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI - { os: macos-latest , ruby: jruby-9.4 } # Ruby 3.1 - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 From cca1f383167463172bf8cf58982161b7c5140967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 24 Nov 2025 17:36:01 +0100 Subject: [PATCH 230/506] Test current behavior regarding depth for Coder Coder currently ignores its depth and always resets it to 0 when generating a new JSON document. --- test/json/json_coder_test.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 83b89a3b1..1f9bc814e 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -132,6 +132,11 @@ def test_json_coder_string_invalid_encoding assert_equal 2, calls end + def test_depth + coder = JSON::Coder.new(object_nl: "\n", array_nl: "\n", space: " ", indent: " ", depth: 1) + assert_equal %({\n "foo": 42\n}), coder.dump(foo: 42) + end + def test_nesting_recovery coder = JSON::Coder.new ary = [] From 9c36681b173545ea0f97fdb84e2b1963fbbc0028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 24 Nov 2025 17:41:33 +0100 Subject: [PATCH 231/506] Respect Coder depth when generating --- ext/json/ext/generator/generator.c | 2 +- java/src/json/ext/GeneratorState.java | 6 ------ lib/json/truffle_ruby/generator.rb | 4 ---- test/json/json_coder_test.rb | 2 +- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 32a9b4851..6ece9f053 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1549,7 +1549,7 @@ static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) .buffer = &buffer, .vstate = Qfalse, .state = state, - .depth = 0, + .depth = state->depth, .obj = obj, .func = generate_json }; diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 19d6a1341..b12ca2c30 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -259,14 +259,12 @@ public IRubyObject generate(ThreadContext context, IRubyObject obj) { @JRubyMethod public IRubyObject generate_new(ThreadContext context, IRubyObject obj, IRubyObject io) { GeneratorState newState = (GeneratorState)dup(); - newState.resetDepth(); return newState.generate(context, obj, io); } @JRubyMethod public IRubyObject generate_new(ThreadContext context, IRubyObject obj) { GeneratorState newState = (GeneratorState)dup(); - newState.resetDepth(); return newState.generate(context, obj, context.nil); } @@ -504,10 +502,6 @@ public IRubyObject depth_set(IRubyObject vDepth) { return vDepth; } - public void resetDepth() { - depth = 0; - } - private ByteList prepareByteList(ThreadContext context, IRubyObject value) { RubyString str = value.convertToString(); if (str.getEncoding() != UTF8Encoding.INSTANCE) { diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 71a909dd6..9ac46cc81 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -352,10 +352,6 @@ def generate_new(obj, anIO = nil) # :nodoc: dup.generate(obj, anIO) end - private def initialize_copy(_orig) - @depth = 0 - end - # Handles @allow_nan, @buffer_initial_length, other ivars must be the default value (see above) private def generate_json(obj, buf) case obj diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 1f9bc814e..47e12ff91 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -134,7 +134,7 @@ def test_json_coder_string_invalid_encoding def test_depth coder = JSON::Coder.new(object_nl: "\n", array_nl: "\n", space: " ", indent: " ", depth: 1) - assert_equal %({\n "foo": 42\n}), coder.dump(foo: 42) + assert_equal %({\n "foo": 42\n }), coder.dump(foo: 42) end def test_nesting_recovery From 28c57df8f7be94b6313352c300c2ee558eaa8c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Tue, 2 Dec 2025 15:45:01 +0100 Subject: [PATCH 232/506] Don't call to_json on the return value of as_json for Float::NAN --- CHANGES.md | 3 ++- lib/json/truffle_ruby/generator.rb | 3 +++ test/json/json_generator_test.rb | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3ebbe1909..6befccdc2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,9 @@ ### Unreleased -* Fixed the parser to no longer ignore invalid escapes in strings. +* Fix the parser to no longer ignore invalid escapes in strings. Only `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t` and `\u` are valid JSON escapes. +* On TruffleRuby, fix the generator to not call `to_json` on the return value of `as_json` for `Float::NAN`. ### 2025-11-07 (2.16.0) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 9ac46cc81..5d815262b 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -651,6 +651,9 @@ def to_json(state = nil, *args) if casted_value.equal?(self) raise GeneratorError.new("#{self} not allowed in JSON", self) end + unless Generator.native_type?(casted_value) + raise GeneratorError.new("#{casted_value.class} returned by #{state.as_json} not allowed in JSON", casted_value) + end state.check_max_nesting state.depth += 1 diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index c01ed678f..9600f4be8 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -883,6 +883,15 @@ def test_json_generate_as_json_convert_to_proc assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: -> (o, is_key) { o.object_id }) end + def test_as_json_nan_does_not_call_to_json + def (obj = Object.new).to_json(*) + "null" + end + assert_raise(JSON::GeneratorError) do + JSON.generate(Float::NAN, strict: true, as_json: proc { obj }) + end + end + def assert_float_roundtrip(expected, actual) assert_equal(expected, JSON.generate(actual)) assert_equal(actual, JSON.parse(JSON.generate(actual)), "JSON: #{JSON.generate(actual)}") From 8fb727901e530dad3321ad6930d8f00dbe3d0b1d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 3 Dec 2025 10:56:53 +0900 Subject: [PATCH 233/506] Fix macro arguments `ALWAYS_INLINE()` and `NOINLINE()` are defined with one argument. --- ext/json/ext/generator/generator.c | 15 +++++++-------- ext/json/ext/parser/parser.c | 14 +++++++------- ext/json/ext/simd/simd.h | 10 +++++----- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 6ece9f053..35d543a7a 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -128,7 +128,7 @@ typedef struct _search_state { #endif /* HAVE_SIMD */ } search_state; -static ALWAYS_INLINE() void search_flush(search_state *search) +ALWAYS_INLINE(static) void search_flush(search_state *search) { // Do not remove this conditional without profiling, specifically escape-heavy text. // escape_UTF8_char_basic will advance search->ptr and search->cursor (effectively a search_flush). @@ -171,7 +171,7 @@ static inline unsigned char search_escape_basic(search_state *search) return 0; } -static ALWAYS_INLINE() void escape_UTF8_char_basic(search_state *search) +ALWAYS_INLINE(static) void escape_UTF8_char_basic(search_state *search) { const unsigned char ch = (unsigned char)*search->ptr; switch (ch) { @@ -258,7 +258,7 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) #ifdef HAVE_SIMD -static ALWAYS_INLINE() char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) +ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) { // Flush the buffer so everything up until the last 'len' characters are unflushed. search_flush(search); @@ -281,7 +281,7 @@ static ALWAYS_INLINE() char *copy_remaining_bytes(search_state *search, unsigned #ifdef HAVE_SIMD_NEON -static ALWAYS_INLINE() unsigned char neon_next_match(search_state *search) +ALWAYS_INLINE(static) unsigned char neon_next_match(search_state *search) { uint64_t mask = search->matches_mask; uint32_t index = trailing_zeros64(mask) >> 2; @@ -395,7 +395,7 @@ static inline unsigned char search_escape_basic_neon(search_state *search) #ifdef HAVE_SIMD_SSE2 -static ALWAYS_INLINE() unsigned char sse2_next_match(search_state *search) +ALWAYS_INLINE(static) unsigned char sse2_next_match(search_state *search) { int mask = search->matches_mask; int index = trailing_zeros(mask); @@ -419,7 +419,7 @@ static ALWAYS_INLINE() unsigned char sse2_next_match(search_state *search) #define TARGET_SSE2 #endif -static TARGET_SSE2 ALWAYS_INLINE() unsigned char search_escape_basic_sse2(search_state *search) +ALWAYS_INLINE(static) TARGET_SSE2 unsigned char search_escape_basic_sse2(search_state *search) { if (RB_UNLIKELY(search->has_matches)) { // There are more matches if search->matches_mask > 0. @@ -1123,8 +1123,7 @@ struct hash_foreach_arg { bool mixed_keys_encountered; }; -NOINLINE() -static void +NOINLINE(static) void json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) { if (arg->mixed_keys_encountered) { diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 0216eef98..5b7cd835c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -88,7 +88,7 @@ static void rvalue_cache_insert_at(rvalue_cache *cache, int index, VALUE rstring #if JSON_CPU_LITTLE_ENDIAN_64BITS #if __has_builtin(__builtin_bswap64) #undef rstring_cache_memcmp -static ALWAYS_INLINE() int rstring_cache_memcmp(const char *str, const char *rptr, const long length) +ALWAYS_INLINE(static) int rstring_cache_memcmp(const char *str, const char *rptr, const long length) { // The libc memcmp has numerous complex optimizations, but in this particular case, // we know the string is small (JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH), so being able to @@ -117,7 +117,7 @@ static ALWAYS_INLINE() int rstring_cache_memcmp(const char *str, const char *rpt #endif #endif -static ALWAYS_INLINE() int rstring_cache_cmp(const char *str, const long length, VALUE rstring) +ALWAYS_INLINE(static) int rstring_cache_cmp(const char *str, const long length, VALUE rstring) { const char *rstring_ptr; long rstring_length; @@ -131,7 +131,7 @@ static ALWAYS_INLINE() int rstring_cache_cmp(const char *str, const long length, } } -static ALWAYS_INLINE() VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) +ALWAYS_INLINE(static) VALUE rstring_cache_fetch(rvalue_cache *cache, const char *str, const long length) { int low = 0; int high = cache->length - 1; @@ -540,7 +540,7 @@ json_eat_comments(JSON_ParserState *state) } } -static ALWAYS_INLINE() void +ALWAYS_INLINE(static) void json_eat_whitespace(JSON_ParserState *state) { while (true) { @@ -661,7 +661,7 @@ static inline const char *json_next_backslash(const char *pe, const char *string return NULL; } -static NOINLINE() VALUE json_string_unescape(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name, JSON_UnescapePositions *positions) +NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_ParserConfig *config, const char *string, const char *stringEnd, bool is_name, JSON_UnescapePositions *positions) { bool intern = is_name || config->freeze; bool symbolize = is_name && config->symbolize_names; @@ -946,7 +946,7 @@ static const bool string_scan_table[256] = { static SIMD_Implementation simd_impl = SIMD_NONE; #endif /* HAVE_SIMD */ -static ALWAYS_INLINE() bool string_scan(JSON_ParserState *state) +ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state) { #ifdef HAVE_SIMD #if defined(HAVE_SIMD_NEON) @@ -1015,7 +1015,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi return Qfalse; } -static ALWAYS_INLINE() VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) +ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) { state->cursor++; const char *start = state->cursor; diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index c9e3b3ec7..f8e5ee188 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -73,14 +73,14 @@ static inline SIMD_Implementation find_simd_implementation(void) #define HAVE_SIMD_NEON 1 // See: https://community.arm.com/arm-community-blogs/b/servers-and-cloud-computing-blog/posts/porting-x86-vector-bitmask-optimizations-to-arm-neon -static ALWAYS_INLINE() uint64_t neon_match_mask(uint8x16_t matches) +ALWAYS_INLINE(static) uint64_t neon_match_mask(uint8x16_t matches) { const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches), 4); const uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(res), 0); return mask & 0x8888888888888888ull; } -static ALWAYS_INLINE() uint64_t compute_chunk_mask_neon(const char *ptr) +ALWAYS_INLINE(static) uint64_t compute_chunk_mask_neon(const char *ptr) { uint8x16_t chunk = vld1q_u8((const unsigned char *)ptr); @@ -93,7 +93,7 @@ static ALWAYS_INLINE() uint64_t compute_chunk_mask_neon(const char *ptr) return neon_match_mask(needs_escape); } -static ALWAYS_INLINE() int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) +ALWAYS_INLINE(static) int string_scan_simd_neon(const char **ptr, const char *end, uint64_t *mask) { while (*ptr + sizeof(uint8x16_t) <= end) { uint64_t chunk_mask = compute_chunk_mask_neon(*ptr); @@ -140,7 +140,7 @@ static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) #define _mm_cmpgt_epu8(a, b) _mm_xor_si128(_mm_cmple_epu8(a, b), _mm_set1_epi8(-1)) #define _mm_cmplt_epu8(a, b) _mm_cmpgt_epu8(b, a) -static TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr) +ALWAYS_INLINE(static) TARGET_SSE2 int compute_chunk_mask_sse2(const char *ptr) { __m128i chunk = _mm_loadu_si128((__m128i const*)ptr); // Trick: c < 32 || c == 34 can be factored as c ^ 2 < 33 @@ -151,7 +151,7 @@ static TARGET_SSE2 ALWAYS_INLINE() int compute_chunk_mask_sse2(const char *ptr) return _mm_movemask_epi8(needs_escape); } -static TARGET_SSE2 ALWAYS_INLINE() int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) +ALWAYS_INLINE(static) TARGET_SSE2 int string_scan_simd_sse2(const char **ptr, const char *end, int *mask) { while (*ptr + sizeof(__m128i) <= end) { int chunk_mask = compute_chunk_mask_sse2(*ptr); From c54de70f9084d16136e23e694033fb54bfdfcef3 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Dec 2025 14:41:35 +0100 Subject: [PATCH 234/506] Improve `JSON.load` and `JSON.unsafe_load` to allow passing options as second argument Otherwise it's very error prone. --- CHANGES.md | 1 + lib/json/common.rb | 21 +++++++++++++++++++-- test/json/json_common_interface_test.rb | 7 +++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6befccdc2..a5728aba7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Improve `JSON.load` and `JSON.unsafe_load` to allow passing options as second argument. * Fix the parser to no longer ignore invalid escapes in strings. Only `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t` and `\u` are valid JSON escapes. * On TruffleRuby, fix the generator to not call `to_json` on the return value of `as_json` for `Float::NAN`. diff --git a/lib/json/common.rb b/lib/json/common.rb index 233b8c7e6..fcdc0d9f0 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -550,6 +550,7 @@ def pretty_generate(obj, opts = nil) :create_additions => nil, } # :call-seq: + # JSON.unsafe_load(source, options = {}) -> object # JSON.unsafe_load(source, proc = nil, options = {}) -> object # # Returns the Ruby objects created by parsing the given +source+. @@ -681,7 +682,12 @@ def pretty_generate(obj, opts = nil) # def unsafe_load(source, proc = nil, options = nil) opts = if options.nil? - _unsafe_load_default_options + if proc && proc.is_a?(Hash) + options, proc = proc, nil + options + else + _unsafe_load_default_options + end else _unsafe_load_default_options.merge(options) end @@ -709,6 +715,7 @@ def unsafe_load(source, proc = nil, options = nil) end # :call-seq: + # JSON.load(source, options = {}) -> object # JSON.load(source, proc = nil, options = {}) -> object # # Returns the Ruby objects created by parsing the given +source+. @@ -845,8 +852,18 @@ def unsafe_load(source, proc = nil, options = nil) # @attributes={"type"=>"Admin", "password"=>"0wn3d"}>} # def load(source, proc = nil, options = nil) + if proc && options.nil? && proc.is_a?(Hash) + options = proc + proc = nil + end + opts = if options.nil? - _load_default_options + if proc && proc.is_a?(Hash) + options, proc = proc, nil + options + else + _load_default_options + end else _load_default_options.merge(options) end diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 37fa43957..13e2ca062 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -149,6 +149,7 @@ def test_load_with_proc def test_load_with_options json = '{ "foo": NaN }' assert JSON.load(json, nil, :allow_nan => true)['foo'].nan? + assert JSON.load(json, :allow_nan => true)['foo'].nan? end def test_load_null @@ -215,6 +216,12 @@ def test_unsafe_load_with_proc assert_equal expected, visited end + def test_unsafe_load_with_options + json = '{ "foo": NaN }' + assert JSON.unsafe_load(json, nil, :allow_nan => true)['foo'].nan? + assert JSON.unsafe_load(json, :allow_nan => true)['foo'].nan? + end + def test_unsafe_load_default_options too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' assert JSON.unsafe_load(too_deep, nil).is_a?(Array) From 386b36fde5ef3955fbbbc70b7f3423636a53d440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 27 Nov 2025 16:12:17 +0100 Subject: [PATCH 235/506] Test and restore behavior around to_json changing depth When serializing an Array, and one of the elements of the Array requires calling `to_json`, if the depth is changed, it will be used for the next entries, which wasn't the case before 5abd43490714460e1ec1fa382f82a09485eed563, and is not the case with TruffleRuby and JRuby. Additionally, with TruffleRuby and JRuby the state's depth after the `to_json` call is used to close the Array, which isn't the case with CRuby. --- ext/json/ext/generator/generator.c | 2 ++ test/json/json_generator_test.rb | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 35d543a7a..9e6e617a5 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1294,6 +1294,8 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d VALUE tmp; if (rb_respond_to(obj, i_to_json)) { tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data)); + GET_STATE(data->vstate); + data->depth = state->depth; Check_Type(tmp, T_STRING); fbuffer_append_str(buffer, tmp); } else { diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 9600f4be8..ab3a6807d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -321,6 +321,30 @@ def test_allow_nan end end + def test_depth_bad_to_json + obj = Object.new + def obj.to_json(state) + state.depth += 1 + "{#{state.object_nl}"\ + "#{state.indent * state.depth}\"foo\":#{state.space}1#{state.object_nl}"\ + "#{state.indent * (state.depth - 1)}}" + end + indent = " " * 2 if RUBY_ENGINE != "ruby" + assert_equal <<~JSON.chomp, JSON.pretty_generate([obj] * 2) + [ + { + "foo": 1 + }, + { + "foo": 1 + } + #{indent}] + JSON + state = JSON::State.new(object_nl: "\n", array_nl: "\n", space: " ", indent: " ") + state.generate(obj) + assert_equal 1, state.depth + end + def test_depth pretty = { object_nl: "\n", array_nl: "\n", space: " ", indent: " " } state = JSON.state.new(**pretty) From e0257b9f82fc0c5aa0de0816abe7d1798c1d6411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 28 Nov 2025 15:53:58 +0100 Subject: [PATCH 236/506] Reproduce C ext behavior of ignoring mutated depth in arrays --- java/src/json/ext/Generator.java | 4 ++-- java/src/json/ext/GeneratorState.java | 2 +- lib/json/truffle_ruby/generator.rb | 2 +- test/json/json_generator_test.rb | 8 +++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 78281dbd4..0caa6948e 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -462,10 +462,10 @@ static void generateArray(ThreadContext context, Session session, RubyArray Date: Wed, 3 Dec 2025 15:13:00 +0100 Subject: [PATCH 237/506] Fix duplicated test_unsafe_load_with_options test case --- test/json/json_common_interface_test.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 13e2ca062..3dfd0623c 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -216,12 +216,6 @@ def test_unsafe_load_with_proc assert_equal expected, visited end - def test_unsafe_load_with_options - json = '{ "foo": NaN }' - assert JSON.unsafe_load(json, nil, :allow_nan => true)['foo'].nan? - assert JSON.unsafe_load(json, :allow_nan => true)['foo'].nan? - end - def test_unsafe_load_default_options too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' assert JSON.unsafe_load(too_deep, nil).is_a?(Array) @@ -237,6 +231,7 @@ def test_unsafe_load_with_options assert_raise(JSON::ParserError) { JSON.unsafe_load(nan_json, nil, :allow_nan => false)['foo'].nan? } # make sure it still uses the defaults when something is provided assert JSON.unsafe_load(nan_json, nil, :allow_blank => true)['foo'].nan? + assert JSON.unsafe_load(nan_json, :allow_nan => true)['foo'].nan? end def test_unsafe_load_null From ccca602274401eedf894c0e633040665dc425e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Wed, 3 Dec 2025 15:43:28 +0100 Subject: [PATCH 238/506] Fix handling of depth --- CHANGES.md | 5 ++ ext/json/ext/generator/generator.c | 50 +++-------------- java/src/json/ext/Generator.java | 43 ++++++++++----- java/src/json/ext/GeneratorState.java | 12 ----- lib/json/common.rb | 2 +- lib/json/truffle_ruby/generator.rb | 22 +++++--- test/json/json_generator_test.rb | 77 ++++++++++++++++++++++----- 7 files changed, 124 insertions(+), 87 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a5728aba7..06e56aed3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,12 @@ * Improve `JSON.load` and `JSON.unsafe_load` to allow passing options as second argument. * Fix the parser to no longer ignore invalid escapes in strings. Only `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t` and `\u` are valid JSON escapes. +* Fixed `JSON::Coder` to use the depth it was initialized with. * On TruffleRuby, fix the generator to not call `to_json` on the return value of `as_json` for `Float::NAN`. +* Fixed handling of `state.depth`: when `to_json` changes `state.depth` but does not restore it, it is reset + automatically to its initial value. + In particular, when a `NestingError` is raised, `depth` is no longer equal to `max_nesting` after the call to + generate, and is reset to its initial value. Similarly when `to_json` raises an exception. ### 2025-11-07 (2.16.0) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9e6e617a5..d202e97ea 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -968,14 +968,16 @@ static void vstate_spill(struct generate_json_data *data) RB_OBJ_WRITTEN(vstate, Qundef, state->as_json); } -static inline VALUE vstate_get(struct generate_json_data *data) +static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj) { if (RB_UNLIKELY(!data->vstate)) { vstate_spill(data); } GET_STATE(data->vstate); state->depth = data->depth; - return data->vstate; + VALUE tmp = rb_funcall(obj, i_to_json, 1, data->vstate); + // no need to restore state->depth, vstate is just a temporary State + return tmp; } static VALUE @@ -1293,9 +1295,7 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d { VALUE tmp; if (rb_respond_to(obj, i_to_json)) { - tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data)); - GET_STATE(data->vstate); - data->depth = state->depth; + tmp = json_call_to_json(data, obj); Check_Type(tmp, T_STRING); fbuffer_append_str(buffer, tmp); } else { @@ -1477,16 +1477,6 @@ static VALUE generate_json_try(VALUE d) return fbuffer_finalize(data->buffer); } -// Preserves the deprecated behavior of State#depth being set. -static VALUE generate_json_ensure_deprecated(VALUE d) -{ - struct generate_json_data *data = (struct generate_json_data *)d; - fbuffer_free(data->buffer); - data->state->depth = data->depth; - - return Qundef; -} - static VALUE generate_json_ensure(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; @@ -1507,13 +1497,13 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, struct generate_json_data data = { .buffer = &buffer, - .vstate = self, + .vstate = Qfalse, // don't use self as it may be frozen and its depth is mutated when calling to_json .state = state, .depth = state->depth, .obj = obj, .func = func }; - return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure_deprecated, (VALUE)&data); + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } /* call-seq: @@ -1532,31 +1522,6 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self) return cState_partial_generate(self, obj, generate_json, io); } -static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 1, 2); - VALUE obj = argv[0]; - VALUE io = argc > 1 ? argv[1] : Qnil; - - GET_STATE(self); - - char stack_buffer[FBUFFER_STACK_SIZE]; - FBuffer buffer = { - .io = RTEST(io) ? io : Qfalse, - }; - fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE); - - struct generate_json_data data = { - .buffer = &buffer, - .vstate = Qfalse, - .state = state, - .depth = state->depth, - .obj = obj, - .func = generate_json - }; - return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); -} - static VALUE cState_initialize(int argc, VALUE *argv, VALUE self) { rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`"); @@ -2145,7 +2110,6 @@ void Init_generator(void) rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0); rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1); rb_define_method(cState, "generate", cState_generate, -1); - rb_define_method(cState, "generate_new", cState_generate_new, -1); // :nodoc: rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 0caa6948e..ea9e6d083 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -54,7 +54,13 @@ static RubyString generateJson(ThreadContext context, T static RubyString generateJson(ThreadContext context, T object, Handler handler, IRubyObject arg0) { Session session = new Session(arg0); - return handler.generateNew(context, session, object); + GeneratorState state = session.getState(context); + int depth = state.depth; + try { + return handler.generateNew(context, session, object); + } finally { + state.depth = depth; + } } /** @@ -79,18 +85,25 @@ static RubyString generateJson(ThreadContext context, T generateJson(ThreadContext context, T object, GeneratorState config, IRubyObject io) { Session session = new Session(config); - Handler handler = getHandlerFor(context.runtime, object); + GeneratorState state = session.getState(context); + int depth = state.depth; - if (io.isNil()) { - return handler.generateNew(context, session, object); - } + try { + Handler handler = getHandlerFor(context.runtime, object); + + if (io.isNil()) { + return handler.generateNew(context, session, object); + } - BufferedOutputStream buffer = - new BufferedOutputStream( - new PatchedIOOutputStream(io, UTF8Encoding.INSTANCE), - IO_BUFFER_SIZE); - handler.generateToBuffer(context, session, object, buffer); - return io; + BufferedOutputStream buffer = + new BufferedOutputStream( + new PatchedIOOutputStream(io, UTF8Encoding.INSTANCE), + IO_BUFFER_SIZE); + handler.generateToBuffer(context, session, object, buffer); + return io; + } finally { + state.depth = depth; + } } /** @@ -784,7 +797,13 @@ static RubyString generateGenericNew(ThreadContext context, Session session, IRu } throw Utils.buildGeneratorError(context, object, object + " not allowed in JSON").toThrowable(); } else if (object.respondsTo("to_json")) { - IRubyObject result = object.callMethod(context, "to_json", state); + int depth = state.depth; + IRubyObject result; + try { + result = object.callMethod(context, "to_json", state); + } finally { + state.depth = depth; + } if (result instanceof RubyString) return (RubyString)result; throw context.runtime.newTypeError("to_json must return a String"); } else { diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 327405a8f..8db31bdaa 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -256,18 +256,6 @@ public IRubyObject generate(ThreadContext context, IRubyObject obj) { return generate(context, obj, context.nil); } - @JRubyMethod - public IRubyObject generate_new(ThreadContext context, IRubyObject obj, IRubyObject io) { - GeneratorState newState = (GeneratorState)dup(); - return newState.generate(context, obj, io); - } - - @JRubyMethod - public IRubyObject generate_new(ThreadContext context, IRubyObject obj) { - GeneratorState newState = (GeneratorState)dup(); - return newState.generate(context, obj, context.nil); - } - @JRubyMethod(name="[]") public IRubyObject op_aref(ThreadContext context, IRubyObject vName) { String name = vName.asJavaString(); diff --git a/lib/json/common.rb b/lib/json/common.rb index fcdc0d9f0..877b96814 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1074,7 +1074,7 @@ def initialize(options = nil, &as_json) # # Serialize the given object into a \JSON document. def dump(object, io = nil) - @state.generate_new(object, io) + @state.generate(object, io) end alias_method :generate, :dump diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 7b9657a3f..d71188849 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -330,6 +330,9 @@ def to_h # created this method raises a # GeneratorError exception. def generate(obj, anIO = nil) + return dup.generate(obj, anIO) if frozen? + + depth = @depth if @indent.empty? and @space.empty? and @space_before.empty? and @object_nl.empty? and @array_nl.empty? and !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) result = generate_json(obj, ''.dup) @@ -346,10 +349,8 @@ def generate(obj, anIO = nil) else result end - end - - def generate_new(obj, anIO = nil) # :nodoc: - dup.generate(obj, anIO) + ensure + @depth = depth unless frozen? end # Handles @allow_nan, @buffer_initial_length, other ivars must be the default value (see above) @@ -490,8 +491,11 @@ module Hash # _depth_ is used to find out nesting depth, to indent accordingly. def to_json(state = nil, *) state = State.from_state(state) + depth = state.depth state.check_max_nesting json_transform(state) + ensure + state.depth = depth end private @@ -555,17 +559,19 @@ def json_transform(state) raise GeneratorError.new("#{value.class} returned by #{state.as_json} not allowed in JSON", value) end result << value.to_json(state) + state.depth = depth else raise GeneratorError.new("#{value.class} not allowed in JSON", value) end elsif value.respond_to?(:to_json) result << value.to_json(state) + state.depth = depth else result << %{"#{String(value)}"} end first = false } - depth = state.depth -= 1 + depth -= 1 unless first result << state.object_nl result << state.indent * depth if indent @@ -582,8 +588,11 @@ module Array # produced JSON string output further. def to_json(state = nil, *) state = State.from_state(state) + depth = state.depth state.check_max_nesting json_transform(state) + ensure + state.depth = depth end private @@ -621,12 +630,13 @@ def json_transform(state) end elsif value.respond_to?(:to_json) result << value.to_json(state) + state.depth = depth else result << %{"#{String(value)}"} end first = false } - state.depth = depth -= 1 + depth -= 1 result << state.array_nl result << state.indent * depth if indent result << ']' diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index e623e0540..9f8b35de0 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -321,7 +321,8 @@ def test_allow_nan end end - def test_depth_bad_to_json + # An object that changes state.depth when it receives to_json(state) + def bad_to_json obj = Object.new def obj.to_json(state) state.depth += 1 @@ -329,21 +330,44 @@ def obj.to_json(state) "#{state.indent * state.depth}\"foo\":#{state.space}1#{state.object_nl}"\ "#{state.indent * (state.depth - 1)}}" end - assert_equal <<~JSON.chomp, JSON.pretty_generate([obj] * 2) + obj + end + + def test_depth_restored_bad_to_json + state = JSON::State.new + state.generate(bad_to_json) + assert_equal 0, state.depth + end + + def test_depth_restored_bad_to_json_in_Array + assert_equal <<~JSON.chomp, JSON.pretty_generate([bad_to_json] * 2) [ { "foo": 1 }, { - "foo": 1 - } + "foo": 1 + } ] JSON - state = JSON::State.new(object_nl: "\n", array_nl: "\n", space: " ", indent: " ") - state.generate(obj) - assert_equal 1, state.depth # FIXME - state.depth = 0 - state.generate([obj]) + state = JSON::State.new + state.generate([bad_to_json]) + assert_equal 0, state.depth + end + + def test_depth_restored_bad_to_json_in_Hash + assert_equal <<~JSON.chomp, JSON.pretty_generate(a: bad_to_json, b: bad_to_json) + { + "a": { + "foo": 1 + }, + "b": { + "foo": 1 + } + } + JSON + state = JSON::State.new + state.generate(a: bad_to_json) assert_equal 0, state.depth end @@ -361,10 +385,36 @@ def test_depth_nesting_error ary = []; ary << ary assert_raise(JSON::NestingError) { generate(ary) } assert_raise(JSON::NestingError) { JSON.pretty_generate(ary) } - s = JSON.state.new - assert_equal 0, s.depth + end + + def test_depth_nesting_error_to_json + ary = []; ary << ary + s = JSON.state.new(depth: 1) assert_raise(JSON::NestingError) { ary.to_json(s) } - assert_equal 100, s.depth + assert_equal 1, s.depth + end + + def test_depth_nesting_error_Hash_to_json + hash = {}; hash[:a] = hash + s = JSON.state.new(depth: 1) + assert_raise(JSON::NestingError) { hash.to_json(s) } + assert_equal 1, s.depth + end + + def test_depth_nesting_error_generate + ary = []; ary << ary + s = JSON.state.new(depth: 1) + assert_raise(JSON::NestingError) { s.generate(ary) } + assert_equal 1, s.depth + end + + def test_depth_exception_calling_to_json + def (obj = Object.new).to_json(*) + raise + end + s = JSON.state.new(depth: 1).freeze + assert_raise(RuntimeError) { s.generate([{ hash: obj }]) } + assert_equal 1, s.depth end def test_buffer_initial_length @@ -1006,7 +1056,8 @@ def test_nesting_recovery state = JSON::State.new ary = [] ary << ary - assert_raise(JSON::NestingError) { state.generate_new(ary) } + assert_raise(JSON::NestingError) { state.generate(ary) } + assert_equal 0, state.depth assert_equal '{"a":1}', state.generate({ a: 1 }) end end From 4bdb2d14fe30733b02e4c58ce9521d70ee56f2af Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Dec 2025 16:23:14 +0100 Subject: [PATCH 239/506] Release 2.17.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 06e56aed3..eb86d86b4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-12-03 (2.17.0) + * Improve `JSON.load` and `JSON.unsafe_load` to allow passing options as second argument. * Fix the parser to no longer ignore invalid escapes in strings. Only `\"`, `\\`, `\b`, `\f`, `\n`, `\r`, `\t` and `\u` are valid JSON escapes. diff --git a/lib/json/version.rb b/lib/json/version.rb index cc25a0453..b7de7c27e 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.16.0' + VERSION = '2.17.0' end From 0fce370c417bc16e7deed688e2d2a5ce6a0b31c7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 4 Dec 2025 09:06:10 +0100 Subject: [PATCH 240/506] Fix a regression in parsing of unicode surogate pairs Fix: https://github.com/ruby/json/issues/912 In the case of surogate pairs we consume two backslashes, so `json_next_backslash` need to ensure it's not sending us back in the stream. --- CHANGES.md | 2 ++ ext/json/ext/parser/parser.c | 4 +++- test/json/json_parser_test.rb | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index eb86d86b4..ba7010dfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix a regression in parsing of unicode surogate pairs (`\uXX\uXX`) that could cause an invalid string to be returned. + ### 2025-12-03 (2.17.0) * Improve `JSON.load` and `JSON.unsafe_load` to allow passing options as second argument. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 5b7cd835c..c84c7ed66 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -651,7 +651,9 @@ static inline const char *json_next_backslash(const char *pe, const char *string positions->size--; const char *next_position = positions->positions[0]; positions->positions++; - return next_position; + if (next_position >= pe) { + return next_position; + } } if (positions->has_more) { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3e662bda3..257e4f173 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -325,6 +325,13 @@ def test_invalid_unicode_escape assert_raise(JSON::ParserError) { parse('"\u111___"') } end + def test_unicode_followed_by_newline + # Ref: https://github.com/ruby/json/issues/912 + assert_equal "🌌\n".bytes, JSON.parse('"\ud83c\udf0c\n"').bytes + assert_equal "🌌\n", JSON.parse('"\ud83c\udf0c\n"') + assert_predicate JSON.parse('"\ud83c\udf0c\n"'), :valid_encoding? + end + def test_invalid_surogates assert_raise(JSON::ParserError) { parse('"\\uD800"') } assert_raise(JSON::ParserError) { parse('"\\uD800_________________"') } From e5e4fd558e5a8d53db63eeaeca9212e8f0042e19 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 4 Dec 2025 09:12:10 +0100 Subject: [PATCH 241/506] Release 2.17.1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ba7010dfd..8417ecd5e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-12-04 (2.17.1) + * Fix a regression in parsing of unicode surogate pairs (`\uXX\uXX`) that could cause an invalid string to be returned. ### 2025-12-03 (2.17.0) diff --git a/lib/json/version.rb b/lib/json/version.rb index b7de7c27e..4ed61c43c 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.17.0' + VERSION = '2.17.1' end From 4218a425effd526a45bdf87addf561f75808755d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 4 Dec 2025 11:50:13 +0100 Subject: [PATCH 242/506] Remove dead code in benchmark/encoder.rb --- benchmark/encoder.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmark/encoder.rb b/benchmark/encoder.rb index edda4e053..4cc8891d2 100644 --- a/benchmark/encoder.rb +++ b/benchmark/encoder.rb @@ -18,7 +18,6 @@ end def implementations(ruby_obj) - state = JSON::State.new(JSON.dump_default_options) coder = JSON::Coder.new implementations = { json: ["json", proc { JSON.generate(ruby_obj) }], From cf3993ca7d68c6060e49cb3f15aef7ee407b2bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 4 Dec 2025 11:52:19 +0100 Subject: [PATCH 243/506] Remove dead code from JSON::TruffleRuby::Generator --- lib/json/truffle_ruby/generator.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index d71188849..9a3f4aed0 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -500,11 +500,6 @@ def to_json(state = nil, *) private - def json_shift(state) - state.object_nl.empty? or return '' - state.indent * state.depth - end - def json_transform(state) depth = state.depth += 1 From 1da3fd9233e5a8232002ecf8ffab73e3c8c5c135 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 10 Dec 2025 10:54:18 +0100 Subject: [PATCH 244/506] Add a specific error for unescaped newlines It's the most likely control character so it's worth giving a better error message for it. --- ext/json/ext/parser/parser.c | 3 +++ test/json/json_parser_test.rb | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c84c7ed66..45de8d1ff 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -752,6 +752,9 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser break; default: if ((unsigned char)*pe < 0x20) { + if (*pe == '\n') { + raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); + } raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); } raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 257e4f173..3f0fb7522 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -164,6 +164,14 @@ def test_parse_complex_objects end end + def test_parse_control_chars_in_string + 0.upto(31) do |ord| + assert_raise JSON::ParserError do + parse(%("#{ord.chr}")) + end + end + end + def test_parse_arrays assert_equal([1,2,3], parse('[1,2,3]')) assert_equal([1.2,2,3], parse('[1.2,2,3]')) From 3459499cb3074cc8f0c8d70fac7d966edf9b0538 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 11 Dec 2025 11:09:05 +0100 Subject: [PATCH 245/506] Add `allow_control_characters` parsing option While it's not allowed by the spec, some parsers like Oj do accept it, and it can be blocking a transition. Having this feature can help people migrate. --- CHANGES.md | 2 + ext/json/ext/parser/parser.c | 36 +++--- java/src/json/ext/ParserConfig.java | 168 ++++++++++++++------------- java/src/json/ext/ParserConfig.rl | 9 +- java/src/json/ext/StringDecoder.java | 22 ++++ test/json/json_parser_test.rb | 6 + 6 files changed, 143 insertions(+), 100 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8417ecd5e..1f1e7e224 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Add `:allow_control_characters` parser options, to allow JSON strings containing unescaped ASCII control characters (e.g. newlines). + ### 2025-12-04 (2.17.1) * Fix a regression in parsing of unicode surogate pairs (`\uXX\uXX`) that could cause an invalid string to be returned. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 45de8d1ff..8f9729ef2 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -7,7 +7,7 @@ static VALUE CNaN, CInfinity, CMinusInfinity; static ID i_new, i_try_convert, i_uminus, i_encode; -static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_symbolize_names, sym_freeze, +static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key; static int binary_encindex; @@ -335,6 +335,7 @@ typedef struct JSON_ParserStruct { int max_nesting; bool allow_nan; bool allow_trailing_comma; + bool allow_control_characters; bool symbolize_names; bool freeze; } JSON_ParserConfig; @@ -752,12 +753,15 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser break; default: if ((unsigned char)*pe < 0x20) { - if (*pe == '\n') { - raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); + if (!config->allow_control_characters) { + if (*pe == '\n') { + raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); + } + raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); } - raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); + } else { + raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); } - raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); break; } } @@ -1009,7 +1013,9 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi break; } default: - raise_parse_error("invalid ASCII control character in string: %s", state); + if (!config->allow_control_characters) { + raise_parse_error("invalid ASCII control character in string: %s", state); + } break; } @@ -1430,14 +1436,15 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) { JSON_ParserConfig *config = (JSON_ParserConfig *)data; - if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; } - else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } - else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } - else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } - else if (key == sym_freeze) { config->freeze = RTEST(val); } - else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } - else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } - else if (key == sym_decimal_class) { + if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; } + else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } + else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } + else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); } + else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } + else if (key == sym_freeze) { config->freeze = RTEST(val); } + else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } + else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } + else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { config->decimal_class = val; @@ -1650,6 +1657,7 @@ void Init_parser(void) sym_max_nesting = ID2SYM(rb_intern("max_nesting")); sym_allow_nan = ID2SYM(rb_intern("allow_nan")); sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma")); + sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters")); sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); sym_freeze = ID2SYM(rb_intern("freeze")); sym_on_load = ID2SYM(rb_intern("on_load")); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 02bd7e5e2..23231646f 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -54,6 +54,7 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowControlCharacters; private boolean allowDuplicateKey; private boolean deprecateDuplicateKey; private boolean symbolizeNames; @@ -178,6 +179,7 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { OptionsReader opts = new OptionsReader(context, options); this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); + this.allowControlCharacters = opts.getBool("allow_control_characters", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); if (opts.hasKey("allow_duplicate_key")) { @@ -288,7 +290,7 @@ private ParserSession(ParserConfig config, RubyString source, ThreadContext cont this.byteList = source.getByteList(); this.data = byteList.unsafeBytes(); this.view = new ByteList(data, false); - this.decoder = new StringDecoder(); + this.decoder = new StringDecoder(config.allowControlCharacters); } private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { @@ -303,11 +305,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 328 "ParserConfig.rl" +// line 331 "ParserConfig.rl" -// line 310 "ParserConfig.java" +// line 313 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -421,7 +423,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 434 "ParserConfig.rl" +// line 437 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -429,14 +431,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 432 "ParserConfig.java" +// line 435 "ParserConfig.java" { cs = JSON_value_start; } -// line 441 "ParserConfig.rl" +// line 444 "ParserConfig.rl" -// line 439 "ParserConfig.java" +// line 442 "ParserConfig.java" { int _klen; int _trans = 0; @@ -462,13 +464,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 419 "ParserConfig.rl" +// line 422 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 471 "ParserConfig.java" +// line 474 "ParserConfig.java" } } @@ -531,25 +533,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 336 "ParserConfig.rl" +// line 339 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 339 "ParserConfig.rl" +// line 342 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 342 "ParserConfig.rl" +// line 345 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 345 "ParserConfig.rl" +// line 348 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -559,7 +561,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 352 "ParserConfig.rl" +// line 355 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -569,7 +571,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 359 "ParserConfig.rl" +// line 362 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -598,7 +600,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 385 "ParserConfig.rl" +// line 388 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -611,7 +613,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 395 "ParserConfig.rl" +// line 398 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -626,7 +628,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 407 "ParserConfig.rl" +// line 410 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -640,7 +642,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 643 "ParserConfig.java" +// line 646 "ParserConfig.java" } } } @@ -660,7 +662,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 442 "ParserConfig.rl" +// line 445 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -673,7 +675,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 676 "ParserConfig.java" +// line 679 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -772,7 +774,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 464 "ParserConfig.rl" +// line 467 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -789,15 +791,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 792 "ParserConfig.java" +// line 795 "ParserConfig.java" { cs = JSON_integer_start; } -// line 480 "ParserConfig.rl" +// line 483 "ParserConfig.rl" int memo = p; -// line 800 "ParserConfig.java" +// line 803 "ParserConfig.java" { int _klen; int _trans = 0; @@ -878,13 +880,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 458 "ParserConfig.rl" +// line 461 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 887 "ParserConfig.java" +// line 890 "ParserConfig.java" } } } @@ -904,7 +906,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 482 "ParserConfig.rl" +// line 485 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -924,7 +926,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 927 "ParserConfig.java" +// line 930 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1026,7 +1028,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 515 "ParserConfig.rl" +// line 518 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1045,15 +1047,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1048 "ParserConfig.java" +// line 1051 "ParserConfig.java" { cs = JSON_float_start; } -// line 533 "ParserConfig.rl" +// line 536 "ParserConfig.rl" int memo = p; -// line 1056 "ParserConfig.java" +// line 1059 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1134,13 +1136,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 506 "ParserConfig.rl" +// line 509 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1143 "ParserConfig.java" +// line 1146 "ParserConfig.java" } } } @@ -1160,7 +1162,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 535 "ParserConfig.rl" +// line 538 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1170,7 +1172,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1173 "ParserConfig.java" +// line 1176 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1184,7 +1186,7 @@ private static byte[] init__JSON_string_actions_0() private static byte[] init__JSON_string_key_offsets_0() { return new byte [] { - 0, 0, 1, 5, 8, 14, 20, 26, 32 + 0, 0, 1, 3, 4, 10, 16, 22, 28 }; } @@ -1194,9 +1196,9 @@ private static byte[] init__JSON_string_key_offsets_0() private static char[] init__JSON_string_trans_keys_0() { return new char [] { - 34, 34, 92, 0, 31, 117, 0, 31, 48, 57, 65, 70, - 97, 102, 48, 57, 65, 70, 97, 102, 48, 57, 65, 70, - 97, 102, 48, 57, 65, 70, 97, 102, 0 + 34, 34, 92, 117, 48, 57, 65, 70, 97, 102, 48, 57, + 65, 70, 97, 102, 48, 57, 65, 70, 97, 102, 48, 57, + 65, 70, 97, 102, 0 }; } @@ -1216,7 +1218,7 @@ private static byte[] init__JSON_string_single_lengths_0() private static byte[] init__JSON_string_range_lengths_0() { return new byte [] { - 0, 0, 1, 1, 3, 3, 3, 3, 0 + 0, 0, 0, 0, 3, 3, 3, 3, 0 }; } @@ -1226,7 +1228,7 @@ private static byte[] init__JSON_string_range_lengths_0() private static byte[] init__JSON_string_index_offsets_0() { return new byte [] { - 0, 0, 2, 6, 9, 13, 17, 21, 25 + 0, 0, 2, 5, 7, 11, 15, 19, 23 }; } @@ -1236,9 +1238,9 @@ private static byte[] init__JSON_string_index_offsets_0() private static byte[] init__JSON_string_indicies_0() { return new byte [] { - 0, 1, 2, 3, 1, 0, 4, 1, 0, 5, 5, 5, - 1, 6, 6, 6, 1, 7, 7, 7, 1, 0, 0, 0, - 1, 1, 0 + 0, 1, 2, 3, 0, 4, 0, 5, 5, 5, 1, 6, + 6, 6, 1, 7, 7, 7, 1, 0, 0, 0, 1, 1, + 0 }; } @@ -1272,7 +1274,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 574 "ParserConfig.rl" +// line 577 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1280,15 +1282,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1283 "ParserConfig.java" +// line 1286 "ParserConfig.java" { cs = JSON_string_start; } -// line 581 "ParserConfig.rl" +// line 584 "ParserConfig.rl" int memo = p; -// line 1291 "ParserConfig.java" +// line 1294 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1369,7 +1371,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 549 "ParserConfig.rl" +// line 552 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1384,13 +1386,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 562 "ParserConfig.rl" +// line 565 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1393 "ParserConfig.java" +// line 1396 "ParserConfig.java" } } } @@ -1410,7 +1412,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 583 "ParserConfig.rl" +// line 586 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1431,7 +1433,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1434 "ParserConfig.java" +// line 1437 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1598,7 +1600,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 637 "ParserConfig.rl" +// line 640 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1612,14 +1614,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); -// line 1615 "ParserConfig.java" +// line 1618 "ParserConfig.java" { cs = JSON_array_start; } -// line 650 "ParserConfig.rl" +// line 653 "ParserConfig.rl" -// line 1622 "ParserConfig.java" +// line 1625 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1662,7 +1664,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 608 "ParserConfig.rl" +// line 611 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1732,7 +1734,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 610 "ParserConfig.rl" +// line 613 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1745,13 +1747,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 621 "ParserConfig.rl" +// line 624 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1754 "ParserConfig.java" +// line 1757 "ParserConfig.java" } } } @@ -1771,7 +1773,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 651 "ParserConfig.rl" +// line 654 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1781,7 +1783,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1784 "ParserConfig.java" +// line 1787 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1958,7 +1960,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 721 "ParserConfig.rl" +// line 724 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -1975,14 +1977,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyHash.newHash(context.runtime); -// line 1978 "ParserConfig.java" +// line 1981 "ParserConfig.java" { cs = JSON_object_start; } -// line 737 "ParserConfig.rl" +// line 740 "ParserConfig.rl" -// line 1985 "ParserConfig.java" +// line 1988 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2025,7 +2027,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 665 "ParserConfig.rl" +// line 668 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2095,7 +2097,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 667 "ParserConfig.rl" +// line 670 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2108,7 +2110,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 678 "ParserConfig.rl" +// line 681 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2139,13 +2141,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 707 "ParserConfig.rl" +// line 710 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2148 "ParserConfig.java" +// line 2151 "ParserConfig.java" } } } @@ -2165,7 +2167,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 738 "ParserConfig.rl" +// line 741 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2176,7 +2178,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2179 "ParserConfig.java" +// line 2182 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2279,7 +2281,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 767 "ParserConfig.rl" +// line 770 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2289,16 +2291,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2292 "ParserConfig.java" +// line 2295 "ParserConfig.java" { cs = JSON_start; } -// line 776 "ParserConfig.rl" +// line 779 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2301 "ParserConfig.java" +// line 2304 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2379,7 +2381,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 753 "ParserConfig.rl" +// line 756 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2391,7 +2393,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2394 "ParserConfig.java" +// line 2397 "ParserConfig.java" } } } @@ -2411,7 +2413,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 779 "ParserConfig.rl" +// line 782 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 533b13227..a1c9ea7f3 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -52,6 +52,7 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowControlCharacters; private boolean allowDuplicateKey; private boolean deprecateDuplicateKey; private boolean symbolizeNames; @@ -170,11 +171,13 @@ public class ParserConfig extends RubyObject { @JRubyMethod(visibility = Visibility.PRIVATE) public IRubyObject initialize(ThreadContext context, IRubyObject options) { + checkFrozen(); Ruby runtime = context.runtime; OptionsReader opts = new OptionsReader(context, options); this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); + this.allowControlCharacters = opts.getBool("allow_control_characters", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); if (opts.hasKey("allow_duplicate_key")) { @@ -285,7 +288,7 @@ public class ParserConfig extends RubyObject { this.byteList = source.getByteList(); this.data = byteList.unsafeBytes(); this.view = new ByteList(data, false); - this.decoder = new StringDecoder(); + this.decoder = new StringDecoder(config.allowControlCharacters); } private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { @@ -565,10 +568,10 @@ public class ParserConfig extends RubyObject { } main := '"' - ( ( ^(["\\]|0..0x1f) + ( ( ^(["\\]) | '\\'["\\/bfnrt] | '\\u'[0-9a-fA-F]{4} - | '\\'^(["\\/bfnrtu]|0..0x1f) + | '\\'^(["\\/bfnrtu]) )* %parse_string ) '"' @exit; }%% diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index 03c78514a..a588d94d7 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -22,12 +22,17 @@ final class StringDecoder extends ByteListTranscoder { * or -1 when not. */ private int surrogatePairStart = -1; + private boolean allowControlCharacters = false; private ByteList out; // Array used for writing multibyte characters into the buffer at once private final byte[] aux = new byte[4]; + public StringDecoder(boolean allowControlCharacters) { + this.allowControlCharacters = allowControlCharacters; + } + ByteList decode(ThreadContext context, ByteList src, int start, int end) { try { init(src, start, end); @@ -42,6 +47,15 @@ ByteList decode(ThreadContext context, ByteList src, int start, int end) { } } + @Override + protected int readUtf8Char(ThreadContext context) { + int c = super.readUtf8Char(context); + if (c < 0x20 && !allowControlCharacters) { + throw invalidControlChar(context); + } + return c; + } + private void handleChar(ThreadContext context, int c) throws IOException { if (c == '\\') { quoteStop(charStart); @@ -184,6 +198,14 @@ protected RaiseException invalidUtf8(ThreadContext context) { context.runtime.newString(message)); } + protected RaiseException invalidControlChar(ThreadContext context) { + ByteList message = new ByteList( + ByteList.plain("invalid ASCII control character in string: ")); + message.append(src, charStart, srcEnd - charStart); + return Utils.newException(context, Utils.M_PARSER_ERROR, + context.runtime.newString(message)); + } + protected RaiseException invalidEscape(ThreadContext context) { ByteList message = new ByteList( ByteList.plain("invalid escape character in string: ")); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3f0fb7522..d29f8077b 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -172,6 +172,12 @@ def test_parse_control_chars_in_string end end + def test_parse_allowed_control_chars_in_string + 0.upto(31) do |ord| + assert_equal ord.chr, parse(%("#{ord.chr}"), allow_control_characters: true) + end + end + def test_parse_arrays assert_equal([1,2,3], parse('[1,2,3]')) assert_equal([1.2,2,3], parse('[1.2,2,3]')) From 1cdd2122d537d93b32d554dd013f607148291ba4 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 11 Dec 2025 11:55:40 +0100 Subject: [PATCH 246/506] Release 2.18.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1f1e7e224..41264de99 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2025-12-11 (2.18.0) + * Add `:allow_control_characters` parser options, to allow JSON strings containing unescaped ASCII control characters (e.g. newlines). ### 2025-12-04 (2.17.1) diff --git a/lib/json/version.rb b/lib/json/version.rb index 4ed61c43c..631beba83 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.17.1' + VERSION = '2.18.0' end From 5793694ee6632245f345e3b923af5ebfd7db68a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 11 Dec 2025 16:12:35 +0100 Subject: [PATCH 247/506] Revert "Skip test failing with JRuby in CI" This reverts commit b7e1734d9ca090e95b1923b05536f5fb6f437060. --- test/json/json_parser_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index d29f8077b..ec9391909 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -843,7 +843,6 @@ def test_parse_whitespace_after_newline def test_frozen parser_config = JSON::Parser::Config.new({}).freeze - omit "JRuby failure in CI" if RUBY_ENGINE == "jruby" assert_raise FrozenError do parser_config.send(:initialize, {}) end From 4808fee9affb801bc9bc63ea2d69d1cb28deda2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=85=E6=88=91=E5=B1=B1=E8=8F=9C=E3=80=85?= Date: Fri, 19 Dec 2025 01:30:37 +0900 Subject: [PATCH 248/506] Update `fpconv_dtoa` definition to use `dest[32]` --- ext/json/ext/vendor/fpconv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fpconv.c b/ext/json/ext/vendor/fpconv.c index 4888ab4b8..6c9bc2c10 100644 --- a/ext/json/ext/vendor/fpconv.c +++ b/ext/json/ext/vendor/fpconv.c @@ -449,7 +449,7 @@ static int filter_special(double fp, char* dest) * } * */ -static int fpconv_dtoa(double d, char dest[28]) +static int fpconv_dtoa(double d, char dest[32]) { char digits[18]; From a5c160f37285cab4406017990a6a02e82e283055 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 22 Dec 2025 09:16:09 +0100 Subject: [PATCH 249/506] Add missing documentation for `allow_control_characters` parsing option --- lib/json.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/json.rb b/lib/json.rb index 43d96afa9..f619d9325 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -173,6 +173,18 @@ # When enabled: # JSON.parse('[1,]', allow_trailing_comma: true) # => [1] # +# --- +# +# Option +allow_control_characters+ (boolean) specifies whether to allow +# unescaped ASCII control characters, such as newlines, in strings; +# defaults to +false+. +# +# With the default, +false+: +# JSON.parse(%{"Hello\nWorld"}) # invalid ASCII control character in string (JSON::ParserError) +# +# When enabled: +# JSON.parse(%{"Hello\nWorld"}, allow_control_characters: true) # => "Hello\nWorld" +# # ====== Output Options # # Option +freeze+ (boolean) specifies whether the returned objects will be frozen; From d21d9362fa36bd1b34c53e28b04ba6e91115a3be Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 30 Dec 2025 20:48:18 -0600 Subject: [PATCH 250/506] Keep track of the the number of additional backslashes to avoid an extra memchr searching the remaining characters when no more backslashes exist. --- ext/json/ext/parser/parser.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 8f9729ef2..2c95cc97e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -643,7 +643,7 @@ static inline VALUE json_string_fastpath(JSON_ParserState *state, JSON_ParserCon typedef struct _json_unescape_positions { long size; const char **positions; - bool has_more; + unsigned long additional_backslashes; } JSON_UnescapePositions; static inline const char *json_next_backslash(const char *pe, const char *stringEnd, JSON_UnescapePositions *positions) @@ -657,7 +657,8 @@ static inline const char *json_next_backslash(const char *pe, const char *string } } - if (positions->has_more) { + if (positions->additional_backslashes) { + positions->additional_backslashes--; return memchr(pe, '\\', stringEnd - pe); } @@ -992,7 +993,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi JSON_UnescapePositions positions = { .size = 0, .positions = backslashes, - .has_more = false, + .additional_backslashes = 0, }; do { @@ -1007,7 +1008,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi backslashes[positions.size] = state->cursor; positions.size++; } else { - positions.has_more = true; + positions.additional_backslashes++; } state->cursor++; break; From 976ba36629b2d20fa4de3da12ed405ab178deeba Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 30 Dec 2025 21:32:03 -0600 Subject: [PATCH 251/506] Simplify unescape_unicode Co-Authored-By: Jean Boussier --- ext/json/ext/parser/parser.c | 106 ++++++++++++++++------------------ test/json/json_parser_test.rb | 1 + 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2c95cc97e..60ed689fd 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -477,23 +477,24 @@ static const signed char digit_values[256] = { -1, -1, -1, -1, -1, -1, -1 }; -static uint32_t unescape_unicode(JSON_ParserState *state, const unsigned char *p) +static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe) { - signed char b; - uint32_t result = 0; - b = digit_values[p[0]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); - result = (result << 4) | (unsigned char)b; - b = digit_values[p[1]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); - result = (result << 4) | (unsigned char)b; - b = digit_values[p[2]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); - result = (result << 4) | (unsigned char)b; - b = digit_values[p[3]]; - if (b < 0) raise_parse_error_at("incomplete unicode character escape sequence at %s", state, (char *)p - 2); - result = (result << 4) | (unsigned char)b; - return result; + if (RB_UNLIKELY(sp > spe - 4)) { + raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); + } + + const unsigned char *p = (const unsigned char *)sp; + + const char b0 = digit_values[p[0]]; + const char b1 = digit_values[p[1]]; + const char b2 = digit_values[p[2]]; + const char b3 = digit_values[p[3]]; + + if (RB_UNLIKELY((b0 | b1 | b2 | b3) < 0)) { + raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); + } + + return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3; } #define GET_PARSER_CONFIG \ @@ -708,50 +709,43 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser case 'f': APPEND_CHAR('\f'); break; - case 'u': - if (pe > stringEnd - 5) { - raise_parse_error_at("incomplete unicode character escape sequence at %s", state, p); - } else { - uint32_t ch = unescape_unicode(state, (unsigned char *) ++pe); - pe += 3; - /* To handle values above U+FFFF, we take a sequence of - * \uXXXX escapes in the U+D800..U+DBFF then - * U+DC00..U+DFFF ranges, take the low 10 bits from each - * to make a 20-bit number, then add 0x10000 to get the - * final codepoint. - * - * See Unicode 15: 3.8 "Surrogates", 5.3 "Handling - * Surrogate Pairs in UTF-16", and 23.6 "Surrogates - * Area". - */ - if ((ch & 0xFC00) == 0xD800) { - pe++; - if (pe > stringEnd - 6) { - raise_parse_error_at("incomplete surrogate pair at %s", state, p); - } - if (pe[0] == '\\' && pe[1] == 'u') { - uint32_t sur = unescape_unicode(state, (unsigned char *) pe + 2); - - if ((sur & 0xFC00) != 0xDC00) { - raise_parse_error_at("invalid surrogate pair at %s", state, p); - } - - ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) - | (sur & 0x3FF)); - pe += 5; - } else { - raise_parse_error_at("incomplete surrogate pair at %s", state, p); - break; + case 'u': { + uint32_t ch = unescape_unicode(state, ++pe, stringEnd); + pe += 3; + /* To handle values above U+FFFF, we take a sequence of + * \uXXXX escapes in the U+D800..U+DBFF then + * U+DC00..U+DFFF ranges, take the low 10 bits from each + * to make a 20-bit number, then add 0x10000 to get the + * final codepoint. + * + * See Unicode 15: 3.8 "Surrogates", 5.3 "Handling + * Surrogate Pairs in UTF-16", and 23.6 "Surrogates + * Area". + */ + if ((ch & 0xFC00) == 0xD800) { + pe++; + if (RB_LIKELY((pe <= stringEnd - 6) && memcmp(pe, "\\u", 2) == 0)) { + uint32_t sur = unescape_unicode(state, pe + 2, stringEnd); + + if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) { + raise_parse_error_at("invalid surrogate pair at %s", state, p); } - } - char buf[4]; - int unescape_len = convert_UTF32_to_UTF8(buf, ch); - MEMCPY(buffer, buf, char, unescape_len); - buffer += unescape_len; - p = ++pe; + ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); + pe += 5; + } else { + raise_parse_error_at("incomplete surrogate pair at %s", state, p); + break; + } } + + char buf[4]; + int unescape_len = convert_UTF32_to_UTF8(buf, ch); + MEMCPY(buffer, buf, char, unescape_len); + buffer += unescape_len; + p = ++pe; break; + } default: if ((unsigned char)*pe < 0x20) { if (!config->allow_control_characters) { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index ec9391909..ac53ba9f0 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -350,6 +350,7 @@ def test_invalid_surogates assert_raise(JSON::ParserError) { parse('"\\uD800"') } assert_raise(JSON::ParserError) { parse('"\\uD800_________________"') } assert_raise(JSON::ParserError) { parse('"\\uD800\\u0041"') } + assert_raise(JSON::ParserError) { parse('"\\uD800\\u004') } end def test_parse_big_integers From 8ad744c53269330cc827bf4ae213991ae11712ce Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 31 Dec 2025 21:40:16 +0900 Subject: [PATCH 252/506] Fix non-portable code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain `char` may be `signed` or `unsigned` depending on the implementation. Also, bitwise ORing of `signed` values ​​is not guaranteed to be `signed`. To ensure portability, should logical-OR each comparison, but casting to `signed char` is usually sufficient. --- ext/json/ext/parser/parser.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 60ed689fd..0ac16918f 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -485,12 +485,12 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const const unsigned char *p = (const unsigned char *)sp; - const char b0 = digit_values[p[0]]; - const char b1 = digit_values[p[1]]; - const char b2 = digit_values[p[2]]; - const char b3 = digit_values[p[3]]; + const signed char b0 = digit_values[p[0]]; + const signed char b1 = digit_values[p[1]]; + const signed char b2 = digit_values[p[2]]; + const signed char b3 = digit_values[p[3]]; - if (RB_UNLIKELY((b0 | b1 | b2 | b3) < 0)) { + if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) { raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); } From a51317c94900ec5ea44ef0fd9327cca4d601bb5c Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 2 Jan 2026 20:47:59 -0600 Subject: [PATCH 253/506] Directly write to the output buffer when converting UTF32 to UTF8. --- ext/json/ext/parser/parser.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 0ac16918f..f1ea1b6ab 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -739,9 +739,7 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser } } - char buf[4]; - int unescape_len = convert_UTF32_to_UTF8(buf, ch); - MEMCPY(buffer, buf, char, unescape_len); + int unescape_len = convert_UTF32_to_UTF8(buffer, ch); buffer += unescape_len; p = ++pe; break; From 86a5cce405102eca15d639e2dae2f539fb3e7c26 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Mon, 12 Jan 2026 18:57:09 -0600 Subject: [PATCH 254/506] use a conditional to select SIMD implementation rather than pointer --- ext/json/ext/generator/generator.c | 53 ++++++++++++++++++------------ 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d202e97ea..fdf832f86 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -63,6 +63,8 @@ struct generate_json_data { long depth; }; +static SIMD_Implementation simd_impl; + static VALUE cState_from_state_s(VALUE self, VALUE opts); static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func, VALUE io); static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj); @@ -155,8 +157,6 @@ static const unsigned char escape_table_basic[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -static unsigned char (*search_escape_basic_impl)(search_state *); - static inline unsigned char search_escape_basic(search_state *search) { while (search->ptr < search->end) { @@ -212,11 +212,39 @@ ALWAYS_INLINE(static) void escape_UTF8_char_basic(search_state *search) * Everything else (should be UTF-8) is just passed through and * appended to the result. */ + + +#if defined(HAVE_SIMD_NEON) +static inline unsigned char search_escape_basic_neon(search_state *search); +#elif defined(HAVE_SIMD_SSE2) +static inline unsigned char search_escape_basic_sse2(search_state *search); +#endif + +static inline unsigned char search_escape_basic(search_state *search); + static inline void convert_UTF8_to_JSON(search_state *search) { - while (search_escape_basic_impl(search)) { +#ifdef HAVE_SIMD +#if defined(HAVE_SIMD_NEON) + while (search_escape_basic_neon(search)) { + escape_UTF8_char_basic(search); + } +#elif defined(HAVE_SIMD_SSE2) + if (simd_impl == SIMD_SSE2) { + while (search_escape_basic_sse2(search)) { + escape_UTF8_char_basic(search); + } + return; + } + while (search_escape_basic(search)) { + escape_UTF8_char_basic(search); + } +#endif +#else + while (search_escape_basic(search)) { escape_UTF8_char_basic(search); } +#endif /* HAVE_SIMD */ } static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) @@ -2181,22 +2209,5 @@ void Init_generator(void) rb_require("json/ext/generator/state"); - - switch (find_simd_implementation()) { -#ifdef HAVE_SIMD -#ifdef HAVE_SIMD_NEON - case SIMD_NEON: - search_escape_basic_impl = search_escape_basic_neon; - break; -#endif /* HAVE_SIMD_NEON */ -#ifdef HAVE_SIMD_SSE2 - case SIMD_SSE2: - search_escape_basic_impl = search_escape_basic_sse2; - break; -#endif /* HAVE_SIMD_SSE2 */ -#endif /* HAVE_SIMD */ - default: - search_escape_basic_impl = search_escape_basic; - break; - } + simd_impl = find_simd_implementation(); } From 73818b80c8fe4394153002a60ecd285e8688b4cd Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 13 Jan 2026 22:17:16 -0600 Subject: [PATCH 255/506] initialize search.chunk_end to silence a warning about it being potentially uninitialized --- ext/json/ext/generator/generator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index fdf832f86..4e6214210 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1119,6 +1119,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data search.matches_mask = 0; search.has_matches = false; search.chunk_base = NULL; + search.chunk_end = NULL; #endif /* HAVE_SIMD */ switch (rb_enc_str_coderange(obj)) { From 15eb40dbdfb299813bbbb72f70e12d1614b26200 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 14 Jan 2026 21:28:37 +0900 Subject: [PATCH 256/506] Remove trailing spaces [ci skip] --- CHANGES.md | 2 +- Rakefile | 2 +- ext/json/ext/generator/generator.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 41264de99..8b51c5856 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -66,7 +66,7 @@ * Fix `JSON.generate` `strict: true` mode to also restrict hash keys. * Fix `JSON::Coder` to also invoke block for hash keys that aren't strings nor symbols. * Fix `JSON.unsafe_load` usage with proc -* Fix the parser to more consistently reject invalid UTF-16 surogate pairs. +* Fix the parser to more consistently reject invalid UTF-16 surogate pairs. * Stop defining `String.json_create`, `String#to_json_raw`, `String#to_json_raw_object` when `json/add` isn't loaded. ### 2025-07-28 (2.13.2) diff --git a/Rakefile b/Rakefile index d09e81940..d05b45b79 100644 --- a/Rakefile +++ b/Rakefile @@ -159,7 +159,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' JAVA_VEC_SOURCES.each do |src| obj = src.sub(/\.java\Z/, '.class') file obj => src do - sh 'javac', '--add-modules', 'jdk.incubator.vector', '-classpath', classpath, '--release', '16', src do |success, status| + sh 'javac', '--add-modules', 'jdk.incubator.vector', '-classpath', classpath, '--release', '16', src do |success, status| if success puts "*** 'jdk.incubator.vector' support enabled ***" else diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 4e6214210..27b2e353f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -213,12 +213,12 @@ ALWAYS_INLINE(static) void escape_UTF8_char_basic(search_state *search) * appended to the result. */ - + #if defined(HAVE_SIMD_NEON) static inline unsigned char search_escape_basic_neon(search_state *search); #elif defined(HAVE_SIMD_SSE2) static inline unsigned char search_escape_basic_sse2(search_state *search); -#endif +#endif static inline unsigned char search_escape_basic(search_state *search); From 643ee11fed6898b7e6f75f266daa6e4b2a07bb07 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 15 Jan 2026 19:12:41 -0600 Subject: [PATCH 257/506] Use __builtin_memcpy, if available, to copy overlapping byte ranges in copy_remaining_bytes to avoid a branch to MEMCPY. Additionally use a space as padding byte instead of an 'X' so it can be represented diretly on AArch64 with a single instruction. --- ext/json/ext/generator/generator.c | 36 +++++++++++++++++++++++++++--- ext/json/ext/simd/simd.h | 2 +- test/json/json_generator_test.rb | 16 +++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 27b2e353f..f17a2a729 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -297,13 +297,43 @@ ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned char *s = (buf->ptr + buf->len); // Pad the buffer with dummy characters that won't need escaping. - // This seem wateful at first sight, but memset of vector length is very fast. - memset(s, 'X', vec_len); + // This seem wasteful at first sight, but memset of vector length is very fast. + // This is a space as it can be directly represented as an immediate on AArch64. + memset(s, ' ', vec_len); // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage. - MEMCPY(s, search->ptr, char, len); +#if defined(__has_builtin) && __has_builtin(__builtin_memcpy) + +#ifdef RBIMPL_ASSERT_OR_ASSUME + RBIMPL_ASSERT_OR_ASSUME(len < 16); +#endif + if (vec_len == 16 && len >= 4) { + // If __builtin_memcpy is available, use it to copy between SIMD_MINIMUM_THRESHOLD and vec_len-1 bytes. + // These copies overlap. The first copy will copy the first 8 (or 4) bytes. The second copy will copy + // the last 8 (or 4) bytes but overlap with the first copy. The overlapping bytes will be in the correct + // position in both copies. + + // Please do not attempt to replace __builtin_memcpy with memcpy without profiling and/or looking at the + // generated assembly. On clang-specifically (tested on Apple clang version 17.0.0 (clang-1700.0.13.3)), + // when using memcpy, the compiler will notice the only difference is a 4 or 8 and generate a conditional + // select instruction instead of direct loads and stores with a branch. This ends up slower than the branch + // plus two loads and stores generated when using __builtin_memcpy. + if (len >= 8) { + __builtin_memcpy(s, search->ptr, 8); + __builtin_memcpy(s + len - 8, search->ptr + len - 8, 8); + } else { + __builtin_memcpy(s, search->ptr, 4); + __builtin_memcpy(s + len - 4, search->ptr + len - 4, 4); + } + } else { + MEMCPY(s, search->ptr, char, len); + } +#else + MEMCPY(s, search->ptr, char, len); +#endif + return s; } diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index f8e5ee188..84f6135ad 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -58,7 +58,7 @@ static inline int trailing_zeros(int input) #ifdef JSON_ENABLE_SIMD -#define SIMD_MINIMUM_THRESHOLD 6 +#define SIMD_MINIMUM_THRESHOLD 4 #if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) #include diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 9f8b35de0..d7c4173e8 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -650,6 +650,22 @@ def test_backslash json = '"\\nabc"' assert_equal json, generate(data) # + data = "\n" + json = '"\\n"' + assert_equal json, generate(data) + # + (0..16).each do |i| + data = ('a' * i) + "\n" + json = '"' + ('a' * i) + '\\n"' + assert_equal json, generate(data) + end + # + (0..16).each do |i| + data = "\n" + ('a' * i) + json = '"' + '\\n' + ('a' * i) + '"' + assert_equal json, generate(data) + end + # data = ["'"] json = '["\\\'"]' assert_equal '["\'"]', generate(data) From 1b276c8623da0ff9bec62396d9a7172365bc1784 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 16 Jan 2026 18:43:08 +0100 Subject: [PATCH 258/506] Extract json_fast_memcpy16 for readability --- ext/json/ext/generator/generator.c | 34 ++++++------------------------ ext/json/ext/json.h | 4 ++++ ext/json/ext/simd/simd.h | 27 ++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index f17a2a729..dbba99c45 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -288,6 +288,8 @@ static inline void escape_UTF8_char(search_state *search, unsigned char ch_len) ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned long vec_len, unsigned long len) { + RBIMPL_ASSERT_OR_ASSUME(len < vec_len); + // Flush the buffer so everything up until the last 'len' characters are unflushed. search_flush(search); @@ -303,37 +305,13 @@ ALWAYS_INLINE(static) char *copy_remaining_bytes(search_state *search, unsigned // Optimistically copy the remaining 'len' characters to the output FBuffer. If there are no characters // to escape, then everything ends up in the correct spot. Otherwise it was convenient temporary storage. -#if defined(__has_builtin) && __has_builtin(__builtin_memcpy) - -#ifdef RBIMPL_ASSERT_OR_ASSUME - RBIMPL_ASSERT_OR_ASSUME(len < 16); -#endif - - if (vec_len == 16 && len >= 4) { - // If __builtin_memcpy is available, use it to copy between SIMD_MINIMUM_THRESHOLD and vec_len-1 bytes. - // These copies overlap. The first copy will copy the first 8 (or 4) bytes. The second copy will copy - // the last 8 (or 4) bytes but overlap with the first copy. The overlapping bytes will be in the correct - // position in both copies. - - // Please do not attempt to replace __builtin_memcpy with memcpy without profiling and/or looking at the - // generated assembly. On clang-specifically (tested on Apple clang version 17.0.0 (clang-1700.0.13.3)), - // when using memcpy, the compiler will notice the only difference is a 4 or 8 and generate a conditional - // select instruction instead of direct loads and stores with a branch. This ends up slower than the branch - // plus two loads and stores generated when using __builtin_memcpy. - if (len >= 8) { - __builtin_memcpy(s, search->ptr, 8); - __builtin_memcpy(s + len - 8, search->ptr + len - 8, 8); - } else { - __builtin_memcpy(s, search->ptr, 4); - __builtin_memcpy(s + len - 4, search->ptr + len - 4, 4); - } + if (vec_len == 16) { + RBIMPL_ASSERT_OR_ASSUME(len >= SIMD_MINIMUM_THRESHOLD); + json_fast_memcpy16(s, search->ptr, len); } else { MEMCPY(s, search->ptr, char, len); } -#else - MEMCPY(s, search->ptr, char, len); -#endif - + return s; } diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 28efa04c2..9379d7ae7 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -5,6 +5,10 @@ #include "ruby/encoding.h" #include +#ifndef RBIMPL_ASSERT_OR_ASSUME +# define RBIMPL_ASSERT_OR_ASSUME(x) +#endif + #if defined(RUBY_DEBUG) && RUBY_DEBUG # define JSON_ASSERT RUBY_ASSERT #else diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 84f6135ad..3bb86acde 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -60,6 +60,33 @@ static inline int trailing_zeros(int input) #define SIMD_MINIMUM_THRESHOLD 4 +ALWAYS_INLINE(static) void json_fast_memcpy16(char *dst, const char *src, size_t len) +{ + RBIMPL_ASSERT_OR_ASSUME(len < 16); + RBIMPL_ASSERT_OR_ASSUME(len >= SIMD_MINIMUM_THRESHOLD); // 4 +#if defined(__has_builtin) && __has_builtin(__builtin_memcpy) + // If __builtin_memcpy is available, use it to copy between SIMD_MINIMUM_THRESHOLD (4) and vec_len-1 (15) bytes. + // These copies overlap. The first copy will copy the first 8 (or 4) bytes. The second copy will copy + // the last 8 (or 4) bytes but overlap with the first copy. The overlapping bytes will be in the correct + // position in both copies. + + // Please do not attempt to replace __builtin_memcpy with memcpy without profiling and/or looking at the + // generated assembly. On clang-specifically (tested on Apple clang version 17.0.0 (clang-1700.0.13.3)), + // when using memcpy, the compiler will notice the only difference is a 4 or 8 and generate a conditional + // select instruction instead of direct loads and stores with a branch. This ends up slower than the branch + // plus two loads and stores generated when using __builtin_memcpy. + if (len >= 8) { + __builtin_memcpy(dst, src, 8); + __builtin_memcpy(dst + len - 8, src + len - 8, 8); + } else { + __builtin_memcpy(dst, src, 4); + __builtin_memcpy(dst + len - 4, src + len - 4, 4); + } +#else + MEMCPY(dst, src, char, len); +#endif +} + #if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) #include From 062fcdd22818d226f0e4b49647eb2aabbfaf002b Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Thu, 22 Jan 2026 22:08:15 +0000 Subject: [PATCH 259/506] Improve class JSON intro --- lib/json.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/json.rb b/lib/json.rb index f619d9325..2f6db4422 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -6,6 +6,15 @@ # # \JSON is a lightweight data-interchange format. # +# \JSON is easy for us humans to read and write, +# and equally simple for machines to read (parse) and write (generate). +# +# \JSON is language-independent, making it an ideal interchange format +# for applications in differing programming languages +# and on differing operating systems. +# +# == \JSON Values +# # A \JSON value is one of the following: # - Double-quoted text: "foo". # - Number: +1+, +1.0+, +2.0e2+. From 79b6e168ba02f6ac7a0dac378968b1c918af91db Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 2 Feb 2026 22:06:44 +0100 Subject: [PATCH 260/506] Ensure `Generator::State` is kept on the stack Fix: https://github.com/ruby/json/issues/929 When calling `cState_partial_generate` from `mHash_to_json` or other `to_json` funcs, `VState` becomes unreachable very quickly, hence the compiler may optimize it out of the stack, and make it invisible to the GC stack scanning. This is particularly liekly given how aggressively we inline. Repro: ```ruby require 'json' test_data = { "flag" => true, "data" => 10000.times.map { [1.0] }, :flag => false, } 10.times do test_data.to_json end ``` But in practice the cause was just that the issued warning calls Hash#inspect on a big hash, which triggers GC. So it can be triggered even more reliably with: ```ruby require 'json' module JSON module Common def self.on_mixed_keys_hash(...) GC.start end end end test_data = { "flag" => true, "data" => 10000.times.map { [1.0] }, :flag => false, } test_data.to_json ``` --- CHANGES.md | 3 +++ ext/json/ext/generator/generator.c | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8b51c5856..0ff0d5177 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +* Fix a potential crash in very specific circumstance if GC triggers during a call to `to_json` + without first invoking a user defined `#to_json` method. + ### 2025-12-11 (2.18.0) * Add `:allow_control_characters` parser options, to allow JSON strings containing unescaped ASCII control characters (e.g. newlines). diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index dbba99c45..ad9259864 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1540,7 +1540,9 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, .obj = obj, .func = func }; - return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); + VALUE result = rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); + RB_GC_GUARD(self); + return result; } /* call-seq: From fff25c9f4b9c590a59ed4a2ec3c545121db844d3 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 2 Feb 2026 19:44:02 +0100 Subject: [PATCH 261/506] fbuffer_append_str: assume string --- ext/json/ext/fbuffer/fbuffer.h | 14 ++++++++------ ext/json/ext/generator/generator.c | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 752d153b3..8ce029a8c 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -161,23 +161,25 @@ static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) static void fbuffer_append_str(FBuffer *fb, VALUE str) { - const char *newstr = StringValuePtr(str); - unsigned long len = RSTRING_LEN(str); + const char *ptr; + unsigned long len; + RSTRING_GETMEM(str, ptr, len); - fbuffer_append(fb, newstr, len); + fbuffer_append(fb, ptr, len); } static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) { - const char *newstr = StringValuePtr(str); - unsigned long len = RSTRING_LEN(str); + const char *ptr; + unsigned long len; + RSTRING_GETMEM(str, ptr, len); fbuffer_inc_capa(fb, repeat * len); while (repeat) { #if JSON_DEBUG fb->requested = len; #endif - fbuffer_append_reserved(fb, newstr, len); + fbuffer_append_reserved(fb, ptr, len); repeat--; } } diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index ad9259864..3290de401 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1374,7 +1374,7 @@ static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *dat static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { VALUE tmp = rb_funcall(obj, i_to_s, 0); - fbuffer_append_str(buffer, tmp); + fbuffer_append_str(buffer, StringValue(tmp)); } #ifdef RUBY_INTEGER_UNIFICATION From 6ec6e7baff137e67831a23090061e84bf028dbfc Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 3 Feb 2026 08:35:11 +0100 Subject: [PATCH 262/506] Release 2.18.1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0ff0d5177..14d9944c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-02-03 (2.18.1) + * Fix a potential crash in very specific circumstance if GC triggers during a call to `to_json` without first invoking a user defined `#to_json` method. diff --git a/lib/json/version.rb b/lib/json/version.rb index 631beba83..95b885710 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.18.0' + VERSION = '2.18.1' end From b11ce01ca6bb47c0f1aab1e72f293af70978711d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 3 Feb 2026 08:39:10 +0100 Subject: [PATCH 263/506] Remove codepaths under !RUBY_INTEGER_UNIFICATION This was for pre Ruby 2.4 support which we dropped a while ago. --- ext/json/ext/generator/generator.c | 38 ------------------------------ 1 file changed, 38 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 3290de401..186a45714 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -74,9 +74,7 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj); -#ifdef RUBY_INTEGER_UNIFICATION static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj); -#endif static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj); @@ -815,7 +813,6 @@ static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) return cState_partial_generate(Vstate, self, generate_json_array, Qfalse); } -#ifdef RUBY_INTEGER_UNIFICATION /* * call-seq: to_json(*) * @@ -828,32 +825,6 @@ static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self) return cState_partial_generate(Vstate, self, generate_json_integer, Qfalse); } -#else -/* - * call-seq: to_json(*) - * - * Returns a JSON string representation for this Integer number. - */ -static VALUE mFixnum_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_fixnum, Qfalse); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string representation for this Integer number. - */ -static VALUE mBignum_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_bignum, Qfalse); -} -#endif - /* * call-seq: to_json(*) * @@ -1377,7 +1348,6 @@ static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *dat fbuffer_append_str(buffer, StringValue(tmp)); } -#ifdef RUBY_INTEGER_UNIFICATION static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { if (FIXNUM_P(obj)) @@ -1385,7 +1355,6 @@ static void generate_json_integer(FBuffer *buffer, struct generate_json_data *da else generate_json_bignum(buffer, data, obj); } -#endif static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { @@ -2165,16 +2134,9 @@ void Init_generator(void) VALUE mArray = rb_define_module_under(mGeneratorMethods, "Array"); rb_define_method(mArray, "to_json", mArray_to_json, -1); -#ifdef RUBY_INTEGER_UNIFICATION VALUE mInteger = rb_define_module_under(mGeneratorMethods, "Integer"); rb_define_method(mInteger, "to_json", mInteger_to_json, -1); -#else - VALUE mFixnum = rb_define_module_under(mGeneratorMethods, "Fixnum"); - rb_define_method(mFixnum, "to_json", mFixnum_to_json, -1); - VALUE mBignum = rb_define_module_under(mGeneratorMethods, "Bignum"); - rb_define_method(mBignum, "to_json", mBignum_to_json, -1); -#endif VALUE mFloat = rb_define_module_under(mGeneratorMethods, "Float"); rb_define_method(mFloat, "to_json", mFloat_to_json, -1); From 7ddf3499d0c7b8112fdb814965845105c05a1e2e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Feb 2026 09:33:07 +0100 Subject: [PATCH 264/506] Cleanup function delecarations --- ext/json/ext/json.h | 4 ++++ ext/json/ext/parser/parser.c | 24 +++++++----------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 9379d7ae7..087a2eae6 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -55,8 +55,12 @@ typedef unsigned char _Bool; #endif #ifndef NORETURN +#if defined(__has_attribute) && __has_attribute(noreturn) +#define NORETURN(x) __attribute__((noreturn)) x +#else #define NORETURN(x) x #endif +#endif #ifndef NOINLINE #if defined(__has_attribute) && __has_attribute(noinline) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index f1ea1b6ab..93e3b021f 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -400,10 +400,7 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state) #define PARSE_ERROR_FRAGMENT_LEN 32 -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_parse_error(const char *format, JSON_ParserState *state) +NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state) { unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; long line, column; @@ -449,10 +446,7 @@ static void raise_parse_error(const char *format, JSON_ParserState *state) rb_exc_raise(exc); } -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) +NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) { state->cursor = at; raise_parse_error(format, state); @@ -777,7 +771,7 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser #define MAX_FAST_INTEGER_SIZE 18 -static VALUE json_decode_large_integer(const char *start, long len) +NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len) { VALUE buffer_v; char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1); @@ -788,8 +782,7 @@ static VALUE json_decode_large_integer(const char *start, long len) return number; } -static inline VALUE -json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end) +static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end) { if (RB_LIKELY(mantissa_digits < MAX_FAST_INTEGER_SIZE)) { if (negative) { @@ -801,7 +794,7 @@ json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const return json_decode_large_integer(start, end - start); } -static VALUE json_decode_large_float(const char *start, long len) +NOINLINE(static) VALUE json_decode_large_float(const char *start, long len) { if (RB_LIKELY(len < 64)) { char buffer[64]; @@ -868,7 +861,7 @@ static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs) return Qfalse; } -static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key) +NOINLINE(static) void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key) { VALUE message = rb_sprintf( "detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", @@ -879,10 +872,7 @@ static void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_ RB_GC_GUARD(message); } -#ifdef RBIMPL_ATTR_NORETURN -RBIMPL_ATTR_NORETURN() -#endif -static void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key) +NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key) { VALUE message = rb_sprintf( "duplicate key %"PRIsVALUE, From 4dcbcb84a2bed50232ec6f31033d0f162b27a46c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Feb 2026 10:03:12 +0100 Subject: [PATCH 265/506] Stop using RB_ALLOCV While it's a very nice API, if the buffer is too big for the stack (> 1024B) the buffer object will be mark like a stack region which is slow and overkill for a char buffer. --- ext/json/ext/parser/parser.c | 54 +++++++++++++++++++++-------------- test/json/json_parser_test.rb | 4 +++ 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 93e3b021f..e4b619b42 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -770,16 +770,36 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser } #define MAX_FAST_INTEGER_SIZE 18 +#define MAX_NUMBER_STACK_BUFFER 128 + +typedef VALUE (*json_number_decode_func_t)(const char *ptr); + +static inline VALUE json_decode_large_number(const char *start, long len, json_number_decode_func_t func) +{ + if (RB_LIKELY(len < MAX_NUMBER_STACK_BUFFER)) { + char buffer[MAX_NUMBER_STACK_BUFFER]; + MEMCPY(buffer, start, char, len); + buffer[len] = '\0'; + return func(buffer); + } else { + VALUE buffer_v = rb_str_tmp_new(len); + char *buffer = RSTRING_PTR(buffer_v); + MEMCPY(buffer, start, char, len); + buffer[len] = '\0'; + VALUE number = func(buffer); + RB_GC_GUARD(buffer_v); + return number; + } +} + +static VALUE json_decode_inum(const char *buffer) +{ + return rb_cstr2inum(buffer, 10); +} NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len) { - VALUE buffer_v; - char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1); - MEMCPY(buffer, start, char, len); - buffer[len] = '\0'; - VALUE number = rb_cstr2inum(buffer, 10); - RB_ALLOCV_END(buffer_v); - return number; + return json_decode_large_number(start, len, json_decode_inum); } static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end) @@ -794,22 +814,14 @@ static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, return json_decode_large_integer(start, end - start); } -NOINLINE(static) VALUE json_decode_large_float(const char *start, long len) +static VALUE json_decode_dnum(const char *buffer) { - if (RB_LIKELY(len < 64)) { - char buffer[64]; - MEMCPY(buffer, start, char, len); - buffer[len] = '\0'; - return DBL2NUM(rb_cstr_to_dbl(buffer, 1)); - } + return DBL2NUM(rb_cstr_to_dbl(buffer, 1)); +} - VALUE buffer_v; - char *buffer = RB_ALLOCV_N(char, buffer_v, len + 1); - MEMCPY(buffer, start, char, len); - buffer[len] = '\0'; - VALUE number = DBL2NUM(rb_cstr_to_dbl(buffer, 1)); - RB_ALLOCV_END(buffer_v); - return number; +NOINLINE(static) VALUE json_decode_large_float(const char *start, long len) +{ + return json_decode_large_number(start, len, json_decode_dnum); } /* Ruby JSON optimized float decoder using vendored Ryu algorithm diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index ac53ba9f0..1b875422a 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -135,6 +135,10 @@ def test_parse_bignum bignum = Integer('1234567890' * 10) assert_equal(bignum, JSON.parse(bignum.to_s)) assert_equal(bignum.to_f, JSON.parse(bignum.to_s + ".0")) + + bignum = Integer('1234567890' * 50) + assert_equal(bignum, JSON.parse(bignum.to_s)) + assert_equal(bignum.to_f, JSON.parse(bignum.to_s + ".0")) end def test_parse_bigdecimals From 672e26e3ee24d07a81d53fc49e9da3f876f034fa Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Sat, 7 Feb 2026 18:46:36 +0000 Subject: [PATCH 266/506] [DOC] Fix links --- lib/json/common.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 877b96814..1a33c4103 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1036,7 +1036,8 @@ class Coder # JSON.new(options = nil, &block) # # Argument +options+, if given, contains a \Hash of options for both parsing and generating. - # See {Parsing Options}[#module-JSON-label-Parsing+Options], and {Generating Options}[#module-JSON-label-Generating+Options]. + # See {Parsing Options}[rdoc-ref:JSON@Parsing+Options], + # and {Generating Options}[rdoc-ref:JSON@Generating+Options]. # # For generation, the strict: true option is always set. When a Ruby object with no native \JSON counterpart is # encountered, the block provided to the initialize method is invoked, and must return a Ruby object that has a native From 0bbcf8037357d61cb91d5f810cdb8b27688e7452 Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Sat, 7 Feb 2026 19:38:19 +0000 Subject: [PATCH 267/506] [DOC] Another link fix --- lib/json/ext/generator/state.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index ce5c185ca..e4f425af6 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -9,7 +9,7 @@ class State # Instantiates a new State object, configured by _opts_. # # Argument +opts+, if given, contains a \Hash of options for the generation. - # See {Generating Options}[#module-JSON-label-Generating+Options]. + # See {Generating Options}[rdoc-ref:JSON@Generating+Options]. def initialize(opts = nil) if opts && !opts.empty? configure(opts) From 9602edb255fd1a24d71ce322964c851f207c2ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 12 Feb 2026 13:39:07 +0100 Subject: [PATCH 268/506] Remove unused method in JSONGeneratorTest --- test/json/json_generator_test.rb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index d7c4173e8..d5f6ac64a 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -39,14 +39,6 @@ def setup JSON end - def silence - v = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = v - end - def test_generate json = generate(@hash) assert_equal(parse(@json2), parse(json)) From e7245b714a7ad33de82f864b55fa8c65940aa89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Thu, 12 Feb 2026 15:12:48 +0100 Subject: [PATCH 269/506] Remove bignum warnings test/json/json_parser_test.rb:141: warning: Integer out of Float range lib/json/common.rb:353: warning: Float 12345678901234567890... out of range --- test/json/json_parser_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 1b875422a..54a6bbbd6 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -138,7 +138,8 @@ def test_parse_bignum bignum = Integer('1234567890' * 50) assert_equal(bignum, JSON.parse(bignum.to_s)) - assert_equal(bignum.to_f, JSON.parse(bignum.to_s + ".0")) + bignum_float = EnvUtil.suppress_warning { bignum.to_f } + assert_equal(bignum_float, EnvUtil.suppress_warning { JSON.parse(bignum.to_s + ".0") }) end def test_parse_bigdecimals From 05cec0cbee69719275607de8488f3ecc1a572f91 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 19 Feb 2026 13:20:18 +0100 Subject: [PATCH 270/506] Add `allow_invalid_escape` parsing option Ref: https://github.com/ruby/json/issues/939 --- CHANGES.md | 2 + ext/json/ext/parser/parser.c | 10 +- java/src/json/ext/ParserConfig.java | 150 ++++++++++++++------------- java/src/json/ext/ParserConfig.rl | 4 +- java/src/json/ext/StringDecoder.java | 13 ++- lib/json.rb | 12 +++ test/json/json_parser_test.rb | 8 ++ 7 files changed, 119 insertions(+), 80 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 14d9944c8..ceb17198c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Add `allow_invalid_escape` parsing option to ignore backslashes that aren't followed by one of the valid escape characters. + ### 2026-02-03 (2.18.1) * Fix a potential crash in very specific circumstance if GC triggers during a call to `to_json` diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e4b619b42..42ca10894 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -7,8 +7,9 @@ static VALUE CNaN, CInfinity, CMinusInfinity; static ID i_new, i_try_convert, i_uminus, i_encode; -static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters, sym_symbolize_names, sym_freeze, - sym_decimal_class, sym_on_load, sym_allow_duplicate_key; +static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters, + sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load, + sym_allow_duplicate_key; static int binary_encindex; static int utf8_encindex; @@ -336,6 +337,7 @@ typedef struct JSON_ParserStruct { bool allow_nan; bool allow_trailing_comma; bool allow_control_characters; + bool allow_invalid_escape; bool symbolize_names; bool freeze; } JSON_ParserConfig; @@ -746,6 +748,8 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser } raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); } + } else if (config->allow_invalid_escape) { + APPEND_CHAR(*pe); } else { raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); } @@ -1435,6 +1439,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); } + else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } @@ -1653,6 +1658,7 @@ void Init_parser(void) sym_allow_nan = ID2SYM(rb_intern("allow_nan")); sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma")); sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters")); + sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape")); sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); sym_freeze = ID2SYM(rb_intern("freeze")); sym_on_load = ID2SYM(rb_intern("on_load")); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 23231646f..dfc1a6c2c 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -55,6 +55,7 @@ public class ParserConfig extends RubyObject { private boolean allowNaN; private boolean allowTrailingComma; private boolean allowControlCharacters; + private boolean allowInvalidEscape; private boolean allowDuplicateKey; private boolean deprecateDuplicateKey; private boolean symbolizeNames; @@ -180,6 +181,7 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); this.allowControlCharacters = opts.getBool("allow_control_characters", false); + this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); if (opts.hasKey("allow_duplicate_key")) { @@ -290,7 +292,7 @@ private ParserSession(ParserConfig config, RubyString source, ThreadContext cont this.byteList = source.getByteList(); this.data = byteList.unsafeBytes(); this.view = new ByteList(data, false); - this.decoder = new StringDecoder(config.allowControlCharacters); + this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); } private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { @@ -305,11 +307,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 331 "ParserConfig.rl" +// line 333 "ParserConfig.rl" -// line 313 "ParserConfig.java" +// line 315 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -423,7 +425,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 437 "ParserConfig.rl" +// line 439 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -431,14 +433,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 435 "ParserConfig.java" +// line 437 "ParserConfig.java" { cs = JSON_value_start; } -// line 444 "ParserConfig.rl" +// line 446 "ParserConfig.rl" -// line 442 "ParserConfig.java" +// line 444 "ParserConfig.java" { int _klen; int _trans = 0; @@ -464,13 +466,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 422 "ParserConfig.rl" +// line 424 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 474 "ParserConfig.java" +// line 476 "ParserConfig.java" } } @@ -533,25 +535,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 339 "ParserConfig.rl" +// line 341 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 342 "ParserConfig.rl" +// line 344 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 345 "ParserConfig.rl" +// line 347 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 348 "ParserConfig.rl" +// line 350 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -561,7 +563,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 355 "ParserConfig.rl" +// line 357 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -571,7 +573,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 362 "ParserConfig.rl" +// line 364 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -600,7 +602,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 388 "ParserConfig.rl" +// line 390 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -613,7 +615,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 398 "ParserConfig.rl" +// line 400 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -628,7 +630,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 410 "ParserConfig.rl" +// line 412 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -642,7 +644,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 646 "ParserConfig.java" +// line 648 "ParserConfig.java" } } } @@ -662,7 +664,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 445 "ParserConfig.rl" +// line 447 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -675,7 +677,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 679 "ParserConfig.java" +// line 681 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -774,7 +776,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 467 "ParserConfig.rl" +// line 469 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -791,15 +793,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 795 "ParserConfig.java" +// line 797 "ParserConfig.java" { cs = JSON_integer_start; } -// line 483 "ParserConfig.rl" +// line 485 "ParserConfig.rl" int memo = p; -// line 803 "ParserConfig.java" +// line 805 "ParserConfig.java" { int _klen; int _trans = 0; @@ -880,13 +882,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 461 "ParserConfig.rl" +// line 463 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 890 "ParserConfig.java" +// line 892 "ParserConfig.java" } } } @@ -906,7 +908,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 485 "ParserConfig.rl" +// line 487 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -926,7 +928,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 930 "ParserConfig.java" +// line 932 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1028,7 +1030,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 518 "ParserConfig.rl" +// line 520 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1047,15 +1049,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1051 "ParserConfig.java" +// line 1053 "ParserConfig.java" { cs = JSON_float_start; } -// line 536 "ParserConfig.rl" +// line 538 "ParserConfig.rl" int memo = p; -// line 1059 "ParserConfig.java" +// line 1061 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1136,13 +1138,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 509 "ParserConfig.rl" +// line 511 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1146 "ParserConfig.java" +// line 1148 "ParserConfig.java" } } } @@ -1162,7 +1164,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 538 "ParserConfig.rl" +// line 540 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1172,7 +1174,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1176 "ParserConfig.java" +// line 1178 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1274,7 +1276,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 577 "ParserConfig.rl" +// line 579 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1282,15 +1284,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1286 "ParserConfig.java" +// line 1288 "ParserConfig.java" { cs = JSON_string_start; } -// line 584 "ParserConfig.rl" +// line 586 "ParserConfig.rl" int memo = p; -// line 1294 "ParserConfig.java" +// line 1296 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1371,7 +1373,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 552 "ParserConfig.rl" +// line 554 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1386,13 +1388,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 565 "ParserConfig.rl" +// line 567 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1396 "ParserConfig.java" +// line 1398 "ParserConfig.java" } } } @@ -1412,7 +1414,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 586 "ParserConfig.rl" +// line 588 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1433,7 +1435,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1437 "ParserConfig.java" +// line 1439 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1600,7 +1602,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 640 "ParserConfig.rl" +// line 642 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1614,14 +1616,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); -// line 1618 "ParserConfig.java" +// line 1620 "ParserConfig.java" { cs = JSON_array_start; } -// line 653 "ParserConfig.rl" +// line 655 "ParserConfig.rl" -// line 1625 "ParserConfig.java" +// line 1627 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1664,7 +1666,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 611 "ParserConfig.rl" +// line 613 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1734,7 +1736,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 613 "ParserConfig.rl" +// line 615 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1747,13 +1749,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 624 "ParserConfig.rl" +// line 626 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1757 "ParserConfig.java" +// line 1759 "ParserConfig.java" } } } @@ -1773,7 +1775,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 654 "ParserConfig.rl" +// line 656 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1783,7 +1785,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1787 "ParserConfig.java" +// line 1789 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1960,7 +1962,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 724 "ParserConfig.rl" +// line 726 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -1977,14 +1979,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyHash.newHash(context.runtime); -// line 1981 "ParserConfig.java" +// line 1983 "ParserConfig.java" { cs = JSON_object_start; } -// line 740 "ParserConfig.rl" +// line 742 "ParserConfig.rl" -// line 1988 "ParserConfig.java" +// line 1990 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2027,7 +2029,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 668 "ParserConfig.rl" +// line 670 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2097,7 +2099,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 670 "ParserConfig.rl" +// line 672 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2110,7 +2112,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 681 "ParserConfig.rl" +// line 683 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2141,13 +2143,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 710 "ParserConfig.rl" +// line 712 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2151 "ParserConfig.java" +// line 2153 "ParserConfig.java" } } } @@ -2167,7 +2169,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 741 "ParserConfig.rl" +// line 743 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2178,7 +2180,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2182 "ParserConfig.java" +// line 2184 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2281,7 +2283,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 770 "ParserConfig.rl" +// line 772 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2291,16 +2293,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2295 "ParserConfig.java" +// line 2297 "ParserConfig.java" { cs = JSON_start; } -// line 779 "ParserConfig.rl" +// line 781 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2304 "ParserConfig.java" +// line 2306 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2381,7 +2383,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 756 "ParserConfig.rl" +// line 758 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2393,7 +2395,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2397 "ParserConfig.java" +// line 2399 "ParserConfig.java" } } } @@ -2413,7 +2415,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 782 "ParserConfig.rl" +// line 784 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index a1c9ea7f3..53678ec97 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -53,6 +53,7 @@ public class ParserConfig extends RubyObject { private boolean allowNaN; private boolean allowTrailingComma; private boolean allowControlCharacters; + private boolean allowInvalidEscape; private boolean allowDuplicateKey; private boolean deprecateDuplicateKey; private boolean symbolizeNames; @@ -178,6 +179,7 @@ public class ParserConfig extends RubyObject { this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); this.allowControlCharacters = opts.getBool("allow_control_characters", false); + this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); if (opts.hasKey("allow_duplicate_key")) { @@ -288,7 +290,7 @@ public class ParserConfig extends RubyObject { this.byteList = source.getByteList(); this.data = byteList.unsafeBytes(); this.view = new ByteList(data, false); - this.decoder = new StringDecoder(config.allowControlCharacters); + this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); } private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index a588d94d7..091b09f56 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -23,14 +23,16 @@ final class StringDecoder extends ByteListTranscoder { */ private int surrogatePairStart = -1; private boolean allowControlCharacters = false; + private boolean allowInvalidEscape = false; private ByteList out; // Array used for writing multibyte characters into the buffer at once private final byte[] aux = new byte[4]; - public StringDecoder(boolean allowControlCharacters) { + public StringDecoder(boolean allowControlCharacters, boolean allowInvalidEscape) { this.allowControlCharacters = allowControlCharacters; + this.allowInvalidEscape = allowInvalidEscape; } ByteList decode(ThreadContext context, ByteList src, int start, int end) { @@ -67,7 +69,8 @@ private void handleChar(ThreadContext context, int c) throws IOException { private void handleEscapeSequence(ThreadContext context) throws IOException { ensureMin(context, 1); - switch (readUtf8Char(context)) { + int character = readUtf8Char(context); + switch (character) { case 'b': append('\b'); break; @@ -105,7 +108,11 @@ private void handleEscapeSequence(ThreadContext context) throws IOException { } break; default: - throw invalidEscape(context); + if (allowInvalidEscape) { + append(character); + } else { + throw invalidEscape(context); + } } } diff --git a/lib/json.rb b/lib/json.rb index 2f6db4422..39e414beb 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -194,6 +194,18 @@ # When enabled: # JSON.parse(%{"Hello\nWorld"}, allow_control_characters: true) # => "Hello\nWorld" # +# --- +# +# Option +allow_invalid_escape+ (boolean) specifies whether to ignore backslahes that are followed +# by an invalid escape character in strings; +# defaults to +false+. +# +# With the default, +false+: +# JSON.parse(%{"Hell\\o"}) # invalid escape character in string (JSON::ParserError) +# +# When enabled: +# JSON.parse(%{"Hell\\o"}, allow_invalid_escape: true) # => "Hello" +# # ====== Output Options # # Option +freeze+ (boolean) specifies whether the returned objects will be frozen; diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 54a6bbbd6..653abf46c 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -183,6 +183,14 @@ def test_parse_allowed_control_chars_in_string end end + def test_parse_invalid_escape + assert_raise JSON::ParserError do + parse(%("fo\\o")) + end + + assert_equal "foo", parse(%("fo\\o"), allow_invalid_escape: true) + end + def test_parse_arrays assert_equal([1,2,3], parse('[1,2,3]')) assert_equal([1.2,2,3], parse('[1.2,2,3]')) From a888d6d57b9032ef4ecf7a1a9d73d551d0ae53c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Fri, 20 Feb 2026 11:58:01 +0100 Subject: [PATCH 271/506] Use single quotes for allow_invalid_escape doc Instead of using %{} which works like double-quoted string and allows escape sequences and in which "other escaped characters (a backslash followed by a character) are interpreted as the character", change the examples for `allow_invalid_escape` to use single-quotes strings, in which "any other character following a backslash [except ' and \] is interpreted as is: a backslash and the character itself." This makes it clearer the JSON document only includes a single backslash. --- lib/json.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json.rb b/lib/json.rb index 39e414beb..f28be7f77 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -201,10 +201,10 @@ # defaults to +false+. # # With the default, +false+: -# JSON.parse(%{"Hell\\o"}) # invalid escape character in string (JSON::ParserError) +# JSON.parse('"Hell\o"') # invalid escape character in string (JSON::ParserError) # # When enabled: -# JSON.parse(%{"Hell\\o"}, allow_invalid_escape: true) # => "Hello" +# JSON.parse('"Hell\o"', allow_invalid_escape: true) # => "Hello" # # ====== Output Options # From 93bc1b38386df5ab1b33eb1285672b0da90043a9 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 24 Feb 2026 21:53:15 -0600 Subject: [PATCH 272/506] Remove unused load_uint8x16_4 function. --- ext/json/ext/simd/simd.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 3bb86acde..611b41b06 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -133,16 +133,6 @@ ALWAYS_INLINE(static) int string_scan_simd_neon(const char **ptr, const char *en return 0; } -static inline uint8x16x4_t load_uint8x16_4(const unsigned char *table) -{ - uint8x16x4_t tab; - tab.val[0] = vld1q_u8(table); - tab.val[1] = vld1q_u8(table+16); - tab.val[2] = vld1q_u8(table+32); - tab.val[3] = vld1q_u8(table+48); - return tab; -} - #endif /* ARM Neon Support.*/ #if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) From 3f32c47de4dec7d06b9dda3430fe5c2e234ac660 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 20 Feb 2026 08:58:18 +0100 Subject: [PATCH 273/506] Reimplement `to_json` methods in Ruby --- ext/json/ext/generator/generator.c | 292 ++++-------------------- lib/json/common.rb | 46 +++- test/json/json_common_interface_test.rb | 12 +- test/json/json_generator_test.rb | 23 -- 4 files changed, 84 insertions(+), 289 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 186a45714..e699d3181 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -74,7 +74,6 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_false(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, VALUE obj); -static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj); @@ -701,206 +700,6 @@ static void convert_UTF8_to_ASCII_only_JSON(search_state *search, const unsigned } } -/* - * Document-module: JSON::Ext::Generator - * - * This is the JSON generator implemented as a C extension. It can be - * configured to be used by setting - * - * JSON.generator = JSON::Ext::Generator - * - * with the method generator= in JSON. - * - */ - -/* Explanation of the following: that's the only way to not pollute - * standard library's docs with GeneratorMethods:: which - * are uninformative and take a large place in a list of classes - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Array - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Bignum - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::FalseClass - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Fixnum - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Float - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Hash - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Integer - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::NilClass - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::Object - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::String - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::String::Extend - * :nodoc: - */ - -/* - * Document-module: JSON::Ext::Generator::GeneratorMethods::TrueClass - * :nodoc: - */ - -/* - * call-seq: to_json(state = nil) - * - * Returns a JSON string containing a JSON object, that is generated from - * this Hash instance. - * _state_ is a JSON::State object, that can also be used to configure the - * produced JSON string output further. - */ -static VALUE mHash_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_object, Qfalse); -} - -/* - * call-seq: to_json(state = nil) - * - * Returns a JSON string containing a JSON array, that is generated from - * this Array instance. - * _state_ is a JSON::State object, that can also be used to configure the - * produced JSON string output further. - */ -static VALUE mArray_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_array, Qfalse); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string representation for this Integer number. - */ -static VALUE mInteger_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_integer, Qfalse); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string representation for this Float number. - */ -static VALUE mFloat_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_float, Qfalse); -} - -/* - * call-seq: to_json(*) - * - * This string should be encoded with UTF-8 A call to this method - * returns a JSON string encoded with UTF16 big endian characters as - * \u????. - */ -static VALUE mString_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - VALUE Vstate = cState_from_state_s(cState, argc == 1 ? argv[0] : Qnil); - return cState_partial_generate(Vstate, self, generate_json_string, Qfalse); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string for true: 'true'. - */ -static VALUE mTrueClass_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - return rb_utf8_str_new("true", 4); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string for false: 'false'. - */ -static VALUE mFalseClass_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - return rb_utf8_str_new("false", 5); -} - -/* - * call-seq: to_json(*) - * - * Returns a JSON string for nil: 'null'. - */ -static VALUE mNilClass_to_json(int argc, VALUE *argv, VALUE self) -{ - rb_check_arity(argc, 0, 1); - return rb_utf8_str_new("null", 4); -} - -/* - * call-seq: to_json(*) - * - * Converts this object to a string (calling #to_s), converts - * it to a JSON string, and returns the result. This is a fallback, if no - * special method #to_json was defined for some object. - */ -static VALUE mObject_to_json(int argc, VALUE *argv, VALUE self) -{ - VALUE state; - VALUE string = rb_funcall(self, i_to_s, 0); - rb_scan_args(argc, argv, "01", &state); - Check_Type(string, T_STRING); - state = cState_from_state_s(cState, state); - return cState_partial_generate(state, string, generate_json_string, Qfalse); -} - static void State_mark(void *ptr) { JSON_Generator_State *state = ptr; @@ -1348,14 +1147,6 @@ static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *dat fbuffer_append_str(buffer, StringValue(tmp)); } -static void generate_json_integer(FBuffer *buffer, struct generate_json_data *data, VALUE obj) -{ - if (FIXNUM_P(obj)) - generate_json_fixnum(buffer, data, obj); - else - generate_json_bignum(buffer, data, obj); -} - static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { double value = RFLOAT_VALUE(obj); @@ -1399,7 +1190,7 @@ static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *d fbuffer_append_str(buffer, fragment); } -static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +static inline void generate_json_general(FBuffer *buffer, struct generate_json_data *data, VALUE obj, bool fallback) { bool as_json_called = false; start: @@ -1426,15 +1217,15 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALU generate_json_bignum(buffer, data, obj); break; case T_HASH: - if (klass != rb_cHash) goto general; + if (fallback && klass != rb_cHash) goto general; generate_json_object(buffer, data, obj); break; case T_ARRAY: - if (klass != rb_cArray) goto general; + if (fallback && klass != rb_cArray) goto general; generate_json_array(buffer, data, obj); break; case T_STRING: - if (klass != rb_cString) goto general; + if (fallback && klass != rb_cString) goto general; if (RB_LIKELY(valid_json_string_p(obj))) { raw_generate_json_string(buffer, data, obj); @@ -1450,7 +1241,7 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALU generate_json_symbol(buffer, data, obj); break; case T_FLOAT: - if (klass != rb_cFloat) goto general; + if (fallback && klass != rb_cFloat) goto general; generate_json_float(buffer, data, obj); break; case T_STRUCT: @@ -1474,6 +1265,16 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALU } } +static void generate_json(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +{ + generate_json_general(buffer, data, obj, true); +} + +static void generate_json_no_fallback(FBuffer *buffer, struct generate_json_data *data, VALUE obj) +{ + generate_json_general(buffer, data, obj, false); +} + static VALUE generate_json_try(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; @@ -1491,7 +1292,7 @@ static VALUE generate_json_ensure(VALUE d) return Qundef; } -static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io) +static inline VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, VALUE io) { GET_STATE(self); @@ -1509,9 +1310,7 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func, .obj = obj, .func = func }; - VALUE result = rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); - RB_GC_GUARD(self); - return result; + return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } /* call-seq: @@ -1530,6 +1329,15 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self) return cState_partial_generate(self, obj, generate_json, io); } +/* :nodoc: */ +static VALUE cState_generate_no_fallback(int argc, VALUE *argv, VALUE self) +{ + rb_check_arity(argc, 1, 2); + VALUE obj = argv[0]; + VALUE io = argc > 1 ? argv[1] : Qnil; + return cState_partial_generate(self, obj, generate_json_no_fallback, io); +} + static VALUE cState_initialize(int argc, VALUE *argv, VALUE self) { rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`"); @@ -2028,7 +1836,7 @@ static VALUE cState_configure(VALUE self, VALUE opts) return self; } -static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) +static VALUE cState_m_do_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io, generator_func func) { JSON_Generator_State state = {0}; state_init(&state); @@ -2046,14 +1854,21 @@ static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) .state = &state, .depth = state.depth, .obj = obj, - .func = generate_json, + .func = func, }; return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data); } -/* - * - */ +static VALUE cState_m_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io) +{ + return cState_m_do_generate(klass, obj, opts, io, generate_json); +} + +static VALUE cState_m_generate_no_fallback(VALUE klass, VALUE obj, VALUE opts, VALUE io) +{ + return cState_m_do_generate(klass, obj, opts, io, generate_json_no_fallback); +} + void Init_generator(void) { #ifdef HAVE_RB_EXT_RACTOR_SAFE @@ -2118,39 +1933,12 @@ void Init_generator(void) rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0); rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1); rb_define_method(cState, "generate", cState_generate, -1); + rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1); rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); rb_define_singleton_method(cState, "generate", cState_m_generate, 3); - - VALUE mGeneratorMethods = rb_define_module_under(mGenerator, "GeneratorMethods"); - - VALUE mObject = rb_define_module_under(mGeneratorMethods, "Object"); - rb_define_method(mObject, "to_json", mObject_to_json, -1); - - VALUE mHash = rb_define_module_under(mGeneratorMethods, "Hash"); - rb_define_method(mHash, "to_json", mHash_to_json, -1); - - VALUE mArray = rb_define_module_under(mGeneratorMethods, "Array"); - rb_define_method(mArray, "to_json", mArray_to_json, -1); - - VALUE mInteger = rb_define_module_under(mGeneratorMethods, "Integer"); - rb_define_method(mInteger, "to_json", mInteger_to_json, -1); - - VALUE mFloat = rb_define_module_under(mGeneratorMethods, "Float"); - rb_define_method(mFloat, "to_json", mFloat_to_json, -1); - - VALUE mString = rb_define_module_under(mGeneratorMethods, "String"); - rb_define_method(mString, "to_json", mString_to_json, -1); - - VALUE mTrueClass = rb_define_module_under(mGeneratorMethods, "TrueClass"); - rb_define_method(mTrueClass, "to_json", mTrueClass_to_json, -1); - - VALUE mFalseClass = rb_define_module_under(mGeneratorMethods, "FalseClass"); - rb_define_method(mFalseClass, "to_json", mFalseClass_to_json, -1); - - VALUE mNilClass = rb_define_module_under(mGeneratorMethods, "NilClass"); - rb_define_method(mNilClass, "to_json", mNilClass_to_json, -1); + rb_define_singleton_method(cState, "_generate_no_fallback", cState_m_generate_no_fallback, 3); rb_global_variable(&Encoding_UTF_8); Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8")); diff --git a/lib/json/common.rb b/lib/json/common.rb index 1a33c4103..fe2dd5226 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -156,15 +156,17 @@ def parser=(parser) # :nodoc: def generator=(generator) # :nodoc: old, $VERBOSE = $VERBOSE, nil @generator = generator - generator_methods = generator::GeneratorMethods - for const in generator_methods.constants - klass = const_get(const) - modul = generator_methods.const_get(const) - klass.class_eval do - instance_methods(false).each do |m| - m.to_s == 'to_json' and remove_method m + if generator.const_defined?(:GeneratorMethods) + generator_methods = generator::GeneratorMethods + for const in generator_methods.constants + klass = const_get(const) + modul = generator_methods.const_get(const) + klass.class_eval do + instance_methods(false).each do |m| + m.to_s == 'to_json' and remove_method m + end + include modul end - include modul end end self.state = generator::State @@ -1096,6 +1098,30 @@ def load_file(path) load(File.read(path, encoding: Encoding::UTF_8)) end end + + module GeneratorMethods + # call-seq: to_json(*) + # + # Converts this object into a JSON string. + # If this object doesn't directly maps to a JSON native type, + # first convert it to a string (calling #to_s), then converts + # it to a JSON string, and returns the result. + # This is a fallback, if no special method #to_json was defined for some object. + def to_json(state = nil, *) + obj = case self + when nil, false, true, Integer, Float, Array, Hash + self + else + "#{self}" + end + + if state.nil? + JSON::State._generate_no_fallback(obj, nil, nil) + else + JSON::State.from_state(state)._generate_no_fallback(obj) + end + end + end end module ::Kernel @@ -1141,3 +1167,7 @@ def JSON(object, opts = nil) JSON[object, opts] end end + +class Object + include JSON::GeneratorMethods +end diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 3dfd0623c..b6001e3fb 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -42,12 +42,6 @@ def setup '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}' end - def test_index - assert_equal @json, JSON[@hash] - assert_equal @json, JSON[@hash_with_method_missing] - assert_equal @hash, JSON[@json] - end - def test_parser assert_match(/::Parser\z/, JSON.parser.name) end @@ -287,6 +281,12 @@ def test_JSON assert_equal @hash, JSON(@json) end + def test_index + assert_equal @json, JSON[@hash] + assert_equal @json, JSON[@hash_with_method_missing] + assert_equal @hash, JSON[@json] + end + def test_load_file test_load_shared(:load_file) end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index d5f6ac64a..9a8ee5157 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -921,29 +921,6 @@ def test_valid_utf8_in_different_encoding assert_equal JSON.dump(utf8_string), JSON.dump(wrong_encoding_string) end end - - def test_string_ext_included_calls_super - included = false - - Module.send(:alias_method, :included_orig, :included) - Module.send(:remove_method, :included) - Module.send(:define_method, :included) do |base| - included_orig(base) - included = true - end - - Class.new(String) do - include JSON::Ext::Generator::GeneratorMethods::String - end - - assert included - ensure - if Module.private_method_defined?(:included_orig) - Module.send(:remove_method, :included) if Module.method_defined?(:included) - Module.send(:alias_method, :included, :included_orig) - Module.send(:remove_method, :included_orig) - end - end end def test_nonutf8_encoding From 6ccc102db674ad67b3d0795cdeb265490d048d73 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 5 Mar 2026 12:38:46 +0100 Subject: [PATCH 274/506] Fix `allow_blank` parsing option to only consider strings. Ref: https://github.com/ruby/json/pull/946 --- CHANGES.md | 1 + lib/json/common.rb | 2 +- test/json/json_common_interface_test.rb | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ceb17198c..b225f4eba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Fix `allow_blank` parsing option to no longer allow invalid types (e.g. `load([], allow_blank: true)` now raise a type error). * Add `allow_invalid_escape` parsing option to ignore backslashes that aren't followed by one of the valid escape characters. ### 2026-02-03 (2.18.1) diff --git a/lib/json/common.rb b/lib/json/common.rb index fe2dd5226..230bf0801 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -880,7 +880,7 @@ def load(source, proc = nil, options = nil) end end - if opts[:allow_blank] && (source.nil? || source.empty?) + if opts[:allow_blank] && (source.nil? || (String === source && source.empty?)) source = 'null' end diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index b6001e3fb..37568b556 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -150,6 +150,8 @@ def test_load_null assert_equal nil, JSON.load(nil, nil, :allow_blank => true) assert_raise(TypeError) { JSON.load(nil, nil, :allow_blank => false) } assert_raise(JSON::ParserError) { JSON.load('', nil, :allow_blank => false) } + assert_raise(TypeError) { JSON.load([], nil, :allow_blank => true) } + assert_raise(TypeError) { JSON.load({}, nil, :allow_blank => true) } end def test_unsafe_load From de993aa76639078da891f46351a36f77d51ad3d3 Mon Sep 17 00:00:00 2001 From: thesmartshadow Date: Thu, 5 Mar 2026 18:58:27 +0000 Subject: [PATCH 275/506] Reject negative depth; add overflow guards to prevent hang/crash --- ext/json/ext/generator/generator.c | 15 +++++++++++++-- test/json/json_generator_test.rb | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index e699d3181..a6302691d 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1575,6 +1575,17 @@ static long long_config(VALUE num) return RTEST(num) ? FIX2LONG(num) : 0; } +// depth must never be negative; reject early with a clear error. +static long depth_config(VALUE num) +{ + if (!RTEST(num)) return 0; + long d = NUM2LONG(num); + if (RB_UNLIKELY(d < 0)) { + rb_raise(rb_eArgError, "depth must be >= 0 (got %ld)", d); + } + return d; +} + /* * call-seq: max_nesting=(depth) * @@ -1731,7 +1742,7 @@ static VALUE cState_depth_set(VALUE self, VALUE depth) { rb_check_frozen(self); GET_STATE(self); - state->depth = long_config(depth); + state->depth = depth_config(depth); return Qnil; } @@ -1796,7 +1807,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_max_nesting) { state->max_nesting = long_config(val); } else if (key == sym_allow_nan) { state->allow_nan = RTEST(val); } else if (key == sym_ascii_only) { state->ascii_only = RTEST(val); } - else if (key == sym_depth) { state->depth = long_config(val); } + else if (key == sym_depth) { state->depth = depth_config(val); } else if (key == sym_buffer_initial_length) { buffer_initial_length_set(state, val); } else if (key == sym_script_safe) { state->script_safe = RTEST(val); } else if (key == sym_escape_slash) { state->script_safe = RTEST(val); } diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 9a8ee5157..eb5a8ca59 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1045,4 +1045,14 @@ def test_nesting_recovery assert_equal 0, state.depth assert_equal '{"a":1}', state.generate({ a: 1 }) end + + def test_negative_depth_raises + assert_raise(ArgumentError) do + JSON.generate({"a" => 1}, depth: -1) + end + assert_raise(ArgumentError) do + JSON.state.new(depth: -1) + end + end + end From a29fcdcb4a78164daa14f6af05812690dd3ac939 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Mar 2026 08:39:13 +0100 Subject: [PATCH 276/506] Add depth validation to Jruby and TruffleRuby implementations --- java/src/json/ext/GeneratorState.java | 8 +++++++- lib/json/truffle_ruby/generator.rb | 14 +++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 8db31bdaa..11d55559c 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -484,9 +484,12 @@ public RubyInteger depth_get(ThreadContext context) { } @JRubyMethod(name="depth=") - public IRubyObject depth_set(IRubyObject vDepth) { + public IRubyObject depth_set(ThreadContext context, IRubyObject vDepth) { checkFrozen(); depth = RubyNumeric.fix2int(vDepth); + if (depth < 0) { + throw context.runtime.newArgumentError("depth must be >= 0 (got: " + depth + ")"); + } return vDepth; } @@ -557,6 +560,9 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { bufferInitialLength = opts.getInt("buffer_initial_length", DEFAULT_BUFFER_INITIAL_LENGTH); depth = opts.getInt("depth", 0); + if (depth < 0) { + throw context.runtime.newArgumentError("depth must be >= 0 (got: " + depth + ")"); + } if (opts.hasKey("allow_duplicate_key")) { this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 9a3f4aed0..13b15459c 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -211,7 +211,14 @@ def buffer_initial_length=(length) # This integer returns the current depth data structure nesting in the # generated JSON. - attr_accessor :depth + attr_reader :depth + + def depth=(depth) + if depth.negative? + raise ArgumentError, "depth must be >= 0 (got #{depth})" + end + @depth = depth + end def check_max_nesting # :nodoc: return if @max_nesting.zero? @@ -260,6 +267,11 @@ def configure(opts) else raise TypeError, "can't convert #{opts.class} into Hash" end + + if opts[:depth]&.negative? + raise ArgumentError, "depth must be >= 0 (got #{opts[:depth]})" + end + opts.each do |key, value| instance_variable_set "@#{key}", value end From 0a4fb79cd97f535701cc2240ac736d76b9af5025 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Mar 2026 08:51:43 +0100 Subject: [PATCH 277/506] fbuffer.h: Use size_t over unsigned long unsigned long is only 32b on some platforms. --- ext/json/ext/fbuffer/fbuffer.h | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 8ce029a8c..d5fd5ea26 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -11,11 +11,11 @@ enum fbuffer_type { typedef struct FBufferStruct { enum fbuffer_type type; - unsigned long initial_length; - unsigned long len; - unsigned long capa; + size_t initial_length; + size_t len; + size_t capa; #if JSON_DEBUG - unsigned long requested; + size_t requested; #endif char *ptr; VALUE io; @@ -32,12 +32,12 @@ typedef struct FBufferStruct { static void fbuffer_free(FBuffer *fb); static void fbuffer_clear(FBuffer *fb); -static void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len); +static void fbuffer_append(FBuffer *fb, const char *newstr, size_t len); static void fbuffer_append_long(FBuffer *fb, long number); static inline void fbuffer_append_char(FBuffer *fb, char newchr); static VALUE fbuffer_finalize(FBuffer *fb); -static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char *stack_buffer, long stack_buffer_size) +static void fbuffer_stack_init(FBuffer *fb, size_t initial_length, char *stack_buffer, size_t stack_buffer_size) { fb->initial_length = (initial_length > 0) ? initial_length : FBUFFER_INITIAL_LENGTH_DEFAULT; if (stack_buffer) { @@ -50,7 +50,7 @@ static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char * #endif } -static inline void fbuffer_consumed(FBuffer *fb, unsigned long consumed) +static inline void fbuffer_consumed(FBuffer *fb, size_t consumed) { #if JSON_DEBUG if (consumed > fb->requested) { @@ -79,7 +79,7 @@ static void fbuffer_flush(FBuffer *fb) fbuffer_clear(fb); } -static void fbuffer_realloc(FBuffer *fb, unsigned long required) +static void fbuffer_realloc(FBuffer *fb, size_t required) { if (required > fb->capa) { if (fb->type == FBUFFER_STACK_ALLOCATED) { @@ -94,7 +94,7 @@ static void fbuffer_realloc(FBuffer *fb, unsigned long required) } } -static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested) +static void fbuffer_do_inc_capa(FBuffer *fb, size_t requested) { if (RB_UNLIKELY(fb->io)) { if (fb->capa < FBUFFER_IO_BUFFER_SIZE) { @@ -108,7 +108,7 @@ static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested) } } - unsigned long required; + size_t required; if (RB_UNLIKELY(!fb->ptr)) { fb->ptr = ALLOC_N(char, fb->initial_length); @@ -120,7 +120,7 @@ static void fbuffer_do_inc_capa(FBuffer *fb, unsigned long requested) fbuffer_realloc(fb, required); } -static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested) +static inline void fbuffer_inc_capa(FBuffer *fb, size_t requested) { #if JSON_DEBUG fb->requested = requested; @@ -131,13 +131,13 @@ static inline void fbuffer_inc_capa(FBuffer *fb, unsigned long requested) } } -static inline void fbuffer_append_reserved(FBuffer *fb, const char *newstr, unsigned long len) +static inline void fbuffer_append_reserved(FBuffer *fb, const char *newstr, size_t len) { MEMCPY(fb->ptr + fb->len, newstr, char, len); fbuffer_consumed(fb, len); } -static inline void fbuffer_append(FBuffer *fb, const char *newstr, unsigned long len) +static inline void fbuffer_append(FBuffer *fb, const char *newstr, size_t len) { if (len > 0) { fbuffer_inc_capa(fb, len); @@ -162,7 +162,7 @@ static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) static void fbuffer_append_str(FBuffer *fb, VALUE str) { const char *ptr; - unsigned long len; + size_t len; RSTRING_GETMEM(str, ptr, len); fbuffer_append(fb, ptr, len); @@ -171,7 +171,7 @@ static void fbuffer_append_str(FBuffer *fb, VALUE str) static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) { const char *ptr; - unsigned long len; + size_t len; RSTRING_GETMEM(str, ptr, len); fbuffer_inc_capa(fb, repeat * len); From a11acc1ff496627e5d72c71d6d1229e8c8ffeaa1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Mar 2026 09:03:58 +0100 Subject: [PATCH 278/506] Release 2.19.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b225f4eba..a66b4d45b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-03-06 (2.19.0) + * Fix `allow_blank` parsing option to no longer allow invalid types (e.g. `load([], allow_blank: true)` now raise a type error). * Add `allow_invalid_escape` parsing option to ignore backslashes that aren't followed by one of the valid escape characters. diff --git a/lib/json/version.rb b/lib/json/version.rb index 95b885710..3f73c0d97 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.18.1' + VERSION = '2.19.0' end From 13689c269970f18316952541f8544830ec2dc5c4 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 8 Mar 2026 09:43:26 +0100 Subject: [PATCH 279/506] Add missing GC_GUARD in `fbuffer_append_str` Ref: fff25c9f4b9c590a59ed4a2ec3c545121db844d3 `StringValuePtr` use `volatile` so the compiler is less likely to re-use the register. But regardless, we should GC_GUARD `str` as we no longer reference it after `GETMEM`. --- ext/json/ext/fbuffer/fbuffer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index d5fd5ea26..9660e70dc 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -166,6 +166,7 @@ static void fbuffer_append_str(FBuffer *fb, VALUE str) RSTRING_GETMEM(str, ptr, len); fbuffer_append(fb, ptr, len); + RB_GC_GUARD(str); } static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) @@ -182,6 +183,7 @@ static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) fbuffer_append_reserved(fb, ptr, len); repeat--; } + RB_GC_GUARD(str); } static inline void fbuffer_append_char(FBuffer *fb, char newchr) From 4a42a04280d96d8dd94558078c16f1c078c38e1b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 8 Mar 2026 10:13:53 +0100 Subject: [PATCH 280/506] Release 2.19.1 --- CHANGES.md | 4 ++++ lib/json/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a66b4d45b..3638ab3ef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ### Unreleased +### 2026-03-08 (2.19.1) + +* Fix a compiler dependent GC bug introduced in `2.18.0`. + ### 2026-03-06 (2.19.0) * Fix `allow_blank` parsing option to no longer allow invalid types (e.g. `load([], allow_blank: true)` now raise a type error). diff --git a/lib/json/version.rb b/lib/json/version.rb index 3f73c0d97..aa56d3a4b 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.0' + VERSION = '2.19.1' end From 7187315b4571ade59d68a1fad84be2794cda744d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 03:05:59 +0000 Subject: [PATCH 281/506] Bump actions/create-github-app-token from 2 to 3 Bumps [actions/create-github-app-token](https://github.com/actions/create-github-app-token) from 2 to 3. - [Release notes](https://github.com/actions/create-github-app-token/releases) - [Commits](https://github.com/actions/create-github-app-token/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/create-github-app-token dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/sync-ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-ruby.yml b/.github/workflows/sync-ruby.yml index 78e94eb9e..f5df56c5b 100644 --- a/.github/workflows/sync-ruby.yml +++ b/.github/workflows/sync-ruby.yml @@ -12,7 +12,7 @@ jobs: - name: Create GitHub App token id: app-token - uses: actions/create-github-app-token@v2 + uses: actions/create-github-app-token@v3 with: app-id: 2060836 private-key: ${{ secrets.RUBY_SYNC_DEFAULT_GEMS_PRIVATE_KEY }} From 393b41c3e5f87491e1e34fa59fa78ff6fa179a74 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 18 Mar 2026 08:56:17 +0100 Subject: [PATCH 282/506] Fix a format string injection vulnerability In `JSON.parse(doc, allow_duplicate_key: false)`. --- CHANGES.md | 2 ++ ext/json/ext/parser/parser.c | 23 +++++++++++++++++++---- test/json/json_parser_test.rb | 7 +++++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3638ab3ef..50cd1299c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. + ### 2026-03-08 (2.19.1) * Fix a compiler dependent GC bug introduced in `2.18.0`. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 42ca10894..05cfaa7c6 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -402,11 +402,9 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state) #define PARSE_ERROR_FRAGMENT_LEN 32 -NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state) +static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column) { unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; - long line, column; - cursor_position(state, &line, &column); const char *ptr = "EOF"; if (state->cursor && state->cursor < state->end) { @@ -441,11 +439,23 @@ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *st VALUE msg = rb_sprintf(format, ptr); VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column); RB_GC_GUARD(msg); + return message; +} +static VALUE parse_error_new(VALUE message, long line, long column) +{ VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message); rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line)); rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column)); - rb_exc_raise(exc); + return exc; +} + +NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state) +{ + long line, column; + cursor_position(state, &line, &column); + VALUE message = build_parse_error_message(format, state, line, column); + rb_exc_raise(parse_error_new(message, line, column)); } NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) @@ -895,6 +905,11 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d rb_inspect(duplicate_key) ); + long line, column; + cursor_position(state, &line, &column); + rb_str_concat(message, build_parse_error_message("", state, line, column)) ; + rb_exc_raise(parse_error_new(message, line, column)); + raise_parse_error(RSTRING_PTR(message), state); RB_GC_GUARD(message); } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 653abf46c..2d2f065ec 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -425,6 +425,13 @@ def test_parse_duplicate_key end end + def test_parse_duplicate_key_escape + error = assert_raise(ParserError) do + JSON.parse('{"%s%s%s%s":1,"%s%s%s%s":2}', allow_duplicate_key: false) + end + assert_match "%s%s%s%s", error.message + end + def test_some_wrong_inputs assert_raise(ParserError) { parse('[] bla') } assert_raise(ParserError) { parse('[] 1') } From 54f8a878aebee090476a53c851c943128894be62 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 18 Mar 2026 18:23:53 +0100 Subject: [PATCH 283/506] Release 2.19.2 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 50cd1299c..83faa865a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-03-08 (2.19.2) + * Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. ### 2026-03-08 (2.19.1) diff --git a/lib/json/version.rb b/lib/json/version.rb index aa56d3a4b..8853ed885 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.1' + VERSION = '2.19.2' end From c86a66178154b46e77987c5006d220a1514c996b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 18 Mar 2026 18:51:18 +0100 Subject: [PATCH 284/506] Add security backports in the changelog --- CHANGES.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 83faa865a..3c23aa8f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,9 +2,9 @@ ### Unreleased -### 2026-03-08 (2.19.2) +### 2026-03-18 (2.19.2) -* Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. +* Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. `CVE-2026-33210`. ### 2026-03-08 (2.19.1) @@ -24,6 +24,10 @@ * Add `:allow_control_characters` parser options, to allow JSON strings containing unescaped ASCII control characters (e.g. newlines). +### 2026-03-18 (2.17.1.2) - Security Backport + +* Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. `CVE-2026-33210`. + ### 2025-12-04 (2.17.1) * Fix a regression in parsing of unicode surogate pairs (`\uXX\uXX`) that could cause an invalid string to be returned. @@ -50,6 +54,10 @@ * Optimized numbers parsing using SWAR (thanks to Scott Myron). * Optimized parsing of pretty printed documents using SWAR (thanks to Scott Myron). +### 2026-03-18 (2.15.2.1) - Security Backport + +* Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. `CVE-2026-33210`. + ### 2025-10-25 (2.15.2) * Fix `JSON::Coder` to have one dedicated depth counter per invocation. From 398340fca24a4aac1315b6c0dd8d4d385731926f Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 19 Mar 2026 18:53:18 +0900 Subject: [PATCH 285/506] Remove unreachable code `rb_exc_raise` never returns. --- ext/json/ext/parser/parser.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 05cfaa7c6..3587c4d7d 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -909,9 +909,6 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d cursor_position(state, &line, &column); rb_str_concat(message, build_parse_error_message("", state, line, column)) ; rb_exc_raise(parse_error_new(message, line, column)); - - raise_parse_error(RSTRING_PTR(message), state); - RB_GC_GUARD(message); } static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count) From c5eb1244c7b2d2af0e7b8a2a4f531373e74fd0d7 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 19 Mar 2026 18:55:51 +0900 Subject: [PATCH 286/506] Reduce an unnecessary intermediate string --- ext/json/ext/parser/parser.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 3587c4d7d..1a8e2aaf4 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -436,9 +436,8 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta } } - VALUE msg = rb_sprintf(format, ptr); - VALUE message = rb_enc_sprintf(enc_utf8, "%s at line %ld column %ld", RSTRING_PTR(msg), line, column); - RB_GC_GUARD(msg); + VALUE message = rb_enc_sprintf(enc_utf8, format, ptr); + rb_str_catf(message, " at line %ld column %ld", line, column); return message; } From 89fe63069da272834c9dcf65ba0b7dc7a093bb97 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Mar 2026 21:28:22 +0100 Subject: [PATCH 287/506] Use embeddable types Ref: https://bugs.ruby-lang.org/issues/21853 Introduced in Ruby 3.3, will officially be public API in 4.1, but can be used sooner by checking the existence of RUBY_TYPED_EMBEDDABLE --- ext/json/ext/generator/extconf.rb | 2 ++ ext/json/ext/generator/generator.c | 15 ++++----------- ext/json/ext/json.h | 6 ++++++ ext/json/ext/parser/extconf.rb | 1 + ext/json/ext/parser/parser.c | 18 ++++++++++-------- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index ee1bbeaba..07ca69d22 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -5,6 +5,8 @@ File.write('Makefile', dummy_makefile("").join) else append_cflags("-std=c99") + have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 + $defs << "-DJSON_GENERATOR" $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index a6302691d..b52bc9751 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -722,27 +722,20 @@ static void State_compact(void *ptr) state->as_json = rb_gc_location(state->as_json); } -static void State_free(void *ptr) -{ - JSON_Generator_State *state = ptr; - ruby_xfree(state); -} - static size_t State_memsize(const void *ptr) { return sizeof(JSON_Generator_State); } static const rb_data_type_t JSON_Generator_State_type = { - "JSON/Generator/State", - { + .wrap_struct_name = "JSON/Generator/State", + .function = { .dmark = State_mark, - .dfree = State_free, + .dfree = RUBY_DEFAULT_FREE, .dsize = State_memsize, .dcompact = State_compact, }, - 0, 0, - RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static void state_init(JSON_Generator_State *state) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 087a2eae6..cad8788e8 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -54,6 +54,10 @@ typedef unsigned char _Bool; # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE +# define RUBY_TYPED_EMBEDDABLE 0 +#endif + #ifndef NORETURN #if defined(__has_attribute) && __has_attribute(noreturn) #define NORETURN(x) __attribute__((noreturn)) x @@ -102,4 +106,6 @@ typedef unsigned char _Bool; #define JSON_CPU_LITTLE_ENDIAN_64BITS 0 #endif + + #endif // _JSON_H_ diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 2440e66d8..a9509e6b2 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -6,6 +6,7 @@ have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby +have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 append_cflags("-std=c99") diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1a8e2aaf4..bab75ab52 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -251,7 +251,9 @@ static void rvalue_stack_free(void *ptr) rvalue_stack *stack = (rvalue_stack *)ptr; if (stack) { ruby_xfree(stack->ptr); +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE ruby_xfree(stack); +#endif } } @@ -262,14 +264,13 @@ static size_t rvalue_stack_memsize(const void *ptr) } static const rb_data_type_t JSON_Parser_rvalue_stack_type = { - "JSON::Ext::Parser/rvalue_stack", - { + .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack", + .function = { .dmark = rvalue_stack_mark, .dfree = rvalue_stack_free, .dsize = rvalue_stack_memsize, }, - 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, }; static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref) @@ -1608,7 +1609,9 @@ static void JSON_ParserConfig_mark(void *ptr) static void JSON_ParserConfig_free(void *ptr) { JSON_ParserConfig *config = ptr; +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE ruby_xfree(config); +#endif } static size_t JSON_ParserConfig_memsize(const void *ptr) @@ -1617,14 +1620,13 @@ static size_t JSON_ParserConfig_memsize(const void *ptr) } static const rb_data_type_t JSON_ParserConfig_type = { - "JSON::Ext::Parser/ParserConfig", - { + .wrap_struct_name = "JSON::Ext::Parser/ParserConfig", + .function = { JSON_ParserConfig_mark, JSON_ParserConfig_free, JSON_ParserConfig_memsize, }, - 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static VALUE cJSON_parser_s_allocate(VALUE klass) From 4e2173d7c85d95cc901f3b03d37fbe55ff7f0a9d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Mar 2026 21:56:51 +0100 Subject: [PATCH 288/506] Fix Ruby 4.1.0dev compatibility --- ext/json/ext/json.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index cad8788e8..abd3dac11 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -54,7 +54,7 @@ typedef unsigned char _Bool; # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif -#ifndef HAVE_RUBY_TYPED_EMBEDDABLE +#ifndef RUBY_TYPED_EMBEDDABLE # define RUBY_TYPED_EMBEDDABLE 0 #endif From 8364dbd491556b48618a36538093d5a5a6ab0c32 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Mar 2026 22:27:06 +0100 Subject: [PATCH 289/506] Revert use of RUBY_TYPED_EMBEDDABLE --- ext/json/ext/generator/extconf.rb | 2 -- ext/json/ext/generator/generator.c | 15 +++++++++++---- ext/json/ext/json.h | 6 ------ ext/json/ext/parser/extconf.rb | 1 - ext/json/ext/parser/parser.c | 18 ++++++++---------- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 07ca69d22..ee1bbeaba 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -5,8 +5,6 @@ File.write('Makefile', dummy_makefile("").join) else append_cflags("-std=c99") - have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 - $defs << "-DJSON_GENERATOR" $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index b52bc9751..a6302691d 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -722,20 +722,27 @@ static void State_compact(void *ptr) state->as_json = rb_gc_location(state->as_json); } +static void State_free(void *ptr) +{ + JSON_Generator_State *state = ptr; + ruby_xfree(state); +} + static size_t State_memsize(const void *ptr) { return sizeof(JSON_Generator_State); } static const rb_data_type_t JSON_Generator_State_type = { - .wrap_struct_name = "JSON/Generator/State", - .function = { + "JSON/Generator/State", + { .dmark = State_mark, - .dfree = RUBY_DEFAULT_FREE, + .dfree = State_free, .dsize = State_memsize, .dcompact = State_compact, }, - .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, + 0, 0, + RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE, }; static void state_init(JSON_Generator_State *state) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index abd3dac11..087a2eae6 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -54,10 +54,6 @@ typedef unsigned char _Bool; # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif -#ifndef RUBY_TYPED_EMBEDDABLE -# define RUBY_TYPED_EMBEDDABLE 0 -#endif - #ifndef NORETURN #if defined(__has_attribute) && __has_attribute(noreturn) #define NORETURN(x) __attribute__((noreturn)) x @@ -106,6 +102,4 @@ typedef unsigned char _Bool; #define JSON_CPU_LITTLE_ENDIAN_64BITS 0 #endif - - #endif // _JSON_H_ diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index a9509e6b2..2440e66d8 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -6,7 +6,6 @@ have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby -have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 append_cflags("-std=c99") diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index bab75ab52..1a8e2aaf4 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -251,9 +251,7 @@ static void rvalue_stack_free(void *ptr) rvalue_stack *stack = (rvalue_stack *)ptr; if (stack) { ruby_xfree(stack->ptr); -#ifndef HAVE_RUBY_TYPED_EMBEDDABLE ruby_xfree(stack); -#endif } } @@ -264,13 +262,14 @@ static size_t rvalue_stack_memsize(const void *ptr) } static const rb_data_type_t JSON_Parser_rvalue_stack_type = { - .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack", - .function = { + "JSON::Ext::Parser/rvalue_stack", + { .dmark = rvalue_stack_mark, .dfree = rvalue_stack_free, .dsize = rvalue_stack_memsize, }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, + 0, 0, + RUBY_TYPED_FREE_IMMEDIATELY, }; static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref) @@ -1609,9 +1608,7 @@ static void JSON_ParserConfig_mark(void *ptr) static void JSON_ParserConfig_free(void *ptr) { JSON_ParserConfig *config = ptr; -#ifndef HAVE_RUBY_TYPED_EMBEDDABLE ruby_xfree(config); -#endif } static size_t JSON_ParserConfig_memsize(const void *ptr) @@ -1620,13 +1617,14 @@ static size_t JSON_ParserConfig_memsize(const void *ptr) } static const rb_data_type_t JSON_ParserConfig_type = { - .wrap_struct_name = "JSON::Ext::Parser/ParserConfig", - .function = { + "JSON::Ext::Parser/ParserConfig", + { JSON_ParserConfig_mark, JSON_ParserConfig_free, JSON_ParserConfig_memsize, }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, + 0, 0, + RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE, }; static VALUE cJSON_parser_s_allocate(VALUE klass) From 9aff85e61cdfbebe5f395f6fe8ba2edbd9969826 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Mar 2026 23:15:47 +0100 Subject: [PATCH 290/506] Run CI against ruby-head-debug --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a79be455..38f9494bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: env: - "" include: + - { os: ubuntu-latest , ruby: debug } - { os: ubuntu-24.04-arm, ruby: 3.4 } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } From 101f7342cd61f21d17405eff52fec23bb958d6c7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 23 Mar 2026 22:27:52 +0100 Subject: [PATCH 291/506] Use embeddable types Ref: https://bugs.ruby-lang.org/issues/21853 Introduced in Ruby 3.3, will officially be public API in 4.1, but can be used sooner by checking the existence of RUBY_TYPED_EMBEDDABLE --- ext/json/ext/generator/extconf.rb | 2 ++ ext/json/ext/generator/generator.c | 15 +++------- ext/json/ext/json.h | 11 +++++++ ext/json/ext/parser/extconf.rb | 4 +++ ext/json/ext/parser/parser.c | 48 +++++++++++++++++------------- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index ee1bbeaba..205a887e7 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -5,6 +5,8 @@ File.write('Makefile', dummy_makefile("").join) else append_cflags("-std=c99") + have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 + $defs << "-DJSON_GENERATOR" $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index a6302691d..b52bc9751 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -722,27 +722,20 @@ static void State_compact(void *ptr) state->as_json = rb_gc_location(state->as_json); } -static void State_free(void *ptr) -{ - JSON_Generator_State *state = ptr; - ruby_xfree(state); -} - static size_t State_memsize(const void *ptr) { return sizeof(JSON_Generator_State); } static const rb_data_type_t JSON_Generator_State_type = { - "JSON/Generator/State", - { + .wrap_struct_name = "JSON/Generator/State", + .function = { .dmark = State_mark, - .dfree = State_free, + .dfree = RUBY_DEFAULT_FREE, .dsize = State_memsize, .dcompact = State_compact, }, - 0, 0, - RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static void state_init(JSON_Generator_State *state) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 087a2eae6..8737989a6 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -54,6 +54,17 @@ typedef unsigned char _Bool; # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif +#ifdef RUBY_TYPED_EMBEDDABLE +# define HAVE_RUBY_TYPED_EMBEDDABLE 1 +#else +# ifdef HAVE_CONST_RUBY_TYPED_EMBEDDABLE +# define RUBY_TYPED_EMBEDDABLE RUBY_TYPED_EMBEDDABLE +# define HAVE_RUBY_TYPED_EMBEDDABLE 1 +# else +# define RUBY_TYPED_EMBEDDABLE 0 +# endif +#endif + #ifndef NORETURN #if defined(__has_attribute) && __has_attribute(noreturn) #define NORETURN(x) __attribute__((noreturn)) x diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 2440e66d8..f12fc2ddd 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -7,6 +7,10 @@ have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby +if RUBY_ENGINE == "ruby" + have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 +end + append_cflags("-std=c99") if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1a8e2aaf4..361db3a95 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -246,12 +246,20 @@ static void rvalue_stack_mark(void *ptr) } } +static void rvalue_stack_free_buffer(rvalue_stack *stack) +{ + ruby_xfree(stack->ptr); + stack->ptr = NULL; +} + static void rvalue_stack_free(void *ptr) { rvalue_stack *stack = (rvalue_stack *)ptr; if (stack) { - ruby_xfree(stack->ptr); + rvalue_stack_free_buffer(stack); +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE ruby_xfree(stack); +#endif } } @@ -262,14 +270,13 @@ static size_t rvalue_stack_memsize(const void *ptr) } static const rb_data_type_t JSON_Parser_rvalue_stack_type = { - "JSON::Ext::Parser/rvalue_stack", - { + .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack", + .function = { .dmark = rvalue_stack_mark, .dfree = rvalue_stack_free, .dsize = rvalue_stack_memsize, }, - 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, }; static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref) @@ -291,8 +298,12 @@ static void rvalue_stack_eagerly_release(VALUE handle) if (handle) { rvalue_stack *stack; TypedData_Get_Struct(handle, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack); - RTYPEDDATA_DATA(handle) = NULL; +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + rvalue_stack_free_buffer(stack); +#else rvalue_stack_free(stack); + RTYPEDDATA_DATA(handle) = NULL; +#endif } } @@ -343,7 +354,7 @@ typedef struct JSON_ParserStruct { } JSON_ParserConfig; typedef struct JSON_ParserStateStruct { - VALUE stack_handle; + VALUE *stack_handle; const char *start; const char *cursor; const char *end; @@ -944,7 +955,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig * if (RB_UNLIKELY(config->on_load_proc)) { value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil); } - rvalue_stack_push(state->stack, value, &state->stack_handle, &state->stack); + rvalue_stack_push(state->stack, value, state->stack_handle, &state->stack); return value; } @@ -1554,11 +1565,13 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) const char *start; RSTRING_GETMEM(Vsource, start, len); + VALUE stack_handle = 0; JSON_ParserState _state = { .start = start, .cursor = start, .end = start + len, .stack = &stack, + .stack_handle = &stack_handle, }; JSON_ParserState *state = &_state; @@ -1566,8 +1579,8 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) // This may be skipped in case of exception, but // it won't cause a leak. - rvalue_stack_eagerly_release(state->stack_handle); - + rvalue_stack_eagerly_release(stack_handle); + RB_GC_GUARD(stack_handle); json_ensure_eof(state); return result; @@ -1605,26 +1618,19 @@ static void JSON_ParserConfig_mark(void *ptr) rb_gc_mark(config->decimal_class); } -static void JSON_ParserConfig_free(void *ptr) -{ - JSON_ParserConfig *config = ptr; - ruby_xfree(config); -} - static size_t JSON_ParserConfig_memsize(const void *ptr) { return sizeof(JSON_ParserConfig); } static const rb_data_type_t JSON_ParserConfig_type = { - "JSON::Ext::Parser/ParserConfig", - { + .wrap_struct_name = "JSON::Ext::Parser/ParserConfig", + .function = { JSON_ParserConfig_mark, - JSON_ParserConfig_free, + RUBY_DEFAULT_FREE, JSON_ParserConfig_memsize, }, - 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static VALUE cJSON_parser_s_allocate(VALUE klass) From 09d4d8899a7ca55cce927509442025f29324294f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 24 Mar 2026 09:35:11 +0100 Subject: [PATCH 292/506] parser.c: handle eagerly freed rstack in rvalue_stack_mark --- ext/json/ext/parser/parser.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 361db3a95..b05f3f527 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -241,8 +241,10 @@ static void rvalue_stack_mark(void *ptr) { rvalue_stack *stack = (rvalue_stack *)ptr; long index; - for (index = 0; index < stack->head; index++) { - rb_gc_mark(stack->ptr[index]); + if (stack && stack->ptr) { + for (index = 0; index < stack->head; index++) { + rb_gc_mark(stack->ptr[index]); + } } } From b575be5302f9f007ef30aafc0523986e5707c426 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 25 Mar 2026 11:45:17 +0100 Subject: [PATCH 293/506] Fix handling of unescaped control characters preceeded by a backslash --- CHANGES.md | 2 ++ ext/json/ext/parser/parser.c | 4 +++- test/json/json_parser_test.rb | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3c23aa8f2..4bb85ca0f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix handling of unescaped control characters preceeded by a backslash. + ### 2026-03-18 (2.19.2) * Fix a format string injection vulnerability in `JSON.parse(doc, allow_duplicate_key: false)`. `CVE-2026-33210`. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b05f3f527..38f3a48ce 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -770,7 +770,9 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser } raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); } - } else if (config->allow_invalid_escape) { + } + + if (config->allow_invalid_escape) { APPEND_CHAR(*pe); } else { raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 2d2f065ec..af08d1835 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -183,6 +183,15 @@ def test_parse_allowed_control_chars_in_string end end + def test_parsse_control_char_and_backslash + backslash_and_control_char = "\\\t" + assert_raise JSON::ParserError do + JSON.parse(%("#{'a' * 30}#{backslash_and_control_char}"), allow_control_characters: true, allow_invalid_escape: false) + end + + JSON.parse(%("#{'a' * 30}#{backslash_and_control_char}"), allow_control_characters: true, allow_invalid_escape: true) + end + def test_parse_invalid_escape assert_raise JSON::ParserError do parse(%("fo\\o")) From 37caa8927287278e45f8e20993ff9e57df2231fd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 25 Mar 2026 12:02:34 +0100 Subject: [PATCH 294/506] Release 2.19.3 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- test/json/json_parser_test.rb | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4bb85ca0f..7c295099e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-03-25 (2.19.3) + * Fix handling of unescaped control characters preceeded by a backslash. ### 2026-03-18 (2.19.2) diff --git a/lib/json/version.rb b/lib/json/version.rb index 8853ed885..e7af97d43 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.2' + VERSION = '2.19.3' end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index af08d1835..67d86a0b3 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -183,7 +183,7 @@ def test_parse_allowed_control_chars_in_string end end - def test_parsse_control_char_and_backslash + def test_parse_control_char_and_backslash backslash_and_control_char = "\\\t" assert_raise JSON::ParserError do JSON.parse(%("#{'a' * 30}#{backslash_and_control_char}"), allow_control_characters: true, allow_invalid_escape: false) From 2b085b5a7315b7e056cc64031fb9fa6be217ba2b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 25 Mar 2026 12:13:01 +0100 Subject: [PATCH 295/506] Remove outdated README-json-jruby.md --- README-json-jruby.md | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 README-json-jruby.md diff --git a/README-json-jruby.md b/README-json-jruby.md deleted file mode 100644 index a66ebd621..000000000 --- a/README-json-jruby.md +++ /dev/null @@ -1,32 +0,0 @@ -JSON-JRuby -========== - -JSON-JRuby is a port of Florian Frank's native -[`json` library](http://json.rubyforge.org/) to JRuby. - - -Development version -=================== - -The latest version is available from the -[Git repository](http://github.com/mernen/json-jruby/tree): - - git clone git://github.com/mernen/json-jruby.git - - -Compiling -========= - -You'll need JRuby version 1.2 or greater to build JSON-JRuby. -Its path must be set on the `jruby.dir` property of -`nbproject/project.properties` (defaults to `../jruby`). - -Additionally, you'll need [Ant](http://ant.apache.org/), and -[Ragel](http://www.colm.net/open-source/ragel/) 6.4 or greater. - -Then, from the folder where the sources are located, type: - - ant clean jar - -to clean any leftovers from previous builds and generate the `.jar` files. -To generate a RubyGem, specify the `gem` action rather than `jar`. From bfd63d435e6bf3e12d5ab6aec70f63f5d5a21137 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 25 Mar 2026 12:15:28 +0100 Subject: [PATCH 296/506] Remove some dead scripts and configs --- .all_images.yml | 17 ----------------- diagrams/.keep | 0 install.rb | 23 ----------------------- 3 files changed, 40 deletions(-) delete mode 100644 .all_images.yml delete mode 100644 diagrams/.keep delete mode 100755 install.rb diff --git a/.all_images.yml b/.all_images.yml deleted file mode 100644 index 9178beb50..000000000 --- a/.all_images.yml +++ /dev/null @@ -1,17 +0,0 @@ -dockerfile: |- - RUN apk add --no-cache build-base git ragel - RUN gem update --system - RUN gem install gem_hadar bundler -script: &script |- - echo -e "\e[1m" - ruby -v - echo -e "\e[0m" - bundle - rake clean test - -images: - ruby:3.1-alpine: *script - ruby:3.0-alpine: *script - ruby:2.7-alpine: *script - ruby:2.6-alpine: *script - ruby:2.5-alpine: *script diff --git a/diagrams/.keep b/diagrams/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/install.rb b/install.rb deleted file mode 100755 index 9d3ae12d8..000000000 --- a/install.rb +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env ruby - -require 'fileutils' -include FileUtils::Verbose -require 'rbconfig' -include\ - begin - RbConfig - rescue NameError - Config - end - -sitelibdir = CONFIG["sitelibdir"] -cd 'lib' do - install('json.rb', sitelibdir) - mkdir_p File.join(sitelibdir, 'json') - for file in Dir['json/**/*}'] - d = File.join(sitelibdir, file) - mkdir_p File.dirname(d) - install(file, d) - end -end -warn " *** Installed PURE ruby library." From c76b244029736d2e5cfe106a435b777bc281c17a Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Wed, 25 Mar 2026 17:44:19 -0700 Subject: [PATCH 297/506] Generate non-frozen strings on TruffleRuby for consistency --- lib/json/truffle_ruby/generator.rb | 10 +++++----- test/json/json_generator_test.rb | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 13b15459c..094592869 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -517,7 +517,7 @@ def json_transform(state) if empty? state.depth -= 1 - return '{}' + return +'{}' end delim = ",#{state.object_nl}" @@ -609,7 +609,7 @@ def json_transform(state) if empty? state.depth -= 1 - return '[]' + return +'[]' end result = '['.dup @@ -734,17 +734,17 @@ def to_json(state = nil, *args) module TrueClass # Returns a JSON string for true: 'true'. - def to_json(*) 'true' end + def to_json(*) +'true' end end module FalseClass # Returns a JSON string for false: 'false'. - def to_json(*) 'false' end + def to_json(*) +'false' end end module NilClass # Returns a JSON string for nil: 'null'. - def to_json(*) 'null' end + def to_json(*) +'null' end end end end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index eb5a8ca59..1e44117e8 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -84,6 +84,24 @@ def test_dump_strict assert_equal '"World"', "World".to_json(strict: true) end + def test_not_frozen + [ + [[], '[]'], + [{}, '{}'], + ["string", '"string"'], + [:sym, '"sym"'], + [1, '1'], + [1.0, '1.0'], + [true, 'true'], + [false, 'false'], + [nil, 'null'], + ].each do |(obj, exp)| + dumped = dump(obj, strict: true) + assert_equal exp, dumped + refute_predicate dumped, :frozen? + end + end + def test_state_depth_to_json depth = Object.new def depth.to_json(state) From e871d077f626e93d26b94e5ec890e16e6ddd3f55 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Wed, 25 Mar 2026 15:40:34 -0700 Subject: [PATCH 298/506] test_broken_bignum: avoid fork and subprocess for robustness The raw fork + Process.waitpid2 pattern causes a SEGV on i686-linux in ruby/ruby CI. The crash occurs in waitpid_blocking_no_SIGCHLD going through the 32-bit vdso, which is a Ruby VM / kernel interaction issue unrelated to the json gem itself. Replace the fork-based approach with an in-process test that temporarily patches Integer#to_s and restores it via ensure. This avoids the fork SEGV on i686, works under valgrind (assert_separately doesn't), and is guarded to only run on CRuby (not JRuby/TruffleRuby) where the C extension behavior being tested exists. --- test/json/json_generator_test.rb | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1e44117e8..245a547ab 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -493,25 +493,20 @@ def foo.to_h assert_equal '2', state.indent end - def test_broken_bignum # [ruby-core:38867] - pid = fork do - x = 1 << 64 - x.class.class_eval do + if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java" + def test_broken_bignum # [ruby-core:38867] + bignum = 1 << 64 + original_to_s = bignum.class.instance_method(:to_s) + bignum.class.class_eval do def to_s end end - begin - JSON::Ext::Generator::State.new.generate(x) - exit 1 - rescue TypeError - exit 0 + assert_raise(TypeError) do + JSON::Ext::Generator::State.new.generate(bignum) end + ensure + bignum.class.define_method(:to_s, original_to_s) if original_to_s end - _, status = Process.waitpid2(pid) - assert status.success? - rescue NotImplementedError - # forking to avoid modifying core class of a parent process and - # introducing race conditions of tests are run in parallel end def test_hash_likeness_set_symbol From d0b47b00118c648645ab2bdb3ec003a61f2b6082 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 26 Mar 2026 07:06:26 +0100 Subject: [PATCH 299/506] Avoid method redefinition warnings in test_broken_bignum --- lib/json/truffle_ruby/generator.rb | 6 ++++-- test/json/json_generator_test.rb | 27 +++++++++++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 094592869..019116454 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -413,7 +413,7 @@ def generate(obj, anIO = nil) buf << obj.to_json(self) end when Integer - buf << obj.to_s + buf << String(obj) when Symbol if @strict fast_serialize_string(obj.name, buf) @@ -652,7 +652,9 @@ def json_transform(state) module Integer # Returns a JSON string representation for this Integer number. - def to_json(*) to_s end + def to_json(*) + String(self) + end end module Float diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 245a547ab..00960dffe 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -493,20 +493,27 @@ def foo.to_h assert_equal '2', state.indent end - if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java" - def test_broken_bignum # [ruby-core:38867] - bignum = 1 << 64 - original_to_s = bignum.class.instance_method(:to_s) - bignum.class.class_eval do - def to_s - end + def test_broken_bignum # [Bug #5173] + bignum = 1 << 64 + bignum_to_s = bignum.to_s + + original_to_s = bignum.class.instance_method(:to_s) + bignum.class.class_eval do + def to_s + nil end + alias_method :to_s, :to_s + end + case RUBY_PLATFORM + when "java" + assert_equal bignum_to_s, JSON.generate(bignum) + else assert_raise(TypeError) do - JSON::Ext::Generator::State.new.generate(bignum) + JSON.generate(bignum) end - ensure - bignum.class.define_method(:to_s, original_to_s) if original_to_s end + ensure + bignum.class.define_method(:to_s, original_to_s) if original_to_s end def test_hash_likeness_set_symbol From 35db8591e9fe659287b7da73daa95029df0145f5 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Thu, 26 Mar 2026 11:29:15 +0100 Subject: [PATCH 300/506] Avoid extra String#+@ calls, interpolated strings are already mutable --- lib/json/truffle_ruby/generator.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 019116454..1c6d99f8f 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -521,7 +521,7 @@ def json_transform(state) end delim = ",#{state.object_nl}" - result = +"{#{state.object_nl}" + result = "{#{state.object_nl}" first = true key_type = nil indent = !state.object_nl.empty? @@ -558,7 +558,7 @@ def json_transform(state) raise TypeError, "#{key.class}#to_s returns an instance of #{key_str.class}, expected a String" end - result = +"#{result}#{key_json}#{state.space_before}:#{state.space}" + result = "#{result}#{key_json}#{state.space_before}:#{state.space}" if state.strict? && !Generator.native_type?(value) if state.as_json value = state.as_json.call(value, false) From ac0670b20b25b72ad82b30c8ce4c382db90437e5 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Thu, 26 Mar 2026 16:42:02 +0100 Subject: [PATCH 301/506] Keep Integer#to_json optimized and adapt the test * Redefining Integer#to_s is bound to break many things, any non-segfault outcome should be acceptable. So adapt the test to just ensure it does not segfault. * Also redefining Kernel#String would also break with the previous code. --- lib/json/truffle_ruby/generator.rb | 6 ++---- test/json/json_generator_test.rb | 10 +++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 1c6d99f8f..aa281eed3 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -413,7 +413,7 @@ def generate(obj, anIO = nil) buf << obj.to_json(self) end when Integer - buf << String(obj) + buf << obj.to_s when Symbol if @strict fast_serialize_string(obj.name, buf) @@ -652,9 +652,7 @@ def json_transform(state) module Integer # Returns a JSON string representation for this Integer number. - def to_json(*) - String(self) - end + def to_json(*) to_s end end module Float diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 00960dffe..0eb85a826 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -504,10 +504,14 @@ def to_s end alias_method :to_s, :to_s end - case RUBY_PLATFORM - when "java" + case RUBY_ENGINE + when "jruby" assert_equal bignum_to_s, JSON.generate(bignum) - else + when "truffleruby" + assert_raise(NoMethodError) do + JSON.generate(bignum) + end + when "ruby" assert_raise(TypeError) do JSON.generate(bignum) end From 0e99fcb581e2f6639a04012e6e6e3dbfcaf02d94 Mon Sep 17 00:00:00 2001 From: Randy Stauner Date: Thu, 26 Mar 2026 12:56:54 -0700 Subject: [PATCH 302/506] Fix json generation for symbols on TruffleRuby --- lib/json/truffle_ruby/generator.rb | 2 +- test/json/json_coder_test.rb | 5 +++++ test/json/json_generator_test.rb | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index aa281eed3..f03ff7fd8 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -48,7 +48,7 @@ module Generator SCRIPT_SAFE_ESCAPE_PATTERN = /[\/"\\\x0-\x1f\u2028-\u2029]/ def self.native_type?(value) # :nodoc: - (false == value || true == value || nil == value || String === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) + (false == value || true == value || nil == value || String === value || Symbol === value || Array === value || Hash === value || Integer === value || Float === value || Fragment === value) end def self.native_key?(key) # :nodoc: diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 47e12ff91..a8477dd7b 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -19,6 +19,11 @@ def test_json_coder_with_proc_with_unsupported_value assert_raise(JSON::GeneratorError) { coder.dump([Object.new]) } end + def test_json_coder_with_proc_returning_symbol + coder = JSON::Coder.new { _1 } + assert_equal %({"sym":"sym"}), coder.dump({ sym: :sym }) + end + def test_json_coder_hash_key obj = Object.new coder = JSON::Coder.new do |obj, is_key| diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 0eb85a826..1cfd19da1 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -82,6 +82,8 @@ def test_dump_strict assert_equal '"hello"', dump(:hello, strict: true) assert_equal '"hello"', :hello.to_json(strict: true) assert_equal '"World"', "World".to_json(strict: true) + assert_equal '["hello"]', dump([:hello], strict: true) + assert_equal '{"hello":"world"}', dump({ hello: :world }, strict: true) end def test_not_frozen From 20454ba2746de820669c36d1f8552cd67bdd23b6 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 15 Apr 2026 08:09:56 +0200 Subject: [PATCH 303/506] Fix handling out of of range exponent in numbers Fix: https://github.com/ruby/json/issues/970 If the parsed exponent overflows a `int32_t` passing it to ryu is incorrect. We could pass it to `rb_cstr_to_dbl` but then Ruby will emit an annoying warning, instead we can coerce to `0.0` and `Inf`. --- CHANGES.md | 2 ++ ext/json/ext/parser/parser.c | 16 ++++++++++++---- test/json/json_ryu_fallback_test.rb | 8 ++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7c295099e..6cdb43fd5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix parsing of out of range floats (very large exponents that lead ot either `0.0` or `Inf`). + ### 2026-03-25 (2.19.3) * Fix handling of unescaped control characters preceeded by a backslash. diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 38f3a48ce..976fbbe75 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -855,7 +855,7 @@ NOINLINE(static) VALUE json_decode_large_float(const char *start, long len) /* Ruby JSON optimized float decoder using vendored Ryu algorithm * Accepts pre-extracted mantissa and exponent from first-pass validation */ -static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int32_t exponent, bool negative, +static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantissa, int mantissa_digits, int64_t exponent, bool negative, const char *start, const char *end) { if (RB_UNLIKELY(config->decimal_class)) { @@ -863,13 +863,21 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text); } + if (RB_UNLIKELY(exponent > INT32_MAX)) { + return CInfinity; + } + + if (RB_UNLIKELY(exponent < INT32_MIN)) { + return rb_float_new(0.0); + } + // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case) // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308) if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) { return json_decode_large_float(start, end - start); } - return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, exponent, negative)); + return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, (int32_t)exponent, negative)); } static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count) @@ -1144,7 +1152,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig const char first_digit = *state->cursor; // Variables for Ryu optimization - extract digits during parsing - int32_t exponent = 0; + int64_t exponent = 0; int decimal_point_pos = -1; uint64_t mantissa = 0; @@ -1188,7 +1196,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig raise_parse_error_at("invalid number: %s", state, start); } - exponent = negative_exponent ? -((int32_t)abs_exponent) : ((int32_t)abs_exponent); + exponent = negative_exponent ? -abs_exponent : abs_exponent; } if (integer) { diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_ryu_fallback_test.rb index 59ba76d39..dc60067b9 100644 --- a/test/json/json_ryu_fallback_test.rb +++ b/test/json/json_ryu_fallback_test.rb @@ -166,4 +166,12 @@ def test_invalid_numbers_rejected end end end + + def test_large_exponent_numbers + assert_equal Float::INFINITY, JSON.parse("1e4294967296") + assert_equal 0.0, JSON.parse("1e-4294967296") + assert_equal 0.0, JSON.parse("99999999999999999e-4294967296") + assert_equal Float::INFINITY, JSON.parse("1e4294967295") + assert_equal Float::INFINITY, JSON.parse("1e4294967297") + end end From 1072482184de233f55c45f875b38766b67b84f3b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 15 Apr 2026 08:37:32 +0200 Subject: [PATCH 304/506] Fix parsing of *negative* out of bound floats. --- ext/json/ext/parser/parser.c | 4 ++-- test/json/json_ryu_fallback_test.rb | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 976fbbe75..f89a1917c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -864,11 +864,11 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis } if (RB_UNLIKELY(exponent > INT32_MAX)) { - return CInfinity; + return negative ? CMinusInfinity : CInfinity; } if (RB_UNLIKELY(exponent < INT32_MIN)) { - return rb_float_new(0.0); + return rb_float_new(negative ? -0.0 : 0.0); } // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case) diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_ryu_fallback_test.rb index dc60067b9..2262a6f59 100644 --- a/test/json/json_ryu_fallback_test.rb +++ b/test/json/json_ryu_fallback_test.rb @@ -173,5 +173,11 @@ def test_large_exponent_numbers assert_equal 0.0, JSON.parse("99999999999999999e-4294967296") assert_equal Float::INFINITY, JSON.parse("1e4294967295") assert_equal Float::INFINITY, JSON.parse("1e4294967297") + + assert_equal -Float::INFINITY, JSON.parse("-1e4294967296") + assert_equal -0.0, JSON.parse("-1e-4294967296") + assert_equal -0.0, JSON.parse("-99999999999999999e-4294967296") + assert_equal -Float::INFINITY, JSON.parse("-1e4294967295") + assert_equal -Float::INFINITY, JSON.parse("-1e4294967297") end end From 18d54757d38fce2eea6cb821458b1df4405a256e Mon Sep 17 00:00:00 2001 From: Kazuhiro NISHIYAMA Date: Thu, 16 Apr 2026 11:22:36 +0900 Subject: [PATCH 305/506] Reduce warnings ``` ../test/json/json_ryu_fallback_test.rb:177: warning: ambiguous first argument; put parentheses or a space even after `-` operator ../test/json/json_ryu_fallback_test.rb:178: warning: ambiguous first argument; put parentheses or a space even after `-` operator ../test/json/json_ryu_fallback_test.rb:179: warning: ambiguous first argument; put parentheses or a space even after `-` operator ../test/json/json_ryu_fallback_test.rb:180: warning: ambiguous first argument; put parentheses or a space even after `-` operator ../test/json/json_ryu_fallback_test.rb:181: warning: ambiguous first argument; put parentheses or a space even after `-` operator ``` --- test/json/json_ryu_fallback_test.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_ryu_fallback_test.rb index 2262a6f59..152de7e36 100644 --- a/test/json/json_ryu_fallback_test.rb +++ b/test/json/json_ryu_fallback_test.rb @@ -174,10 +174,10 @@ def test_large_exponent_numbers assert_equal Float::INFINITY, JSON.parse("1e4294967295") assert_equal Float::INFINITY, JSON.parse("1e4294967297") - assert_equal -Float::INFINITY, JSON.parse("-1e4294967296") - assert_equal -0.0, JSON.parse("-1e-4294967296") - assert_equal -0.0, JSON.parse("-99999999999999999e-4294967296") - assert_equal -Float::INFINITY, JSON.parse("-1e4294967295") - assert_equal -Float::INFINITY, JSON.parse("-1e4294967297") + assert_equal(-Float::INFINITY, JSON.parse("-1e4294967296")) + assert_equal(-0.0, JSON.parse("-1e-4294967296")) + assert_equal(-0.0, JSON.parse("-99999999999999999e-4294967296")) + assert_equal(-Float::INFINITY, JSON.parse("-1e4294967295")) + assert_equal(-Float::INFINITY, JSON.parse("-1e4294967297")) end end From f1e6163ee1affbc9a80903158b92712228601316 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Sat, 18 Apr 2026 02:03:40 -0700 Subject: [PATCH 306/506] Fix references to NAN and INFINITY in documentation comments --- lib/json.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json.rb b/lib/json.rb index f28be7f77..26d601926 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -335,8 +335,8 @@ # JSON.generate(JSON::MinusInfinity) # # Allow: -# ruby = [Float::NaN, Float::Infinity, Float::MinusInfinity] -# JSON.generate(ruby, allow_nan: true) # => '[NaN,Infinity,-Infinity]' +# ruby = [Float::NAN, Float::INFINITY, JSON::NaN, JSON::Infinity, JSON::MinusInfinity] +# JSON.generate(ruby, allow_nan: true) # => '[NaN,Infinity,NaN,Infinity,-Infinity]' # # --- # From 6688a814bf8b3d718701ac4cf328f7c42be4792b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 19 Apr 2026 06:06:06 +0900 Subject: [PATCH 307/506] Release 2.19.4 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 6cdb43fd5..edb172a52 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-04-19 (2.19.4) + * Fix parsing of out of range floats (very large exponents that lead ot either `0.0` or `Inf`). ### 2026-03-25 (2.19.3) diff --git a/lib/json/version.rb b/lib/json/version.rb index e7af97d43..009a2d24b 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.3' + VERSION = '2.19.4' end From 7dd6b63d50167788ee96cf3f05b7aa30d07ec8be Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 19 Apr 2026 06:09:40 +0900 Subject: [PATCH 308/506] Fix typo in changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index edb172a52..d96209008 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ ### 2026-04-19 (2.19.4) -* Fix parsing of out of range floats (very large exponents that lead ot either `0.0` or `Inf`). +* Fix parsing of out of range floats (very large exponents that lead to either `0.0` or `Inf`). ### 2026-03-25 (2.19.3) From 4ef7a456545bffd1bafa6b9014b4ce9fbffc8c0e Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 17 Apr 2026 21:10:37 -0500 Subject: [PATCH 309/506] Use RB_ENC_CODERANGE to first check the cached coderange before calling rb_enc_str_coderange if the coderange is unknown. --- ext/json/ext/generator/generator.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index b52bc9751..04cc6e7b8 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -819,9 +819,17 @@ static VALUE encode_json_string_rescue(VALUE str, VALUE exception) return Qundef; } +static inline int json_str_coderange(VALUE str) { + int coderange = RB_ENC_CODERANGE(str); + if (coderange == RUBY_ENC_CODERANGE_UNKNOWN) { + coderange = rb_enc_str_coderange(str); + } + return coderange; +} + static inline bool valid_json_string_p(VALUE str) { - int coderange = rb_enc_str_coderange(str); + int coderange = json_str_coderange(str); if (RB_LIKELY(coderange == ENC_CODERANGE_7BIT)) { return true; @@ -893,7 +901,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data search.chunk_end = NULL; #endif /* HAVE_SIMD */ - switch (rb_enc_str_coderange(obj)) { + switch (json_str_coderange(obj)) { case ENC_CODERANGE_7BIT: case ENC_CODERANGE_VALID: if (RB_UNLIKELY(data->state->ascii_only)) { From cfbe356b4f9ecef22d5c934628e3ec9b227796f6 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Sun, 19 Apr 2026 22:49:05 -0500 Subject: [PATCH 310/506] Force ensure_valid_encoding to be inlined. And move the encoding convertion logic in another function with NOINLINE. The overwelming majority of strings are correctly encoded, so we want to inline the very cheap check, however we don't want to inline the much larger piece of code required to re-encode the string. Co-Authored-By: Jean Boussier --- ext/json/ext/generator/generator.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 04cc6e7b8..882d47a61 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -842,12 +842,8 @@ static inline bool valid_json_string_p(VALUE str) return false; } -static inline VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) +NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) { - if (RB_LIKELY(valid_json_string_p(str))) { - return str; - } - if (!as_json_called && data->state->strict && RTEST(data->state->as_json)) { VALUE coerced_str = json_call_as_json(data->state, str, Qfalse); if (coerced_str != str) { @@ -883,6 +879,16 @@ static inline VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); } +ALWAYS_INLINE(static) VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) +{ + if (RB_LIKELY(valid_json_string_p(str))) { + return str; + } + else { + return convert_invalid_encoding(data, str, as_json_called, is_key); + } +} + static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append_char(buffer, '"'); From fa0671ccee70fd6653057df75a97b1ea3334ea87 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Apr 2026 00:03:27 +0200 Subject: [PATCH 311/506] Test TruffleRuby release in CI for improved stability --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38f9494bb..adeedc5a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,8 @@ jobs: - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI - { os: macos-latest , ruby: jruby-9.4 } # Ruby 3.1 - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 - - { os: macos-latest , ruby: truffleruby-head } - - { os: ubuntu-latest , ruby: truffleruby-head } + - { os: macos-latest , ruby: truffleruby } + - { os: ubuntu-latest , ruby: truffleruby } exclude: - { os: windows-latest, ruby: jruby } From f6ca597cb1dca96d68206c60bd1a1fd1c954fecd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 1 May 2026 06:57:40 +0200 Subject: [PATCH 312/506] Avoid spamming too many deprecations while parsing --- ext/json/ext/parser/parser.c | 8 +- java/src/json/ext/ParserConfig.java | 150 ++++++++++++++-------------- java/src/json/ext/ParserConfig.rl | 4 +- 3 files changed, 86 insertions(+), 76 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index f89a1917c..96b0d3045 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -364,6 +364,7 @@ typedef struct JSON_ParserStateStruct { rvalue_cache name_cache; int in_array; int current_nesting; + unsigned int emitted_deprecations; } JSON_ParserState; static inline size_t rest(JSON_ParserState *state) { @@ -945,7 +946,12 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi case JSON_IGNORE: break; case JSON_DEPRECATED: - emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); + // Only emit the first few deprecations to avoid spamming. + if (state->emitted_deprecations < 5) { + emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); + state->emitted_deprecations++; + } + break; case JSON_RAISE: raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index dfc1a6c2c..1fee0a596 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -285,6 +285,7 @@ private static class ParserSession { private final byte[] data; private final StringDecoder decoder; private int currentNesting = 0; + private int emittedDeprecations = 0; private ParserSession(ParserConfig config, RubyString source, ThreadContext context, RuntimeInfo info) { this.config = config; @@ -307,11 +308,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 333 "ParserConfig.rl" +// line 334 "ParserConfig.rl" -// line 315 "ParserConfig.java" +// line 316 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -425,7 +426,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 439 "ParserConfig.rl" +// line 440 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -433,14 +434,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 437 "ParserConfig.java" +// line 438 "ParserConfig.java" { cs = JSON_value_start; } -// line 446 "ParserConfig.rl" +// line 447 "ParserConfig.rl" -// line 444 "ParserConfig.java" +// line 445 "ParserConfig.java" { int _klen; int _trans = 0; @@ -466,13 +467,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 424 "ParserConfig.rl" +// line 425 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 476 "ParserConfig.java" +// line 477 "ParserConfig.java" } } @@ -535,25 +536,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 341 "ParserConfig.rl" +// line 342 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 344 "ParserConfig.rl" +// line 345 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 347 "ParserConfig.rl" +// line 348 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 350 "ParserConfig.rl" +// line 351 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -563,7 +564,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 357 "ParserConfig.rl" +// line 358 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -573,7 +574,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 364 "ParserConfig.rl" +// line 365 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -602,7 +603,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 390 "ParserConfig.rl" +// line 391 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -615,7 +616,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 400 "ParserConfig.rl" +// line 401 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -630,7 +631,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 412 "ParserConfig.rl" +// line 413 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -644,7 +645,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 648 "ParserConfig.java" +// line 649 "ParserConfig.java" } } } @@ -664,7 +665,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 447 "ParserConfig.rl" +// line 448 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -677,7 +678,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 681 "ParserConfig.java" +// line 682 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -776,7 +777,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 469 "ParserConfig.rl" +// line 470 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -793,15 +794,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 797 "ParserConfig.java" +// line 798 "ParserConfig.java" { cs = JSON_integer_start; } -// line 485 "ParserConfig.rl" +// line 486 "ParserConfig.rl" int memo = p; -// line 805 "ParserConfig.java" +// line 806 "ParserConfig.java" { int _klen; int _trans = 0; @@ -882,13 +883,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 463 "ParserConfig.rl" +// line 464 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 892 "ParserConfig.java" +// line 893 "ParserConfig.java" } } } @@ -908,7 +909,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 487 "ParserConfig.rl" +// line 488 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -928,7 +929,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 932 "ParserConfig.java" +// line 933 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1030,7 +1031,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 520 "ParserConfig.rl" +// line 521 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1049,15 +1050,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1053 "ParserConfig.java" +// line 1054 "ParserConfig.java" { cs = JSON_float_start; } -// line 538 "ParserConfig.rl" +// line 539 "ParserConfig.rl" int memo = p; -// line 1061 "ParserConfig.java" +// line 1062 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1138,13 +1139,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 511 "ParserConfig.rl" +// line 512 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1148 "ParserConfig.java" +// line 1149 "ParserConfig.java" } } } @@ -1164,7 +1165,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 540 "ParserConfig.rl" +// line 541 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1174,7 +1175,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1178 "ParserConfig.java" +// line 1179 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1276,7 +1277,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 579 "ParserConfig.rl" +// line 580 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1284,15 +1285,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1288 "ParserConfig.java" +// line 1289 "ParserConfig.java" { cs = JSON_string_start; } -// line 586 "ParserConfig.rl" +// line 587 "ParserConfig.rl" int memo = p; -// line 1296 "ParserConfig.java" +// line 1297 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1373,7 +1374,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 554 "ParserConfig.rl" +// line 555 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1388,13 +1389,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 567 "ParserConfig.rl" +// line 568 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1398 "ParserConfig.java" +// line 1399 "ParserConfig.java" } } } @@ -1414,7 +1415,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 588 "ParserConfig.rl" +// line 589 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1435,7 +1436,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1439 "ParserConfig.java" +// line 1440 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { @@ -1602,7 +1603,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 642 "ParserConfig.rl" +// line 643 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1616,14 +1617,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); -// line 1620 "ParserConfig.java" +// line 1621 "ParserConfig.java" { cs = JSON_array_start; } -// line 655 "ParserConfig.rl" +// line 656 "ParserConfig.rl" -// line 1627 "ParserConfig.java" +// line 1628 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1666,7 +1667,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 613 "ParserConfig.rl" +// line 614 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1736,7 +1737,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 615 "ParserConfig.rl" +// line 616 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1749,13 +1750,13 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } break; case 1: -// line 626 "ParserConfig.rl" +// line 627 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1759 "ParserConfig.java" +// line 1760 "ParserConfig.java" } } } @@ -1775,7 +1776,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 656 "ParserConfig.rl" +// line 657 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1785,7 +1786,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1789 "ParserConfig.java" +// line 1790 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { @@ -1962,7 +1963,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 726 "ParserConfig.rl" +// line 728 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -1979,14 +1980,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyHash.newHash(context.runtime); -// line 1983 "ParserConfig.java" +// line 1984 "ParserConfig.java" { cs = JSON_object_start; } -// line 742 "ParserConfig.rl" +// line 744 "ParserConfig.rl" -// line 1990 "ParserConfig.java" +// line 1991 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2029,7 +2030,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 670 "ParserConfig.rl" +// line 671 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2099,7 +2100,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 672 "ParserConfig.rl" +// line 673 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2112,7 +2113,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 1: -// line 683 "ParserConfig.rl" +// line 684 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2128,7 +2129,8 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) if (!config.allowDuplicateKey) { if (((RubyHash)result).hasKey(lastName)) { - if (config.deprecateDuplicateKey) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; context.runtime.getWarnings().warning( "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" ); @@ -2143,13 +2145,13 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } break; case 2: -// line 712 "ParserConfig.rl" +// line 714 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2153 "ParserConfig.java" +// line 2155 "ParserConfig.java" } } } @@ -2169,7 +2171,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 743 "ParserConfig.rl" +// line 745 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2180,7 +2182,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2184 "ParserConfig.java" +// line 2186 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { @@ -2283,7 +2285,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 772 "ParserConfig.rl" +// line 774 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2293,16 +2295,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2297 "ParserConfig.java" +// line 2299 "ParserConfig.java" { cs = JSON_start; } -// line 781 "ParserConfig.rl" +// line 783 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2306 "ParserConfig.java" +// line 2308 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2383,7 +2385,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 758 "ParserConfig.rl" +// line 760 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2395,7 +2397,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2399 "ParserConfig.java" +// line 2401 "ParserConfig.java" } } } @@ -2415,7 +2417,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 784 "ParserConfig.rl" +// line 786 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 53678ec97..2312823a6 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -283,6 +283,7 @@ public class ParserConfig extends RubyObject { private final byte[] data; private final StringDecoder decoder; private int currentNesting = 0; + private int emittedDeprecations = 0; private ParserSession(ParserConfig config, RubyString source, ThreadContext context, RuntimeInfo info) { this.config = config; @@ -695,7 +696,8 @@ public class ParserConfig extends RubyObject { if (!config.allowDuplicateKey) { if (((RubyHash)result).hasKey(lastName)) { - if (config.deprecateDuplicateKey) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; context.runtime.getWarnings().warning( "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" ); From 4a1a4a4861255014b125f3c0c52b968ee7f5bcfe Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 4 May 2026 06:49:14 +0200 Subject: [PATCH 313/506] Release 2.19.5 --- CHANGES.md | 4 ++++ lib/json/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d96209008..e4fe73780 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ### Unreleased +### 2026-05-04 (2.19.5) + +* Cap the parser to emit a maximum of 5 deprecation warnings per document. Emitting more is not helpful. + ### 2026-04-19 (2.19.4) * Fix parsing of out of range floats (very large exponents that lead to either `0.0` or `Inf`). diff --git a/lib/json/version.rb b/lib/json/version.rb index 009a2d24b..ebb55656d 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.4' + VERSION = '2.19.5' end From ab6972d797a17487a16f30195fb0e745858a54a2 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 12 May 2026 08:19:06 +0200 Subject: [PATCH 314/506] Add missing write barrier in ParserConfig This couldn't possibly lead to a crash unless you'd mess with GC.stress or `send(:initialize)` on an old object. But for correctness, it should be there. --- ext/json/ext/parser/parser.c | 40 +++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 96b0d3045..a740bd42e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1471,9 +1471,22 @@ static VALUE convert_encoding(VALUE source) return rb_funcall(source, i_encode, 1, Encoding_UTF_8); } +struct parser_config_init_args { + JSON_ParserConfig *config; + VALUE self; +}; + +static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val) +{ + *dest = val; + if (self) RB_OBJ_WRITTEN(self, Qundef, val); +} + static int parser_config_init_i(VALUE key, VALUE val, VALUE data) { - JSON_ParserConfig *config = (JSON_ParserConfig *)data; + struct parser_config_init_args *args = (struct parser_config_init_args *)data; + JSON_ParserConfig *config = args->config; + VALUE self = args->self; if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; } else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } @@ -1482,15 +1495,15 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } - else if (key == sym_on_load) { config->on_load_proc = RTEST(val) ? val : Qfalse; } + else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); } else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { - config->decimal_class = val; + parser_config_wb_write(self, &config->decimal_class, val); config->decimal_method_id = i_try_convert; } else if (rb_respond_to(val, i_new)) { - config->decimal_class = val; + parser_config_wb_write(self, &config->decimal_class, val); config->decimal_method_id = i_new; } else if (RB_TYPE_P(val, T_CLASS)) { VALUE name = rb_class_name(val); @@ -1499,7 +1512,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) if (last_colon) { const char *mod_path_end = last_colon - 1; VALUE mod_path = rb_str_substr(name, 0, mod_path_end - name_cstr); - config->decimal_class = rb_path_to_class(mod_path); + parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path)); const char *method_name_beg = last_colon + 1; long before_len = method_name_beg - name_cstr; @@ -1507,7 +1520,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) VALUE method_name = rb_str_substr(name, before_len, len); config->decimal_method_id = SYM2ID(rb_str_intern(method_name)); } else { - config->decimal_class = rb_mKernel; + parser_config_wb_write(self, &config->decimal_class, rb_mKernel); config->decimal_method_id = SYM2ID(rb_str_intern(name)); } } @@ -1517,16 +1530,21 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) return ST_CONTINUE; } -static void parser_config_init(JSON_ParserConfig *config, VALUE opts) +static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self) { config->max_nesting = 100; + struct parser_config_init_args args = { + .config = config, + .self = self, + }; + if (!NIL_P(opts)) { Check_Type(opts, T_HASH); if (RHASH_SIZE(opts) > 0) { // We assume in most cases few keys are set so it's faster to go over // the provided keys than to check all possible keys. - rb_hash_foreach(opts, parser_config_init_i, (VALUE)config); + rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args); } } @@ -1560,9 +1578,7 @@ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) rb_check_frozen(self); GET_PARSER_CONFIG; - parser_config_init(config, opts); - - RB_OBJ_WRITTEN(self, Qundef, config->decimal_class); + parser_config_init(config, opts, self); return self; } @@ -1624,7 +1640,7 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts) JSON_ParserConfig _config = {0}; JSON_ParserConfig *config = &_config; - parser_config_init(config, opts); + parser_config_init(config, opts, false); return cParser_parse(config, Vsource); } From d644602ac8efff874c7b36ac0c21a19790692988 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 12 May 2026 08:49:34 +0200 Subject: [PATCH 315/506] generator.c: Handle stupidly large depth Reported-by: Lexterl33t --- CHANGES.md | 2 ++ ext/json/ext/fbuffer/fbuffer.h | 11 ++++++++++- test/json/json_generator_test.rb | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e4fe73780..f6a75ba0b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Cleanly handle overly large `depth` generator argument. + ### 2026-05-04 (2.19.5) * Cap the parser to emit a maximum of 5 deprecation warnings per document. Emitting more is not helpful. diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index 9660e70dc..b84a07355 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -131,6 +131,15 @@ static inline void fbuffer_inc_capa(FBuffer *fb, size_t requested) } } +static inline size_t fbuffer_size_mul_or_raise(size_t a, size_t b) +{ + size_t result = a * b; + if (RB_UNLIKELY(a != 0 && (result / a) != b)) { + rb_raise(rb_eArgError, "Buffer overflow, the resulting document is too large to be generated"); + } + return result; +} + static inline void fbuffer_append_reserved(FBuffer *fb, const char *newstr, size_t len) { MEMCPY(fb->ptr + fb->len, newstr, char, len); @@ -175,7 +184,7 @@ static void fbuffer_append_str_repeat(FBuffer *fb, VALUE str, size_t repeat) size_t len; RSTRING_GETMEM(str, ptr, len); - fbuffer_inc_capa(fb, repeat * len); + fbuffer_inc_capa(fb, fbuffer_size_mul_or_raise(repeat, len)); while (repeat) { #if JSON_DEBUG fb->requested = len; diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1cfd19da1..87d9cd7f7 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1081,4 +1081,15 @@ def test_negative_depth_raises end end + def test_large_depth_raises + assert_raise(RangeError, ArgumentError) do + JSON.generate([[1]], + indent: " " * 5, + array_nl: "\n", + depth: 3_689_348_814_741_910_324, + max_nesting: 0 + ) + end + end + end From 96966221b8c5df24454ba6030a95c6e1c37f86d7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 12 May 2026 09:06:44 +0200 Subject: [PATCH 316/506] Add ruby-asan to CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adeedc5a9..c4bc00fa8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - "" include: - { os: ubuntu-latest , ruby: debug } + - { os: ubuntu-latest , ruby: asan } - { os: ubuntu-24.04-arm, ruby: 3.4 } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } From 1e276ebb793437065ce741113a3eadc1a52847c0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 09:00:46 +0200 Subject: [PATCH 317/506] Release 2.19.6 --- CHANGES.md | 3 +++ lib/json/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f6a75ba0b..0bb996969 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,10 @@ ### Unreleased +### 2026-05-28 (2.19.6) + * Cleanly handle overly large `depth` generator argument. +* Add missing write barrier in `ParserConfig`. ### 2026-05-04 (2.19.5) diff --git a/lib/json/version.rb b/lib/json/version.rb index ebb55656d..7ea306656 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.5' + VERSION = '2.19.6' end From e8800cb0c58cf8ecda0943f94cbf63606cdca13b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 09:41:37 +0200 Subject: [PATCH 318/506] Further validate generator `depth` config Co-Authored-By: Yuhang Wu --- ext/json/ext/generator/generator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 882d47a61..b2082a7ca 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1590,6 +1590,9 @@ static long depth_config(VALUE num) if (RB_UNLIKELY(d < 0)) { rb_raise(rb_eArgError, "depth must be >= 0 (got %ld)", d); } + if (RB_UNLIKELY(d > INT_MAX)) { + rb_raise(rb_eArgError, "depth is too large (got %ld)", d); + } return d; } From dba1d885eae95ad2ca2b1986192ffd4397196525 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 09:53:19 +0200 Subject: [PATCH 319/506] generator.c: trigger write barriers in cState_init_copy Reported-By: Yuhang Wu --- ext/json/ext/generator/generator.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index b2082a7ca..9260712c9 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1367,12 +1367,14 @@ static VALUE cState_init_copy(VALUE obj, VALUE orig) if (!objState) rb_raise(rb_eArgError, "unallocated JSON::State"); MEMCPY(objState, origState, JSON_Generator_State, 1); - objState->indent = origState->indent; - objState->space = origState->space; - objState->space_before = origState->space_before; - objState->object_nl = origState->object_nl; - objState->array_nl = origState->array_nl; - objState->as_json = origState->as_json; + + RB_OBJ_WRITTEN(obj, Qundef, objState->indent); + RB_OBJ_WRITTEN(obj, Qundef, objState->space); + RB_OBJ_WRITTEN(obj, Qundef, objState->space_before); + RB_OBJ_WRITTEN(obj, Qundef, objState->object_nl); + RB_OBJ_WRITTEN(obj, Qundef, objState->array_nl); + RB_OBJ_WRITTEN(obj, Qundef, objState->as_json); + return obj; } From 5ca8a67f52be73f68b7cd3b1f62809e3118c9d36 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 10:22:50 +0200 Subject: [PATCH 320/506] parser.c: Ensure the user provided string can't be mutated Reported-By: Yuhang Wu --- ext/json/ext/parser/parser.c | 38 +++++++++++++++++++++-------------- test/json/json_parser_test.rb | 16 +++++++++++++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a740bd42e..fc16f727f 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1457,18 +1457,21 @@ static void json_ensure_eof(JSON_ParserState *state) static VALUE convert_encoding(VALUE source) { - int encindex = RB_ENCODING_GET(source); + StringValue(source); + int encindex = RB_ENCODING_GET(source); - if (RB_LIKELY(encindex == utf8_encindex)) { - return source; - } + if (RB_LIKELY(encindex == utf8_encindex)) { + return source; + } - if (encindex == binary_encindex) { - // For historical reason, we silently reinterpret binary strings as UTF-8 - return rb_enc_associate_index(rb_str_dup(source), utf8_encindex); - } + if (encindex == binary_encindex) { + // For historical reason, we silently reinterpret binary strings as UTF-8 + return rb_enc_associate_index(rb_str_dup(source), utf8_encindex); + } - return rb_funcall(source, i_encode, 1, Encoding_UTF_8); + source = rb_funcall(source, i_encode, 1, Encoding_UTF_8); + StringValue(source); + return source; } struct parser_config_init_args { @@ -1583,10 +1586,16 @@ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) return self; } -static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) +static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) { - Vsource = convert_encoding(StringValue(Vsource)); - StringValue(Vsource); + VALUE Vsource = convert_encoding(src); + + // Ensure the string isn't mutated under us. + // The classic API to use is `rb_str_locktmp`, but then we'd + // need to use `rb_protect` to make sure we always unlock. + if (Vsource == src) { + Vsource = rb_str_new_frozen(Vsource); + } VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA]; rvalue_stack stack = { @@ -1597,6 +1606,7 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) long len; const char *start; + RSTRING_GETMEM(Vsource, start, len); VALUE stack_handle = 0; @@ -1615,6 +1625,7 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE Vsource) // it won't cause a leak. rvalue_stack_eagerly_release(stack_handle); RB_GC_GUARD(stack_handle); + RB_GC_GUARD(Vsource); json_ensure_eof(state); return result; @@ -1635,9 +1646,6 @@ static VALUE cParserConfig_parse(VALUE self, VALUE Vsource) static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts) { - Vsource = convert_encoding(StringValue(Vsource)); - StringValue(Vsource); - JSON_ParserConfig _config = {0}; JSON_ParserConfig *config = &_config; parser_config_init(config, opts, false); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 67d86a0b3..1969e79b3 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -878,6 +878,22 @@ def test_frozen end end + def test_mutating_source_string_during_parsing + expected = ([1] * 100) + [2.3] + ([1] * 100) + source = JSON.generate(expected) + expected.delete_at(100) + + fake_decimal_class = Class.new + fake_decimal_class.define_method(:initialize) do |number| + source.tr!('1', '0') + number.to_f + end + + actual = JSON.parse(source, decimal_class: fake_decimal_class) + actual.delete_at(100) + assert_equal expected, actual + end + private def assert_equal_float(expected, actual, delta = 1e-2) From f033b9d3421c450108913d724810938e2d055e84 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 11:09:28 +0200 Subject: [PATCH 321/506] Fix some more edge cases with out of range floats Co-Authored-By: Yuhang Wu --- ext/json/ext/parser/parser.c | 6 +++++- test/json/json_ryu_fallback_test.rb | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index fc16f727f..363f109d3 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1202,7 +1202,11 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig raise_parse_error_at("invalid number: %s", state, start); } - exponent = negative_exponent ? -abs_exponent : abs_exponent; + if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) { + exponent = negative_exponent ? INT64_MIN : INT64_MAX; + } else { + exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent; + } } if (integer) { diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_ryu_fallback_test.rb index 152de7e36..a61b3e668 100644 --- a/test/json/json_ryu_fallback_test.rb +++ b/test/json/json_ryu_fallback_test.rb @@ -179,5 +179,13 @@ def test_large_exponent_numbers assert_equal(-0.0, JSON.parse("-99999999999999999e-4294967296")) assert_equal(-Float::INFINITY, JSON.parse("-1e4294967295")) assert_equal(-Float::INFINITY, JSON.parse("-1e4294967297")) + + assert_equal(Float::INFINITY, JSON.parse("1e9223372036854775808")) + assert_equal(Float::INFINITY, JSON.parse("1e9999999999999999999")) + assert_equal(Float::INFINITY, JSON.parse("1e18446744073709551616")) + assert_equal(Float::INFINITY, JSON.parse("1e10000000000000000000")) + assert_equal(Float::INFINITY, JSON.parse("1e184467440737095516160")) + assert_equal 0.0, JSON.parse("1e-18446744073709551615") + assert_equal 0.0, JSON.parse("1e-9223372036854775809") end end From ab6c8f2cdbc9cfa6079f5d0679afbc407a227c6d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 12:03:18 +0200 Subject: [PATCH 322/506] Release 2.19.7 --- CHANGES.md | 7 +++++++ lib/json/version.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0bb996969..73fc7af11 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,13 @@ ### Unreleased +### 2026-05-28 (2.19.7) + +* Fix some more edge cases with out of range floats. +* Ensure the string provided to `JSON.parse` can't be mutated during parsing. +* Add missing write barriers in `State#dup`. +* Further validate generator `depth` config. + ### 2026-05-28 (2.19.6) * Cleanly handle overly large `depth` generator argument. diff --git a/lib/json/version.rb b/lib/json/version.rb index 7ea306656..a69590ff9 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.6' + VERSION = '2.19.7' end From c7a7b2be6f20e52439f4fdc5263e9b539fc6ad6c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 28 May 2026 14:01:46 +0200 Subject: [PATCH 323/506] Add a security note in README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index fee99305e..e1592ac68 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,17 @@ There are also the methods `Kernel#j` for generate, and `Kernel#jj` for `pretty_generate` output to the console, that work analogous to Core Ruby's `p` and the `pp` library's `pp` methods. +## Security + +When parsing or serializing untrusted input, parser and generator options should never be user controlled. + +```ruby +# Dangerous, DO NOT DO THIS. +JSON.generate(params[:data], params[:options]) +``` + +Security vulnerability reports relying on attacker controlled parsing or generator options will be handled as regular bug fixes. + ## Development ### Prerequisites From 59501c07de4257714d94a2d5bd44f08fad1a4a4c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 May 2026 08:33:39 +0200 Subject: [PATCH 324/506] Get rid of all_images gem --- Gemfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile b/Gemfile index 85e8f54eb..defb4c819 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,6 @@ group :development do gem "rake-compiler" gem "test-unit" gem "test-unit-ruby-core" - gem "all_images", "~> 0" unless RUBY_PLATFORM =~ /java/ gem "simplecov", require: false if RUBY_VERSION < "3.1." From be8d068a8eb1124fdc2273e102dc986edf1140f7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 May 2026 08:24:44 +0200 Subject: [PATCH 325/506] Handle invalid types passed as `max_nesting` option Co-Authored-By: Yuhang Wu --- CHANGES.md | 2 ++ ext/json/ext/generator/generator.c | 2 +- lib/json/truffle_ruby/generator.rb | 3 +++ test/json/json_generator_test.rb | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 73fc7af11..9fc06804b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Handle invalid types passed as `max_nesting` option. + ### 2026-05-28 (2.19.7) * Fix some more edge cases with out of range floats. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9260712c9..110b5f6b3 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1581,7 +1581,7 @@ static VALUE cState_max_nesting(VALUE self) static long long_config(VALUE num) { - return RTEST(num) ? FIX2LONG(num) : 0; + return RTEST(num) ? NUM2LONG(num) : 0; } // depth must never be negative; reject early with a clear error. diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index f03ff7fd8..4fb1885e0 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -307,6 +307,9 @@ def configure(opts) if !opts.key?(:max_nesting) # defaults to 100 @max_nesting = 100 elsif opts[:max_nesting] + unless opts[:max_nesting].is_a?(Integer) + raise TypeError, ":max_nesting must be an Integer, got: #{opts[:max_nesting].class}" + end @max_nesting = opts[:max_nesting] else @max_nesting = 0 diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 87d9cd7f7..753ee0fbd 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -601,6 +601,8 @@ def test_nesting assert_equal too_deep, ok ok = generate too_deep_ary, :max_nesting => 0 assert_equal too_deep, ok + + assert_raise(TypeError) { generate too_deep_ary, max_nesting: "garbage" } end def test_backslash From 3f44b26cf34f37e97065ff37f5eaecac69d5f28e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 29 May 2026 09:05:01 +0200 Subject: [PATCH 326/506] Prevent buffer over-read when generating EOF error When reaching EOS right after an escape the cursor is one byte too far. Reported-By: Yuhang Wu --- ext/json/ext/parser/parser.c | 14 ++++++++++++++ test/json/json_parser_test.rb | 12 ++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 363f109d3..503bed1fd 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -385,6 +385,13 @@ static inline char peek(JSON_ParserState *state) static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out) { + JSON_ASSERT(state->cursor <= state->end); + + // Redundant but helpful for hardening + if (RB_UNLIKELY(state->cursor > state->end)) { + state->cursor = state->end; + } + const char *cursor = state->cursor; long column = 0; long line = 1; @@ -1022,6 +1029,13 @@ ALWAYS_INLINE(static) bool string_scan(JSON_ParserState *state) } state->cursor++; } + + // If the string ended with an unterminated escape sequence, we might + // have gone past the end. + if (RB_UNLIKELY(state->cursor > state->end)) { + state->cursor = state->end; + } + return false; } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 1969e79b3..292ca1a67 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -545,22 +545,26 @@ def test_nesting end def test_backslash + assert_raise(JSON::ParserError) do + JSON.parse('"\\') + end + data = [ '\\.(?i:gif|jpe?g|png)$' ] json = '["\\\\.(?i:gif|jpe?g|png)$"]' assert_equal data, parse(json) - # + data = [ '\\"' ] json = '["\\\\\""]' assert_equal data, parse(json) - # + json = '["/"]' data = [ '/' ] assert_equal data, parse(json) - # + json = '["\""]' data = ['"'] assert_equal data, parse(json) - # + json = '["\\/"]' data = ["/"] assert_equal data, parse(json) From 5233dd9b851a4924f793aec1a1658ed8b66a34c7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Jun 2026 08:28:31 +0200 Subject: [PATCH 327/506] Release 2.19.8 --- CHANGES.md | 3 +++ lib/json/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9fc06804b..1628bd0bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +### 2026-06-03 (2.19.8) + +* Fix 1-byte buffer overread on EOS errors. * Handle invalid types passed as `max_nesting` option. ### 2026-05-28 (2.19.7) diff --git a/lib/json/version.rb b/lib/json/version.rb index a69590ff9..30c0a71d2 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.7' + VERSION = '2.19.8' end From f7fe86ea6929493b9a6f9802dc6761ae89af1ec9 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Jun 2026 08:48:47 +0200 Subject: [PATCH 328/506] parser.c: noinline `json_eat_comments` Comments shouldn't be present in performance sensitive documents, so it's best not to inline it to make space for more important code. --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 503bed1fd..56b214fc4 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -529,7 +529,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const static const rb_data_type_t JSON_ParserConfig_type; -static void +NOINLINE(static) void json_eat_comments(JSON_ParserState *state) { const char *start = state->cursor; From 90354007ea5b6e576bef37e0eed29c058cc456d4 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Jun 2026 09:25:51 +0200 Subject: [PATCH 329/506] Integrate with Ruby 4.1 `ruby_sized_xfree` Ensure that buffer sizes have been properly tracked when running in debug mode, and appease the GC slightly when running in release mode. Ref: https://bugs.ruby-lang.org/issues/21861 --- CHANGES.md | 2 ++ ext/json/ext/fbuffer/fbuffer.h | 4 ++-- ext/json/ext/generator/extconf.rb | 1 + ext/json/ext/json.h | 18 ++++++++++++++++++ ext/json/ext/parser/extconf.rb | 1 + ext/json/ext/parser/parser.c | 6 +++--- 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1628bd0bf..c4808e535 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Integrate with Ruby 4.1 `ruby_sized_xfree`. + ### 2026-06-03 (2.19.8) * Fix 1-byte buffer overread on EOS errors. diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index b84a07355..b4f5266ca 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -64,7 +64,7 @@ static inline void fbuffer_consumed(FBuffer *fb, size_t consumed) static void fbuffer_free(FBuffer *fb) { if (fb->ptr && fb->type == FBUFFER_HEAP_ALLOCATED) { - ruby_xfree(fb->ptr); + JSON_SIZED_FREE_N(fb->ptr, fb->capa); } } @@ -88,7 +88,7 @@ static void fbuffer_realloc(FBuffer *fb, size_t required) fb->type = FBUFFER_HEAP_ALLOCATED; MEMCPY(fb->ptr, old_buffer, char, fb->len); } else { - REALLOC_N(fb->ptr, char, required); + JSON_SIZED_REALLOC_N(fb->ptr, char, required, fb->capa); } fb->capa = required; } diff --git a/ext/json/ext/generator/extconf.rb b/ext/json/ext/generator/extconf.rb index 205a887e7..33af03ea3 100644 --- a/ext/json/ext/generator/extconf.rb +++ b/ext/json/ext/generator/extconf.rb @@ -6,6 +6,7 @@ else append_cflags("-std=c99") have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 + have_func("ruby_xfree_sized", "ruby.h") # RUBY_VERSION >= 4.1 $defs << "-DJSON_GENERATOR" $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 8737989a6..cf9420d4d 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -49,6 +49,24 @@ typedef unsigned char _Bool; #endif #endif +#ifndef HAVE_RUBY_XFREE_SIZED +static inline void ruby_xfree_sized(void *ptr, size_t oldsize) +{ + ruby_xfree(ptr); +} + +static inline void *ruby_xrealloc2_sized(void *ptr, size_t new_elems, size_t elem_size, size_t old_elems) +{ + return ruby_xrealloc2(ptr, new_elems, elem_size); +} +#endif + +# define JSON_SIZED_REALLOC_N(v, T, m, n) \ + ((v) = (T *)ruby_xrealloc2_sized((void *)(v), (m), sizeof(T), (n))) + +# define JSON_SIZED_FREE(v) ruby_xfree_sized((void *)(v), sizeof(*(v))) +# define JSON_SIZED_FREE_N(v, n) ruby_xfree_sized((void *)(v), sizeof(*(v)) * (n)) + #ifndef HAVE_RB_EXT_RACTOR_SAFE # undef RUBY_TYPED_FROZEN_SHAREABLE # define RUBY_TYPED_FROZEN_SHAREABLE 0 diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index f12fc2ddd..a9d740c75 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -6,6 +6,7 @@ have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby +have_func("ruby_xfree_sized", "ruby.h") # RUBY_VERSION >= 4.1 if RUBY_ENGINE == "ruby" have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 56b214fc4..d0482f686 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -211,7 +211,7 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu if (stack->type == RVALUE_STACK_STACK_ALLOCATED) { stack = rvalue_stack_spill(stack, handle, stack_ref); } else { - REALLOC_N(stack->ptr, VALUE, required); + JSON_SIZED_REALLOC_N(stack->ptr, VALUE, required, stack->capa); stack->capa = required; } return stack; @@ -250,7 +250,7 @@ static void rvalue_stack_mark(void *ptr) static void rvalue_stack_free_buffer(rvalue_stack *stack) { - ruby_xfree(stack->ptr); + JSON_SIZED_FREE_N(stack->ptr, stack->capa); stack->ptr = NULL; } @@ -260,7 +260,7 @@ static void rvalue_stack_free(void *ptr) if (stack) { rvalue_stack_free_buffer(stack); #ifndef HAVE_RUBY_TYPED_EMBEDDABLE - ruby_xfree(stack); + JSON_SIZED_FREE(stack); #endif } } From ec47912123966edd7a80074cdc8df68bbda39149 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Jun 2026 09:36:15 +0200 Subject: [PATCH 330/506] Add bigdecimal in Gemfile Avoid warning on newer rubies --- Gemfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile b/Gemfile index defb4c819..7956c2dc2 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ gemspec group :development do gem "ruby_memcheck" if RUBY_PLATFORM =~ /linux/i + gem "bigdecimal" gem "ostruct" gem "rake" gem "rake-compiler" From 52c9d9bdf2af835061d081481a88aa9560b1ae45 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 3 Jun 2026 09:45:45 +0200 Subject: [PATCH 331/506] Coverage: ignore test/* --- test/json/test_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index 24cde4348..4c5a91a19 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -21,6 +21,7 @@ track_files 'lib/**/*.rb' add_filter 'lib/json/truffle_ruby' unless RUBY_ENGINE == 'truffleruby' + add_filter 'test/' end end From a760d9ae9bf30c785fcb5d5ba79a445f01dc2c43 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Jun 2026 12:47:06 -0400 Subject: [PATCH 332/506] Make the JSON parse loop iterative As opposed to a recursive loop. We do this by keeping a stack of frames (very similar to how the stack of values was already stored). Each frame represents the state of a container. Since there are only 2 in JSON, it doesn't have to get too complex. Each frame in the iterative parser now holds an enum describing its "phase", in order to support suspending parsing. For performance to be on par with the previous implementation, we directly jump to the proper handler in most cases. `JSON_ParserState.stack` was renamed into `.value_stack`, as with the introduction of the frames stack, the naming became confusing. Co-Authored-By: Jean Boussier --- ext/json/ext/parser/parser.c | 650 +++++++++++++++++++++++++---------- 1 file changed, 462 insertions(+), 188 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d0482f686..0b0bfd713 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -309,31 +309,57 @@ static void rvalue_stack_eagerly_release(VALUE handle) } } -static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) -{ - int len = 1; - if (ch <= 0x7F) { - buf[0] = (char) ch; - } else if (ch <= 0x07FF) { - buf[0] = (char) ((ch >> 6) | 0xC0); - buf[1] = (char) ((ch & 0x3F) | 0x80); - len++; - } else if (ch <= 0xFFFF) { - buf[0] = (char) ((ch >> 12) | 0xE0); - buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); - buf[2] = (char) ((ch & 0x3F) | 0x80); - len += 2; - } else if (ch <= 0x1fffff) { - buf[0] =(char) ((ch >> 18) | 0xF0); - buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); - buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); - buf[3] =(char) ((ch & 0x3F) | 0x80); - len += 3; - } else { - buf[0] = '?'; - } - return len; -} +/* frame stack */ + +// Iterative (non-recursive) parsing keeps an explicit stack of the containers +// currently being built, instead of relying on the C call stack. Each frame +// only needs enough bookkeeping to close its container: which kind it is, the +// rvalue_stack position where its children start (so we know how many to pop), +// and the cursor at its opening brace (used to rewind for duplicate key +// errors). Frames hold no VALUEs, so this stack needs no GC marking; it reuses +// the same stack-allocated-with-heap-spill strategy as the rvalue_stack so that +// it's freed even if parsing raises. +// +// The lifecycle helpers below (grow/push/peek/pop/spill/free/eagerly_release +// and the rb_data_type_t) deliberately mirror their rvalue_stack counterparts +// -- the element type and the absence of a mark function are the only real +// differences. Keep the two in sync: a fix to the spill/release or +// HAVE_RUBY_TYPED_EMBEDDABLE handling in one almost certainly belongs in the +// other. +#define JSON_FRAME_STACK_INITIAL_CAPA 32 + +enum json_frame_type { + JSON_FRAME_ROOT, + JSON_FRAME_ARRAY, + JSON_FRAME_OBJECT, +}; + +// Where a frame is within its container's grammar. This is the entirety of the +// parser's "what to do next" state: json_parse_any dispatches on the top +// frame's phase and holds no resume state in C locals, so a parse can stop at +// any value boundary and be resumed purely from the (persistable) frame stack. +enum json_frame_phase { + JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':') + JSON_PHASE_ARRAY_COMMA, // after a value: expecting ',' or the closing ']' + JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',') + JSON_PHASE_OBJECT_COMMA, // after a value: expecting ',' or the closing '}' + JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':' + JSON_PHASE_DONE, // root only: the document value has been parsed +}; + +typedef struct json_frame_struct { + enum json_frame_type type; + enum json_frame_phase phase; + long value_stack_head; // rvalue_stack->head when this container opened + const char *start_cursor; // object frames only (the '{'); NULL otherwise +} json_frame; + +typedef struct json_frame_stack_struct { + enum rvalue_stack_type type; // shared with rvalue_stack: is ptr stack- or heap-allocated + long capa; + long head; + json_frame *ptr; +} json_frame_stack; enum duplicate_key_action { JSON_DEPRECATED = 0, @@ -356,17 +382,143 @@ typedef struct JSON_ParserStruct { } JSON_ParserConfig; typedef struct JSON_ParserStateStruct { - VALUE *stack_handle; + VALUE *value_stack_handle; + VALUE *frame_stack_handle; const char *start; const char *cursor; const char *end; - rvalue_stack *stack; + rvalue_stack *value_stack; + json_frame_stack *frames; rvalue_cache name_cache; int in_array; int current_nesting; unsigned int emitted_deprecations; } JSON_ParserState; +static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref); + +static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref) +{ + long required = stack->capa * 2; + + if (stack->type == RVALUE_STACK_STACK_ALLOCATED) { + stack = json_frame_stack_spill(stack, handle, stack_ref); + } else { + JSON_SIZED_REALLOC_N(stack->ptr, json_frame, required, stack->capa); + stack->capa = required; + } + return stack; +} + +static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame) +{ + json_frame_stack *stack = state->frames; + if (RB_UNLIKELY(stack->head >= stack->capa)) { + stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames); + } + + json_frame *frame_ptr = &stack->ptr[stack->head++]; + *frame_ptr = frame; + return frame_ptr; +} + +static inline json_frame *json_frame_stack_peek(json_frame_stack *stack) +{ + return &stack->ptr[stack->head - 1]; +} + +static inline void json_frame_stack_pop(json_frame_stack *stack) +{ + stack->head--; +} + +static void json_frame_stack_free_buffer(json_frame_stack *stack) +{ + JSON_SIZED_FREE_N(stack->ptr, stack->capa); + stack->ptr = NULL; +} + +static void json_frame_stack_free(void *ptr) +{ + json_frame_stack *stack = (json_frame_stack *)ptr; + if (stack) { + json_frame_stack_free_buffer(stack); +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE + JSON_SIZED_FREE(stack); +#endif + } +} + +static size_t json_frame_stack_memsize(const void *ptr) +{ + const json_frame_stack *stack = (const json_frame_stack *)ptr; + return sizeof(json_frame_stack) + sizeof(json_frame) * stack->capa; +} + +static const rb_data_type_t JSON_Parser_frame_stack_type = { + .wrap_struct_name = "JSON::Ext::Parser/frame_stack", + .function = { + .dmark = NULL, + .dfree = json_frame_stack_free, + .dsize = json_frame_stack_memsize, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, +}; + +static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref) +{ + json_frame_stack *stack; + *handle = TypedData_Make_Struct(0, json_frame_stack, &JSON_Parser_frame_stack_type, stack); + *stack_ref = stack; + MEMCPY(stack, old_stack, json_frame_stack, 1); + + stack->capa = old_stack->capa << 1; + stack->ptr = ALLOC_N(json_frame, stack->capa); + stack->type = RVALUE_STACK_HEAP_ALLOCATED; + MEMCPY(stack->ptr, old_stack->ptr, json_frame, old_stack->head); + return stack; +} + +static void json_frame_stack_eagerly_release(VALUE handle) +{ + if (handle) { + json_frame_stack *stack; + TypedData_Get_Struct(handle, json_frame_stack, &JSON_Parser_frame_stack_type, stack); +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + json_frame_stack_free_buffer(stack); +#else + json_frame_stack_free(stack); + RTYPEDDATA_DATA(handle) = NULL; +#endif + } +} + +static int convert_UTF32_to_UTF8(char *buf, uint32_t ch) +{ + int len = 1; + if (ch <= 0x7F) { + buf[0] = (char) ch; + } else if (ch <= 0x07FF) { + buf[0] = (char) ((ch >> 6) | 0xC0); + buf[1] = (char) ((ch & 0x3F) | 0x80); + len++; + } else if (ch <= 0xFFFF) { + buf[0] = (char) ((ch >> 12) | 0xE0); + buf[1] = (char) (((ch >> 6) & 0x3F) | 0x80); + buf[2] = (char) ((ch & 0x3F) | 0x80); + len += 2; + } else if (ch <= 0x1fffff) { + buf[0] =(char) ((ch >> 18) | 0xF0); + buf[1] =(char) (((ch >> 12) & 0x3F) | 0x80); + buf[2] =(char) (((ch >> 6) & 0x3F) | 0x80); + buf[3] =(char) ((ch & 0x3F) | 0x80); + len += 3; + } else { + buf[0] = '?'; + } + return len; +} + static inline size_t rest(JSON_ParserState *state) { return state->end - state->cursor; } @@ -890,8 +1042,8 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count) { - VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->stack, count)); - rvalue_stack_pop(state->stack, count); + VALUE array = rb_ary_new_from_values(count, rvalue_stack_peek(state->value_stack, count)); + rvalue_stack_pop(state->value_stack, count); if (config->freeze) { RB_OBJ_FREEZE(array); @@ -945,7 +1097,7 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi { size_t entries_count = count / 2; VALUE object = rb_hash_new_capa(entries_count); - const VALUE *pairs = rvalue_stack_peek(state->stack, count); + const VALUE *pairs = rvalue_stack_peek(state->value_stack, count); rb_hash_bulk_insert(count, pairs, object); if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) { @@ -966,7 +1118,7 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi } } - rvalue_stack_pop(state->stack, count); + rvalue_stack_pop(state->value_stack, count); if (config->freeze) { RB_OBJ_FREEZE(object); @@ -980,7 +1132,7 @@ static inline VALUE json_push_value(JSON_ParserState *state, JSON_ParserConfig * if (RB_UNLIKELY(config->on_load_proc)) { value = rb_proc_call_with_block(config->on_load_proc, 1, &value, Qnil); } - rvalue_stack_push(state->stack, value, state->stack_handle, &state->stack); + rvalue_stack_push(state->value_stack, value, state->value_stack_handle, &state->value_stack); return value; } @@ -1247,92 +1399,241 @@ static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_Par return json_parse_number(state, config, true, start); } +// How many values (array elements, or interleaved object keys+values) have been +// pushed onto the rvalue stack since this container opened. Used to size the +// bulk decode on close, and to tell the first key/colon from later ones. +static inline long json_frame_entry_count(const json_frame *frame, const rvalue_stack *value_stack) +{ + return value_stack->head - frame->value_stack_head; +} + +// A complete value now sits on top of the rvalue stack. Advance the frame that +// was waiting for it: the root document is done, or the enclosing container +// moves on to expecting a ',' or its closing bracket. The caller passes the +// frame it already has in hand -- the one that was expecting the value -- which +// after a container close is the freshly re-exposed parent. +static inline void json_value_completed(json_frame *frame) +{ + // TODO: consider a lookup table? + switch (frame->type) { + case JSON_FRAME_ROOT: + frame->phase = JSON_PHASE_DONE; + break; + case JSON_FRAME_ARRAY: + frame->phase = JSON_PHASE_ARRAY_COMMA; + break; + case JSON_FRAME_OBJECT: + frame->phase = JSON_PHASE_OBJECT_COMMA; + break; + } +} + +// Parse an arbitrary JSON value iteratively. This is a state machine driven +// entirely by the top frame's phase so it can stop at any value boundary and +// resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the +// bottom of the stack, so the stack is never empty mid-parse and the document +// itself is just another frame whose value, once parsed, leaves its phase DONE. static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { - json_eat_whitespace(state); + while (true) { + json_frame *frame = json_frame_stack_peek(state->frames); - switch (peek(state)) { - case 'n': - if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) { - state->cursor += 4; - return json_push_value(state, config, Qnil); + switch (frame->phase) { + case JSON_PHASE_DONE: { + // The root document value is parsed; it is the lone survivor on + // the rvalue stack. + return *rvalue_stack_peek(state->value_stack, 1); } - raise_parse_error("unexpected token %s", state); - break; - case 't': - if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) { - state->cursor += 4; - return json_push_value(state, config, Qtrue); - } + case JSON_PHASE_VALUE: { + JSON_PHASE_VALUE: + json_eat_whitespace(state); - raise_parse_error("unexpected token %s", state); - break; - case 'f': - // Note: memcmp with a small power of two compile to an integer comparison - if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) { - state->cursor += 5; - return json_push_value(state, config, Qfalse); - } + switch (peek(state)) { + case 'n': + if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) { + state->cursor += 4; + json_push_value(state, config, Qnil); + json_value_completed(frame); + break; + } - raise_parse_error("unexpected token %s", state); - break; - case 'N': - // Note: memcmp with a small power of two compile to an integer comparison - if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) { - state->cursor += 3; - return json_push_value(state, config, CNaN); - } + raise_parse_error("unexpected token %s", state); + case 't': + if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) { + state->cursor += 4; + json_push_value(state, config, Qtrue); + json_value_completed(frame); + break; + } - raise_parse_error("unexpected token %s", state); - break; - case 'I': - if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) { - state->cursor += 8; - return json_push_value(state, config, CInfinity); + raise_parse_error("unexpected token %s", state); + case 'f': + // Note: memcmp with a small power of two compile to an integer comparison + if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) { + state->cursor += 5; + json_push_value(state, config, Qfalse); + json_value_completed(frame); + break; + } + + raise_parse_error("unexpected token %s", state); + case 'N': + // Note: memcmp with a small power of two compile to an integer comparison + if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) { + state->cursor += 3; + json_push_value(state, config, CNaN); + json_value_completed(frame); + break; + } + + raise_parse_error("unexpected token %s", state); + case 'I': + if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) { + state->cursor += 8; + json_push_value(state, config, CInfinity); + json_value_completed(frame); + break; + } + + raise_parse_error("unexpected token %s", state); + case '-': { + // Note: memcmp with a small power of two compile to an integer comparison + if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { + if (config->allow_nan) { + state->cursor += 9; + json_push_value(state, config, CMinusInfinity); + json_value_completed(frame); + break; + } else { + raise_parse_error("unexpected token %s", state); + } + } + json_push_value(state, config, json_parse_negative_number(state, config)); + json_value_completed(frame); + break; + } + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + json_push_value(state, config, json_parse_positive_number(state, config)); + json_value_completed(frame); + break; + case '"': + // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} + json_parse_string(state, config, false); + json_value_completed(frame); + break; + case '[': { + state->cursor++; + json_eat_whitespace(state); + + if (peek(state) == ']') { + state->cursor++; + json_push_value(state, config, json_decode_array(state, config, 0)); + json_value_completed(frame); + break; + } + + state->current_nesting++; + if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { + rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); + } + state->in_array++; + + // Phase stays VALUE: the next iteration reads the first element. + frame = json_frame_stack_push(state, (json_frame){ + .type = JSON_FRAME_ARRAY, + .phase = JSON_PHASE_VALUE, + .value_stack_head = state->value_stack->head, + }); + goto JSON_PHASE_VALUE; + break; + } + case '{': { + const char *object_start_cursor = state->cursor; + + state->cursor++; + json_eat_whitespace(state); + + if (peek(state) == '}') { + state->cursor++; + json_push_value(state, config, json_decode_object(state, config, 0)); + json_value_completed(frame); + break; + } + + state->current_nesting++; + if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { + rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); + } + + // Phase KEY: the next iteration reads the first key. + frame = json_frame_stack_push(state, (json_frame){ + .type = JSON_FRAME_OBJECT, + .phase = JSON_PHASE_OBJECT_KEY, + .value_stack_head = state->value_stack->head, + .start_cursor = object_start_cursor, + }); + goto JSON_PHASE_OBJECT_KEY; + break; + } + + case 0: + raise_parse_error("unexpected end of input", state); + + default: + raise_parse_error("unexpected character: %s", state); + } + break; } - raise_parse_error("unexpected token %s", state); - break; - case '-': { - // Note: memcmp with a small power of two compile to an integer comparison - if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { - if (config->allow_nan) { - state->cursor += 9; - return json_push_value(state, config, CMinusInfinity); + case JSON_PHASE_OBJECT_KEY: { + JSON_PHASE_OBJECT_KEY: + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + + json_eat_whitespace(state); + + if (RB_LIKELY(peek(state) == '"')) { + json_parse_string(state, config, true); + frame->phase = JSON_PHASE_OBJECT_COLON; + goto JSON_PHASE_OBJECT_COLON; } else { - raise_parse_error("unexpected token %s", state); + // The message differs for the first key vs. a key after a + // ',': the first is the only one reached with nothing pushed + // for this object yet. + if (json_frame_entry_count(frame, state->value_stack) == 0) { + raise_parse_error("expected object key, got %s", state); + } else { + raise_parse_error("expected object key, got: %s", state); + } } + break; } - return json_push_value(state, config, json_parse_negative_number(state, config)); - break; - } - case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - return json_push_value(state, config, json_parse_positive_number(state, config)); - break; - case '"': { - // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} - return json_parse_string(state, config, false); - break; - } - case '[': { - state->cursor++; - json_eat_whitespace(state); - long stack_head = state->stack->head; - if (peek(state) == ']') { - state->cursor++; - return json_push_value(state, config, json_decode_array(state, config, 0)); - } else { - state->current_nesting++; - if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { - rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); + case JSON_PHASE_OBJECT_COLON: { + JSON_PHASE_OBJECT_COLON: + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + + json_eat_whitespace(state); + + if (RB_LIKELY(peek(state) == ':')) { + state->cursor++; + frame->phase = JSON_PHASE_VALUE; + goto JSON_PHASE_VALUE; + } else { + // First colon (only the first pair's key is pushed, nothing + // else) vs. a later one. + if (json_frame_entry_count(frame, state->value_stack) == 1) { + raise_parse_error("expected ':' after object key", state); + } else { + raise_parse_error("expected ':' after object key, got: %s", state); + } } - state->in_array++; - json_parse_any(state, config); + break; } - while (true) { + case JSON_PHASE_ARRAY_COMMA: { + JSON_ASSERT(frame->type == JSON_FRAME_ARRAY); + json_eat_whitespace(state); const char next_char = peek(state); @@ -1342,115 +1643,68 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if (config->allow_trailing_comma) { json_eat_whitespace(state); if (peek(state) == ']') { - continue; + // Trailing comma: stay in COMMA to close on the next iteration. + break; } } - json_parse_any(state, config); - continue; - } - - if (next_char == ']') { + frame->phase = JSON_PHASE_VALUE; + goto JSON_PHASE_VALUE; + } else if (next_char == ']') { state->cursor++; - long count = state->stack->head - stack_head; + long count = json_frame_entry_count(frame, state->value_stack); state->current_nesting--; state->in_array--; - return json_push_value(state, config, json_decode_array(state, config, count)); + json_frame_stack_pop(state->frames); + json_push_value(state, config, json_decode_array(state, config, count)); + json_value_completed(json_frame_stack_peek(state->frames)); + } else { + raise_parse_error("expected ',' or ']' after array value", state); } - - raise_parse_error("expected ',' or ']' after array value", state); + break; } - break; - } - case '{': { - const char *object_start_cursor = state->cursor; - state->cursor++; - json_eat_whitespace(state); - long stack_head = state->stack->head; + case JSON_PHASE_OBJECT_COMMA: { + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - if (peek(state) == '}') { - state->cursor++; - return json_push_value(state, config, json_decode_object(state, config, 0)); - } else { - state->current_nesting++; - if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { - rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); - } - - if (peek(state) != '"') { - raise_parse_error("expected object key, got %s", state); - } - json_parse_string(state, config, true); - - json_eat_whitespace(state); - if (peek(state) != ':') { - raise_parse_error("expected ':' after object key", state); - } - state->cursor++; - - json_parse_any(state, config); - } - - while (true) { json_eat_whitespace(state); - const char next_char = peek(state); - if (next_char == '}') { - state->cursor++; - state->current_nesting--; - size_t count = state->stack->head - stack_head; - // Temporary rewind cursor in case an error is raised - const char *final_cursor = state->cursor; - state->cursor = object_start_cursor; - VALUE object = json_decode_object(state, config, count); - state->cursor = final_cursor; - - return json_push_value(state, config, object); - } - - if (next_char == ',') { + if (RB_LIKELY(next_char == ',')) { state->cursor++; json_eat_whitespace(state); if (config->allow_trailing_comma) { if (peek(state) == '}') { - continue; + // Trailing comma: stay in COMMA to close on the next iteration. + break; } } - if (RB_UNLIKELY(peek(state) != '"')) { - raise_parse_error("expected object key, got: %s", state); - } - json_parse_string(state, config, true); + frame->phase = JSON_PHASE_OBJECT_KEY; + goto JSON_PHASE_OBJECT_KEY; - json_eat_whitespace(state); - if (RB_UNLIKELY(peek(state) != ':')) { - raise_parse_error("expected ':' after object key, got: %s", state); - } + break; + } else if (next_char == '}') { state->cursor++; + state->current_nesting--; + size_t count = json_frame_entry_count(frame, state->value_stack); - json_parse_any(state, config); + // Temporary rewind cursor in case an error is raised + const char *final_cursor = state->cursor; + state->cursor = frame->start_cursor; + VALUE object = json_decode_object(state, config, count); + state->cursor = final_cursor; - continue; + json_frame_stack_pop(state->frames); + json_push_value(state, config, object); + json_value_completed(json_frame_stack_peek(state->frames)); + break; + } else { + raise_parse_error("expected ',' or '}' after object value, got: %s", state); } - - raise_parse_error("expected ',' or '}' after object value, got: %s", state); } - break; } - - case 0: - raise_parse_error("unexpected end of input", state); - break; - - default: - raise_parse_error("unexpected character: %s", state); - break; } - - raise_parse_error("unreachable: %s", state); - return Qundef; } static void json_ensure_eof(JSON_ParserState *state) @@ -1616,24 +1870,42 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) } VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA]; - rvalue_stack stack = { + rvalue_stack value_stack = { .type = RVALUE_STACK_STACK_ALLOCATED, .ptr = rvalue_stack_buffer, .capa = RVALUE_STACK_INITIAL_CAPA, }; + // Seed the frame stack with the root frame, establishing the invariant that + // json_parse_any always has a top frame to dispatch on (so the stack is never + // empty mid-parse). + json_frame frame_stack_buffer[JSON_FRAME_STACK_INITIAL_CAPA]; + frame_stack_buffer[0] = (json_frame){ + .type = JSON_FRAME_ROOT, + .phase = JSON_PHASE_VALUE, + }; + json_frame_stack frames = { + .type = RVALUE_STACK_STACK_ALLOCATED, + .ptr = frame_stack_buffer, + .capa = JSON_FRAME_STACK_INITIAL_CAPA, + .head = 1, + }; + long len; const char *start; RSTRING_GETMEM(Vsource, start, len); - VALUE stack_handle = 0; + VALUE value_stack_handle = 0; + VALUE frame_stack_handle = 0; JSON_ParserState _state = { .start = start, .cursor = start, .end = start + len, - .stack = &stack, - .stack_handle = &stack_handle, + .value_stack = &value_stack, + .value_stack_handle = &value_stack_handle, + .frames = &frames, + .frame_stack_handle = &frame_stack_handle, }; JSON_ParserState *state = &_state; @@ -1641,8 +1913,10 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) // This may be skipped in case of exception, but // it won't cause a leak. - rvalue_stack_eagerly_release(stack_handle); - RB_GC_GUARD(stack_handle); + rvalue_stack_eagerly_release(value_stack_handle); + json_frame_stack_eagerly_release(frame_stack_handle); + RB_GC_GUARD(value_stack_handle); + RB_GC_GUARD(frame_stack_handle); RB_GC_GUARD(Vsource); json_ensure_eof(state); From 216e0ffd27c80d343bfdf402e2df7a305fa23dae Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 4 Jun 2026 08:59:55 +0200 Subject: [PATCH 333/506] Reove outdated benchmark notes We've been faster on all benchmarks for a while now. All these notes have been addressed a long time ago. --- benchmark/encoder.rb | 19 ------------------- benchmark/parser.rb | 12 ++++-------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/benchmark/encoder.rb b/benchmark/encoder.rb index 4cc8891d2..f29334bd9 100644 --- a/benchmark/encoder.rb +++ b/benchmark/encoder.rb @@ -63,37 +63,18 @@ def benchmark_encoding(benchmark_name, ruby_obj, check_expected: true, except: [ puts end -# NB: Notes are based on ruby 3.3.4 (2024-07-09 revision be1089c8ec) +YJIT [arm64-darwin23] - -# On the first two micro benchmarks, the limiting factor is the fixed cost of initializing the -# generator state. Since `JSON.generate` now lazily allocate the `State` object we're now ~10-20% faster -# than `Oj.dump`. benchmark_encoding "small mixed", [1, "string", { a: 1, b: 2 }, [3, 4, 5]] benchmark_encoding "small nested array", [[1,2,3,4,5]]*10 benchmark_encoding "small hash", { "username" => "jhawthorn", "id" => 123, "event" => "wrote json serializer" } -# On string encoding we're ~20% faster when dealing with mostly ASCII, but ~50% slower when dealing -# with mostly multi-byte characters. There's likely some gains left to be had in multi-byte handling. benchmark_encoding "mixed utf8", ([("a" * 5000) + "€" + ("a" * 5000)] * 500) benchmark_encoding "mostly utf8", ([("€" * 3333)] * 500) -# On these benchmarks we perform well, we're on par or a bit better. benchmark_encoding "integers", (1_000_000..1_001_000).to_a, except: %i(json_state) benchmark_encoding "activitypub.json", JSON.load_file("#{__dir__}/data/activitypub.json") benchmark_encoding "citm_catalog.json", JSON.load_file("#{__dir__}/data/citm_catalog.json") benchmark_encoding "twitter.json", JSON.load_file("#{__dir__}/data/twitter.json") -# This benchmark spent the overwhelming majority of its time in `ruby_dtoa`. We rely on Ruby's implementation -# which uses a relatively old version of dtoa.c from David M. Gay. -# Oj in `compat` mode is ~10% slower than `json`, but in its default mode is noticeably faster here because -# it limits the precision of floats, breaking roundtripping. That's not something we should emulate. -# -# Since a few years there are now much faster float to string implementations such as Ryu, Dragonbox, etc, -# but all these are implemented in C++11 or newer, making it hard if not impossible to include them. -# Short of a pure C99 implementation of these newer algorithms, there isn't much that can be done to match -# Oj speed without losing precision. benchmark_encoding "canada.json", JSON.load_file("#{__dir__}/data/canada.json"), check_expected: false -# We're about 10% faster when `to_json` calls are involved, but this wasn't particularly optimized, there might be -# opportunities here. benchmark_encoding "many #to_json calls", [{object: Object.new, int: 12, float: 54.3, class: Float, time: Time.now, date: Date.today}] * 20 diff --git a/benchmark/parser.rb b/benchmark/parser.rb index 71c527ea4..54ed32151 100644 --- a/benchmark/parser.rb +++ b/benchmark/parser.rb @@ -42,18 +42,14 @@ def benchmark_parsing(name, json_output) puts end -# NB: Notes are based on ruby 3.3.4 (2024-07-09 revision be1089c8ec) +YJIT [arm64-darwin23] - benchmark_parsing "small nested array", JSON.dump([[1,2,3,4,5]]*10) benchmark_parsing "small hash", JSON.dump({ "username" => "jhawthorn", "id" => 123, "event" => "wrote json serializer" }) -benchmark_parsing "test from oj", < Date: Thu, 4 Jun 2026 10:08:07 +0200 Subject: [PATCH 334/506] parser.c: Mark some paths as unreacheable Generate very sligthly better code (one less instruction...). Also extract `json_on_duplicate_key` as `NOINLINE`. --- ext/json/ext/parser/parser.c | 45 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 0b0bfd713..eb7dc9aa8 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1093,6 +1093,27 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d rb_exc_raise(parse_error_new(message, line, column)); } +NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) +{ + switch (config->on_duplicate_key) { + case JSON_IGNORE: + return; + + case JSON_DEPRECATED: + // Only emit the first few deprecations to avoid spamming. + if (state->emitted_deprecations < 5) { + emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); + state->emitted_deprecations++; + } + return; + + case JSON_RAISE: + raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); + return; + } + UNREACHABLE; +} + static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count) { size_t entries_count = count / 2; @@ -1101,21 +1122,7 @@ static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfi rb_hash_bulk_insert(count, pairs, object); if (RB_UNLIKELY(RHASH_SIZE(object) < entries_count)) { - switch (config->on_duplicate_key) { - case JSON_IGNORE: - break; - case JSON_DEPRECATED: - // Only emit the first few deprecations to avoid spamming. - if (state->emitted_deprecations < 5) { - emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); - state->emitted_deprecations++; - } - - break; - case JSON_RAISE: - raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); - break; - } + json_on_duplicate_key(state, config, count, pairs); } rvalue_stack_pop(state->value_stack, count); @@ -1414,18 +1421,18 @@ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_ // after a container close is the freshly re-exposed parent. static inline void json_value_completed(json_frame *frame) { - // TODO: consider a lookup table? switch (frame->type) { case JSON_FRAME_ROOT: frame->phase = JSON_PHASE_DONE; - break; + return; case JSON_FRAME_ARRAY: frame->phase = JSON_PHASE_ARRAY_COMMA; - break; + return; case JSON_FRAME_OBJECT: frame->phase = JSON_PHASE_OBJECT_COMMA; - break; + return; } + UNREACHABLE; } // Parse an arbitrary JSON value iteratively. This is a state machine driven From b1a0891cc3d2f063c7e8300cb8cf8bfb34f95a6e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 5 Jun 2026 09:40:39 +0200 Subject: [PATCH 335/506] Fix memsize function for embedded types We shouldn't report the size of the embedded struct. --- ext/json/ext/generator/generator.c | 4 ++++ ext/json/ext/parser/parser.c | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 110b5f6b3..82853633b 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -724,7 +724,11 @@ static void State_compact(void *ptr) static size_t State_memsize(const void *ptr) { +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + return 0; +#else return sizeof(JSON_Generator_State); +#endif } static const rb_data_type_t JSON_Generator_State_type = { diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index eb7dc9aa8..2d47f31c0 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -268,7 +268,11 @@ static void rvalue_stack_free(void *ptr) static size_t rvalue_stack_memsize(const void *ptr) { const rvalue_stack *stack = (const rvalue_stack *)ptr; - return sizeof(rvalue_stack) + sizeof(VALUE) * stack->capa; + size_t memsize = sizeof(VALUE) * stack->capa; +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE + memsize += sizeof(rvalue_stack); +#endif + return memsize; } static const rb_data_type_t JSON_Parser_rvalue_stack_type = { @@ -452,7 +456,12 @@ static void json_frame_stack_free(void *ptr) static size_t json_frame_stack_memsize(const void *ptr) { const json_frame_stack *stack = (const json_frame_stack *)ptr; - return sizeof(json_frame_stack) + sizeof(json_frame) * stack->capa; + + size_t memsize = sizeof(json_frame) * stack->capa; +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE + memsize += sizeof(json_frame_stack); +#endif + return memsize; } static const rb_data_type_t JSON_Parser_frame_stack_type = { @@ -1961,7 +1970,11 @@ static void JSON_ParserConfig_mark(void *ptr) static size_t JSON_ParserConfig_memsize(const void *ptr) { +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + return 0; +#else return sizeof(JSON_ParserConfig); +#endif } static const rb_data_type_t JSON_ParserConfig_type = { From 84fbc08bc9688bf0e41fe173ef87e87b20e69f41 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 5 Jun 2026 10:03:44 +0200 Subject: [PATCH 336/506] Implement GC compaction for JSON_ParserConfig and JSON_Parser_rvalue_stack --- ext/json/ext/parser/parser.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2d47f31c0..ca082812e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -243,7 +243,7 @@ static void rvalue_stack_mark(void *ptr) long index; if (stack && stack->ptr) { for (index = 0; index < stack->head; index++) { - rb_gc_mark(stack->ptr[index]); + rb_gc_mark_movable(stack->ptr[index]); } } } @@ -275,12 +275,24 @@ static size_t rvalue_stack_memsize(const void *ptr) return memsize; } +static void rvalue_stack_compact(void *ptr) +{ + rvalue_stack *stack = (rvalue_stack *)ptr; + long index; + if (stack && stack->ptr) { + for (index = 0; index < stack->head; index++) { + stack->ptr[index] = rb_gc_location(stack->ptr[index]); + } + } +} + static const rb_data_type_t JSON_Parser_rvalue_stack_type = { .wrap_struct_name = "JSON::Ext::Parser/rvalue_stack", .function = { .dmark = rvalue_stack_mark, .dfree = rvalue_stack_free, .dsize = rvalue_stack_memsize, + .dcompact = rvalue_stack_compact, }, .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, }; @@ -1964,8 +1976,8 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts) static void JSON_ParserConfig_mark(void *ptr) { JSON_ParserConfig *config = ptr; - rb_gc_mark(config->on_load_proc); - rb_gc_mark(config->decimal_class); + rb_gc_mark_movable(config->on_load_proc); + rb_gc_mark_movable(config->decimal_class); } static size_t JSON_ParserConfig_memsize(const void *ptr) @@ -1977,12 +1989,20 @@ static size_t JSON_ParserConfig_memsize(const void *ptr) #endif } +static void JSON_ParserConfig_compact(void *ptr) +{ + JSON_ParserConfig *config = ptr; + config->on_load_proc = rb_gc_location(config->on_load_proc); + config->decimal_class = rb_gc_location(config->decimal_class); +} + static const rb_data_type_t JSON_ParserConfig_type = { .wrap_struct_name = "JSON::Ext::Parser/ParserConfig", .function = { - JSON_ParserConfig_mark, - RUBY_DEFAULT_FREE, - JSON_ParserConfig_memsize, + .dmark = JSON_ParserConfig_mark, + .dfree = RUBY_DEFAULT_FREE, + .dsize = JSON_ParserConfig_memsize, + .dcompact = JSON_ParserConfig_compact, }, .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; From 887274e642c6e0f0445d0ad4995e3fc29a025692 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 5 Jun 2026 09:04:22 -0500 Subject: [PATCH 337/506] Reorder the json_frame_type and json_frame_phase enum to simplify the transition from a JSON_PHASE_VALUE to the next phase. --- ext/json/ext/parser/parser.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index ca082812e..b820066c5 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -345,22 +345,25 @@ static void rvalue_stack_eagerly_release(VALUE handle) #define JSON_FRAME_STACK_INITIAL_CAPA 32 enum json_frame_type { - JSON_FRAME_ROOT, - JSON_FRAME_ARRAY, - JSON_FRAME_OBJECT, + JSON_FRAME_ROOT, // == JSON_PHASE_DONE + JSON_FRAME_ARRAY, // == JSON_PHASE_ARRAY_COMMA + JSON_FRAME_OBJECT, // = JSON_PHASE_OBJECT_COMMA }; // Where a frame is within its container's grammar. This is the entirety of the // parser's "what to do next" state: json_parse_any dispatches on the top // frame's phase and holds no resume state in C locals, so a parse can stop at // any value boundary and be resumed purely from the (persistable) frame stack. +// +// The first three phases are deliberately equal to the corresponding json_frame_type +// to simplify the transition of phase in json_value_completed. enum json_frame_phase { + JSON_PHASE_DONE = JSON_FRAME_ROOT, // root only: the document value has been parsed + JSON_PHASE_ARRAY_COMMA = JSON_FRAME_ARRAY, // after a value: expecting ',' or the closing ']' + JSON_PHASE_OBJECT_COMMA = JSON_FRAME_OBJECT, // after a value: expecting ',' or the closing '}' JSON_PHASE_VALUE, // expecting a value (document root, array element, or object value after ':') - JSON_PHASE_ARRAY_COMMA, // after a value: expecting ',' or the closing ']' JSON_PHASE_OBJECT_KEY, // expecting a '"' key (after '{' or ',') - JSON_PHASE_OBJECT_COMMA, // after a value: expecting ',' or the closing '}' JSON_PHASE_OBJECT_COLON, // object only: after a key, expecting ':' - JSON_PHASE_DONE, // root only: the document value has been parsed }; typedef struct json_frame_struct { @@ -1442,18 +1445,11 @@ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_ // after a container close is the freshly re-exposed parent. static inline void json_value_completed(json_frame *frame) { - switch (frame->type) { - case JSON_FRAME_ROOT: - frame->phase = JSON_PHASE_DONE; - return; - case JSON_FRAME_ARRAY: - frame->phase = JSON_PHASE_ARRAY_COMMA; - return; - case JSON_FRAME_OBJECT: - frame->phase = JSON_PHASE_OBJECT_COMMA; - return; - } - UNREACHABLE; + JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT); + JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY); + JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT); + + frame->phase = (enum json_frame_phase) frame->type; } // Parse an arbitrary JSON value iteratively. This is a state machine driven From b07f74bd732d0a4b424341279575323b3cee426f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 5 Jun 2026 18:23:15 +0200 Subject: [PATCH 338/506] parser.c: Extract json_match_keyword Extracted from: https://github.com/ruby/json/pull/994 Modern compilers shouldn't have problem computing `strlen` at compile time and generating the same code. --- ext/json/ext/parser/parser.c | 52 +++++++++++++++++-------------- test/json/json_ext_parser_test.rb | 2 +- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b820066c5..a61f997a1 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1425,9 +1425,7 @@ static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_Par static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config) { - const char *start = state->cursor; - state->cursor++; - return json_parse_number(state, config, true, start); + return json_parse_number(state, config, true, state->cursor - 1); } // How many values (array elements, or interleaved object keys+values) have been @@ -1452,6 +1450,22 @@ static inline void json_value_completed(json_frame *frame) frame->phase = (enum json_frame_phase) frame->type; } +static inline bool json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset) +{ + // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this + // `strlen` and several other computations in that routine, such as eliminating the `if (resumable)` branch. + + size_t len = strlen(keyword); + + // Note: memcmp with a small power of two and a literal string compile to an integer comparison / + // That's why we sometime compare starting from the first byte and sometimes from the second. + if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) { + state->cursor += len; + return true; + } + return false; +} + // Parse an arbitrary JSON value iteratively. This is a state machine driven // entirely by the top frame's phase so it can stop at any value boundary and // resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the @@ -1475,8 +1489,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) switch (peek(state)) { case 'n': - if (rest(state) >= 4 && (memcmp(state->cursor, "null", 4) == 0)) { - state->cursor += 4; + if (json_match_keyword(state, "null", 0)) { json_push_value(state, config, Qnil); json_value_completed(frame); break; @@ -1484,8 +1497,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); case 't': - if (rest(state) >= 4 && (memcmp(state->cursor, "true", 4) == 0)) { - state->cursor += 4; + if (json_match_keyword(state, "true", 0)) { json_push_value(state, config, Qtrue); json_value_completed(frame); break; @@ -1493,9 +1505,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); case 'f': - // Note: memcmp with a small power of two compile to an integer comparison - if (rest(state) >= 5 && (memcmp(state->cursor + 1, "alse", 4) == 0)) { - state->cursor += 5; + if (json_match_keyword(state, "false", 1)) { json_push_value(state, config, Qfalse); json_value_completed(frame); break; @@ -1504,8 +1514,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); case 'N': // Note: memcmp with a small power of two compile to an integer comparison - if (config->allow_nan && rest(state) >= 3 && (memcmp(state->cursor + 1, "aN", 2) == 0)) { - state->cursor += 3; + if (config->allow_nan && json_match_keyword(state, "NaN", 1)) { json_push_value(state, config, CNaN); json_value_completed(frame); break; @@ -1513,8 +1522,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); case 'I': - if (config->allow_nan && rest(state) >= 8 && (memcmp(state->cursor, "Infinity", 8) == 0)) { - state->cursor += 8; + if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { json_push_value(state, config, CInfinity); json_value_completed(frame); break; @@ -1522,17 +1530,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("unexpected token %s", state); case '-': { - // Note: memcmp with a small power of two compile to an integer comparison - if (rest(state) >= 9 && (memcmp(state->cursor + 1, "Infinity", 8) == 0)) { - if (config->allow_nan) { - state->cursor += 9; - json_push_value(state, config, CMinusInfinity); - json_value_completed(frame); - break; - } else { - raise_parse_error("unexpected token %s", state); - } + state->cursor++; + if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { + json_push_value(state, config, CMinusInfinity); + json_value_completed(frame); + break; } + json_push_value(state, config, json_parse_negative_number(state, config)); json_value_completed(frame); break; diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index e610f642f..d585b8d0d 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -26,7 +26,7 @@ def test_error_messages ex = assert_raise(ParserError) { parse('-Infinity something') } unless RUBY_PLATFORM =~ /java/ - assert_equal "unexpected token '-Infinity' at line 1 column 1", ex.message + assert_equal "invalid number: '-Infinity' at line 1 column 1", ex.message end ex = assert_raise(ParserError) { parse('NaN something') } From 65904e8825cf31bc682a28adbdf7e09206b1e0b8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 5 Jun 2026 18:57:19 +0200 Subject: [PATCH 339/506] parser.c: refactor json_push_value / json_value_completed Makes each case simpler, but also more consistent. --- ext/json/ext/parser/parser.c | 54 +++++++++++++++++------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a61f997a1..308b47c37 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1236,7 +1236,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi case '"': { VALUE string = json_string_unescape(state, config, start, state->cursor, is_name, &positions); state->cursor++; - return json_push_value(state, config, string); + return string; } case '\\': { if (RB_LIKELY(positions.size < JSON_MAX_UNESCAPE_POSITIONS)) { @@ -1271,12 +1271,16 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars raise_parse_error("unexpected end of input, expected closing \"", state); } + VALUE string; if (RB_LIKELY(*state->cursor == '"')) { - VALUE string = json_string_fastpath(state, config, start, state->cursor, is_name); + string = json_string_fastpath(state, config, start, state->cursor, is_name); state->cursor++; - return json_push_value(state, config, string); } - return json_parse_escaped_string(state, config, is_name, start); + else { + string = json_parse_escaped_string(state, config, is_name, start); + } + + return string; } #if JSON_CPU_LITTLE_ENDIAN_64BITS @@ -1487,27 +1491,25 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_PHASE_VALUE: json_eat_whitespace(state); + VALUE value; switch (peek(state)) { case 'n': if (json_match_keyword(state, "null", 0)) { - json_push_value(state, config, Qnil); - json_value_completed(frame); + value = Qnil; break; } raise_parse_error("unexpected token %s", state); case 't': if (json_match_keyword(state, "true", 0)) { - json_push_value(state, config, Qtrue); - json_value_completed(frame); + value = Qtrue; break; } raise_parse_error("unexpected token %s", state); case 'f': if (json_match_keyword(state, "false", 1)) { - json_push_value(state, config, Qfalse); - json_value_completed(frame); + value = Qfalse; break; } @@ -1515,16 +1517,14 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case 'N': // Note: memcmp with a small power of two compile to an integer comparison if (config->allow_nan && json_match_keyword(state, "NaN", 1)) { - json_push_value(state, config, CNaN); - json_value_completed(frame); + value = CNaN; break; } raise_parse_error("unexpected token %s", state); case 'I': if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { - json_push_value(state, config, CInfinity); - json_value_completed(frame); + value = CInfinity; break; } @@ -1532,23 +1532,18 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case '-': { state->cursor++; if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { - json_push_value(state, config, CMinusInfinity); - json_value_completed(frame); - break; + value = CMinusInfinity; + } else { + value = json_parse_negative_number(state, config); } - - json_push_value(state, config, json_parse_negative_number(state, config)); - json_value_completed(frame); break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - json_push_value(state, config, json_parse_positive_number(state, config)); - json_value_completed(frame); + value = json_parse_positive_number(state, config); break; case '"': // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} - json_parse_string(state, config, false); - json_value_completed(frame); + value = json_parse_string(state, config, false); break; case '[': { state->cursor++; @@ -1556,8 +1551,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if (peek(state) == ']') { state->cursor++; - json_push_value(state, config, json_decode_array(state, config, 0)); - json_value_completed(frame); + value = json_decode_array(state, config, 0); break; } @@ -1584,8 +1578,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if (peek(state) == '}') { state->cursor++; - json_push_value(state, config, json_decode_object(state, config, 0)); - json_value_completed(frame); + value = json_decode_object(state, config, 0); break; } @@ -1611,6 +1604,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) default: raise_parse_error("unexpected character: %s", state); } + + json_push_value(state, config, value); + json_value_completed(frame); break; } @@ -1621,7 +1617,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_eat_whitespace(state); if (RB_LIKELY(peek(state) == '"')) { - json_parse_string(state, config, true); + json_push_value(state, config, json_parse_string(state, config, true)); frame->phase = JSON_PHASE_OBJECT_COLON; goto JSON_PHASE_OBJECT_COLON; } else { From 7b284d4c1d5aa0d13685b66ec90dfb88f4f9ba46 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 5 Jun 2026 23:13:47 +0200 Subject: [PATCH 340/506] ALWAYS_INLINE for json_match_keyword() --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 308b47c37..fbfbfac9d 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1454,7 +1454,7 @@ static inline void json_value_completed(json_frame *frame) frame->phase = (enum json_frame_phase) frame->type; } -static inline bool json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset) +ALWAYS_INLINE(static) bool json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset) { // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this // `strlen` and several other computations in that routine, such as eliminating the `if (resumable)` branch. From ad6b9cff9eea22bd4b28b87f47a12a973feded27 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 6 Jun 2026 08:52:23 +0200 Subject: [PATCH 341/506] json_parse_any: eliminate the loop and switch Each phase now takes care of dispatching to the next one. --- ext/json/ext/parser/parser.c | 457 +++++++++++++++++++---------------- 1 file changed, 245 insertions(+), 212 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index fbfbfac9d..e5f702612 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1477,258 +1477,291 @@ ALWAYS_INLINE(static) bool json_match_keyword(JSON_ParserState *state, const cha // itself is just another frame whose value, once parsed, leaves its phase DONE. static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) { - while (true) { - json_frame *frame = json_frame_stack_peek(state->frames); - - switch (frame->phase) { - case JSON_PHASE_DONE: { - // The root document value is parsed; it is the lone survivor on - // the rvalue stack. - return *rvalue_stack_peek(state->value_stack, 1); - } - - case JSON_PHASE_VALUE: { - JSON_PHASE_VALUE: - json_eat_whitespace(state); - - VALUE value; - switch (peek(state)) { - case 'n': - if (json_match_keyword(state, "null", 0)) { - value = Qnil; - break; - } - - raise_parse_error("unexpected token %s", state); - case 't': - if (json_match_keyword(state, "true", 0)) { - value = Qtrue; - break; - } - - raise_parse_error("unexpected token %s", state); - case 'f': - if (json_match_keyword(state, "false", 1)) { - value = Qfalse; - break; - } - - raise_parse_error("unexpected token %s", state); - case 'N': - // Note: memcmp with a small power of two compile to an integer comparison - if (config->allow_nan && json_match_keyword(state, "NaN", 1)) { - value = CNaN; - break; - } - - raise_parse_error("unexpected token %s", state); - case 'I': - if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { - value = CInfinity; - break; - } + json_frame *frame = json_frame_stack_peek(state->frames); - raise_parse_error("unexpected token %s", state); - case '-': { - state->cursor++; - if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { - value = CMinusInfinity; - } else { - value = json_parse_negative_number(state, config); - } - break; - } - case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - value = json_parse_positive_number(state, config); - break; - case '"': - // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} - value = json_parse_string(state, config, false); - break; - case '[': { - state->cursor++; - json_eat_whitespace(state); - - if (peek(state) == ']') { - state->cursor++; - value = json_decode_array(state, config, 0); - break; - } + switch (frame->phase) { + case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; + case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; + case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; + case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY; + case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; + } + UNREACHABLE_RETURN(Qundef); - state->current_nesting++; - if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { - rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); - } - state->in_array++; - - // Phase stays VALUE: the next iteration reads the first element. - frame = json_frame_stack_push(state, (json_frame){ - .type = JSON_FRAME_ARRAY, - .phase = JSON_PHASE_VALUE, - .value_stack_head = state->value_stack->head, - }); - goto JSON_PHASE_VALUE; - break; - } - case '{': { - const char *object_start_cursor = state->cursor; + JSON_PHASE_DONE: { + // The root document value is parsed; it is the lone survivor on + // the rvalue stack. + return *rvalue_stack_peek(state->value_stack, 1); + } - state->cursor++; - json_eat_whitespace(state); + JSON_PHASE_VALUE: { + json_eat_whitespace(state); - if (peek(state) == '}') { - state->cursor++; - value = json_decode_object(state, config, 0); - break; - } + VALUE value; + switch (peek(state)) { + case 'n': + if (json_match_keyword(state, "null", 0)) { + value = Qnil; + break; + } + raise_parse_error("unexpected token %s", state); - state->current_nesting++; - if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { - rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); - } + case 't': + if (json_match_keyword(state, "true", 0)) { + value = Qtrue; + break; + } + raise_parse_error("unexpected token %s", state); - // Phase KEY: the next iteration reads the first key. - frame = json_frame_stack_push(state, (json_frame){ - .type = JSON_FRAME_OBJECT, - .phase = JSON_PHASE_OBJECT_KEY, - .value_stack_head = state->value_stack->head, - .start_cursor = object_start_cursor, - }); - goto JSON_PHASE_OBJECT_KEY; - break; - } + case 'f': + if (json_match_keyword(state, "false", 1)) { + value = Qfalse; + break; + } + raise_parse_error("unexpected token %s", state); - case 0: - raise_parse_error("unexpected end of input", state); + case 'N': + // Note: memcmp with a small power of two compile to an integer comparison + if (config->allow_nan && json_match_keyword(state, "NaN", 1)) { + value = CNaN; + break; + } + raise_parse_error("unexpected token %s", state); - default: - raise_parse_error("unexpected character: %s", state); + case 'I': + if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { + value = CInfinity; + break; } + raise_parse_error("unexpected token %s", state); - json_push_value(state, config, value); - json_value_completed(frame); + case '-': { + state->cursor++; + if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { + value = CMinusInfinity; + } else { + value = json_parse_negative_number(state, config); + } break; } - case JSON_PHASE_OBJECT_KEY: { - JSON_PHASE_OBJECT_KEY: - JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + value = json_parse_positive_number(state, config); + break; + case '"': + // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} + value = json_parse_string(state, config, false); + break; + + case '[': { + state->cursor++; json_eat_whitespace(state); - if (RB_LIKELY(peek(state) == '"')) { - json_push_value(state, config, json_parse_string(state, config, true)); - frame->phase = JSON_PHASE_OBJECT_COLON; - goto JSON_PHASE_OBJECT_COLON; - } else { - // The message differs for the first key vs. a key after a - // ',': the first is the only one reached with nothing pushed - // for this object yet. - if (json_frame_entry_count(frame, state->value_stack) == 0) { - raise_parse_error("expected object key, got %s", state); - } else { - raise_parse_error("expected object key, got: %s", state); - } + if (peek(state) == ']') { + state->cursor++; + value = json_decode_array(state, config, 0); + break; } - break; - } - case JSON_PHASE_OBJECT_COLON: { - JSON_PHASE_OBJECT_COLON: - JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + state->current_nesting++; + if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { + rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); + } + state->in_array++; + + // Phase stays VALUE: the next iteration reads the first element. + frame = json_frame_stack_push(state, (json_frame){ + .type = JSON_FRAME_ARRAY, + .phase = JSON_PHASE_VALUE, + .value_stack_head = state->value_stack->head, + }); + goto JSON_PHASE_VALUE; + } + case '{': { + const char *object_start_cursor = state->cursor; + state->cursor++; json_eat_whitespace(state); - if (RB_LIKELY(peek(state) == ':')) { + if (peek(state) == '}') { state->cursor++; - frame->phase = JSON_PHASE_VALUE; - goto JSON_PHASE_VALUE; - } else { - // First colon (only the first pair's key is pushed, nothing - // else) vs. a later one. - if (json_frame_entry_count(frame, state->value_stack) == 1) { - raise_parse_error("expected ':' after object key", state); - } else { - raise_parse_error("expected ':' after object key, got: %s", state); - } + value = json_decode_object(state, config, 0); + break; } - break; + + state->current_nesting++; + if (RB_UNLIKELY(config->max_nesting && (config->max_nesting < state->current_nesting))) { + rb_raise(eNestingError, "nesting of %d is too deep", state->current_nesting); + } + + // Phase KEY: the next iteration reads the first key. + frame = json_frame_stack_push(state, (json_frame){ + .type = JSON_FRAME_OBJECT, + .phase = JSON_PHASE_OBJECT_KEY, + .value_stack_head = state->value_stack->head, + .start_cursor = object_start_cursor, + }); + goto JSON_PHASE_OBJECT_KEY; } - case JSON_PHASE_ARRAY_COMMA: { - JSON_ASSERT(frame->type == JSON_FRAME_ARRAY); + case 0: + raise_parse_error("unexpected end of input", state); - json_eat_whitespace(state); + default: + raise_parse_error("unexpected character: %s", state); + } - const char next_char = peek(state); + json_push_value(state, config, value); + json_value_completed(frame); - if (RB_LIKELY(next_char == ',')) { - state->cursor++; - if (config->allow_trailing_comma) { - json_eat_whitespace(state); - if (peek(state) == ']') { - // Trailing comma: stay in COMMA to close on the next iteration. - break; - } - } - frame->phase = JSON_PHASE_VALUE; - goto JSON_PHASE_VALUE; - } else if (next_char == ']') { - state->cursor++; - long count = json_frame_entry_count(frame, state->value_stack); - state->current_nesting--; - state->in_array--; - json_frame_stack_pop(state->frames); - json_push_value(state, config, json_decode_array(state, config, count)); - json_value_completed(json_frame_stack_peek(state->frames)); - } else { - raise_parse_error("expected ',' or ']' after array value", state); - } - break; + switch (frame->phase) { + case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; + case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; + case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; + case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; + } + UNREACHABLE_RETURN(Qundef); + } + + JSON_PHASE_OBJECT_KEY: { + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + + json_eat_whitespace(state); + + if (RB_LIKELY(peek(state) == '"')) { + json_push_value(state, config, json_parse_string(state, config, true)); + frame->phase = JSON_PHASE_OBJECT_COLON; + goto JSON_PHASE_OBJECT_COLON; + } else { + // The message differs for the first key vs. a key after a + // ',': the first is the only one reached with nothing pushed + // for this object yet. + if (json_frame_entry_count(frame, state->value_stack) == 0) { + raise_parse_error("expected object key, got %s", state); + } else { + raise_parse_error("expected object key, got: %s", state); + } + } + UNREACHABLE_RETURN(Qundef); + } + + JSON_PHASE_OBJECT_COLON: { + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + + json_eat_whitespace(state); + + if (RB_LIKELY(peek(state) == ':')) { + state->cursor++; + frame->phase = JSON_PHASE_VALUE; + goto JSON_PHASE_VALUE; + } else { + // First colon (only the first pair's key is pushed, nothing + // else) vs. a later one. + if (json_frame_entry_count(frame, state->value_stack) == 1) { + raise_parse_error("expected ':' after object key", state); + } else { + raise_parse_error("expected ':' after object key, got: %s", state); } + } + UNREACHABLE_RETURN(Qundef); + } + + JSON_PHASE_ARRAY_COMMA: { + JSON_ASSERT(frame->type == JSON_FRAME_ARRAY); - case JSON_PHASE_OBJECT_COMMA: { - JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); + json_eat_whitespace(state); + + const char next_char = peek(state); + if (RB_LIKELY(next_char == ',')) { + state->cursor++; + if (config->allow_trailing_comma) { json_eat_whitespace(state); - const char next_char = peek(state); + if (peek(state) == ']') { + // Trailing comma: stay in COMMA to close on the next iteration. + goto JSON_PHASE_ARRAY_COMMA; + } + } + frame->phase = JSON_PHASE_VALUE; + goto JSON_PHASE_VALUE; + } else if (next_char == ']') { + state->cursor++; + long count = json_frame_entry_count(frame, state->value_stack); + state->current_nesting--; + state->in_array--; + json_frame_stack_pop(state->frames); + json_push_value(state, config, json_decode_array(state, config, count)); + frame = json_frame_stack_peek(state->frames); + json_value_completed(frame); + + switch (frame->phase) { + case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; + case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; + case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; + case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; + } + } else { + raise_parse_error("expected ',' or ']' after array value", state); + } + UNREACHABLE_RETURN(Qundef); + } - if (RB_LIKELY(next_char == ',')) { - state->cursor++; - json_eat_whitespace(state); + JSON_PHASE_OBJECT_COMMA: { + JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - if (config->allow_trailing_comma) { - if (peek(state) == '}') { - // Trailing comma: stay in COMMA to close on the next iteration. - break; - } - } + json_eat_whitespace(state); + const char next_char = peek(state); - frame->phase = JSON_PHASE_OBJECT_KEY; - goto JSON_PHASE_OBJECT_KEY; + if (RB_LIKELY(next_char == ',')) { + state->cursor++; - break; - } else if (next_char == '}') { - state->cursor++; - state->current_nesting--; - size_t count = json_frame_entry_count(frame, state->value_stack); - - // Temporary rewind cursor in case an error is raised - const char *final_cursor = state->cursor; - state->cursor = frame->start_cursor; - VALUE object = json_decode_object(state, config, count); - state->cursor = final_cursor; - - json_frame_stack_pop(state->frames); - json_push_value(state, config, object); - json_value_completed(json_frame_stack_peek(state->frames)); - break; - } else { - raise_parse_error("expected ',' or '}' after object value, got: %s", state); + if (config->allow_trailing_comma) { + json_eat_whitespace(state); + if (peek(state) == '}') { + // Trailing comma: stay in COMMA to close on the next iteration. + goto JSON_PHASE_OBJECT_COMMA; } } + + frame->phase = JSON_PHASE_OBJECT_KEY; + goto JSON_PHASE_OBJECT_KEY; + } else if (next_char == '}') { + state->cursor++; + state->current_nesting--; + size_t count = json_frame_entry_count(frame, state->value_stack); + + // Temporary rewind cursor in case an error is raised + const char *final_cursor = state->cursor; + state->cursor = frame->start_cursor; + VALUE object = json_decode_object(state, config, count); + state->cursor = final_cursor; + + json_push_value(state, config, object); + json_frame_stack_pop(state->frames); + frame = json_frame_stack_peek(state->frames); + json_value_completed(frame); + + switch (frame->phase) { + case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; + case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; + case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; + case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; + } + } else { + raise_parse_error("expected ',' or '}' after object value, got: %s", state); } + UNREACHABLE_RETURN(Qundef); } + + UNREACHABLE_RETURN(Qundef); } static void json_ensure_eof(JSON_ParserState *state) From 6444417670cd765604b4eab44e4e36180d2cafdf Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 6 Jun 2026 10:45:11 +0200 Subject: [PATCH 342/506] Add note explaining why rvalue_stack is not WB_PROTECTED --- ext/json/ext/parser/parser.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e5f702612..c0631728c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -294,6 +294,8 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = { .dsize = rvalue_stack_memsize, .dcompact = rvalue_stack_compact, }, + // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED + // because it churns a lot of values so trigering write barriers every time is very costly. .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, }; From 8d7f975b012054c1c8073b4444c6f533ce6d0e8b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 7 Jun 2026 10:11:23 +0200 Subject: [PATCH 343/506] Mark JSON_Parser_frame_stack_type as WB protected It already implictly is on recent rubies because it has no mark function, but might as well make it explicit. --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c0631728c..524b80200 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -488,7 +488,7 @@ static const rb_data_type_t JSON_Parser_frame_stack_type = { .dfree = json_frame_stack_free, .dsize = json_frame_stack_memsize, }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, }; static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref) From bbcf0a3254037f6e3a024707475b1c109e58cf4f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 7 Jun 2026 10:07:27 +0200 Subject: [PATCH 344/506] Compile UNREACHABLE_RETURN into `rb_bug` when in debug mode This makes it much easier to debug. --- ext/json/ext/json.h | 8 ++++++++ ext/json/ext/parser/parser.c | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index cf9420d4d..78b0eee0a 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -11,6 +11,9 @@ #if defined(RUBY_DEBUG) && RUBY_DEBUG # define JSON_ASSERT RUBY_ASSERT +# ifndef JSON_DEBUG +# define JSON_DEBUG 1 +# endif #else # ifdef JSON_DEBUG # include @@ -20,6 +23,11 @@ # endif #endif +#ifdef JSON_DEBUG +# define JSON_UNREACHABLE_RETURN(val) rb_bug("Unreachable") +#else +# define JSON_UNREACHABLE_RETURN UNREACHABLE_RETURN +#endif /* shims */ #if SIZEOF_UINT64_T == SIZEOF_LONG_LONG diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 524b80200..6b8164c06 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1489,7 +1489,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY; case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); JSON_PHASE_DONE: { // The root document value is parsed; it is the lone survivor on @@ -1623,10 +1623,10 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } JSON_PHASE_OBJECT_KEY: { @@ -1648,7 +1648,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("expected object key, got: %s", state); } } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } JSON_PHASE_OBJECT_COLON: { @@ -1669,7 +1669,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) raise_parse_error("expected ':' after object key, got: %s", state); } } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } JSON_PHASE_ARRAY_COMMA: { @@ -1705,13 +1705,13 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } } else { raise_parse_error("expected ',' or ']' after array value", state); } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } JSON_PHASE_OBJECT_COMMA: { @@ -1754,16 +1754,16 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } } else { raise_parse_error("expected ',' or '}' after object value, got: %s", state); } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } - UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(Qundef); } static void json_ensure_eof(JSON_ParserState *state) From 138b9a2c4932ad2df1024ce3af18a5cc3ae9e901 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 7 Jun 2026 10:38:16 +0200 Subject: [PATCH 345/506] Deprecate default support of JavaScript comments in the parser Add the `allow_comments: true` parsing option. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 59 +++--- java/src/json/ext/ParserConfig.java | 289 +++++++++++++++++----------- java/src/json/ext/ParserConfig.rl | 30 ++- lib/json.rb | 18 +- test/json/json_parser_test.rb | 30 ++- 6 files changed, 280 insertions(+), 147 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c4808e535..69bdb076d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* Deprecate default support of JavaScript comments in the parser and add `allow_comments: true` parsing option. * Integrate with Ruby 4.1 `ruby_sized_xfree`. ### 2026-06-03 (2.19.8) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 6b8164c06..dc76ca2cd 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -7,9 +7,9 @@ static VALUE CNaN, CInfinity, CMinusInfinity; static ID i_new, i_try_convert, i_uminus, i_encode; -static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_control_characters, - sym_allow_invalid_escape, sym_symbolize_names, sym_freeze, sym_decimal_class, sym_on_load, - sym_allow_duplicate_key; +static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, + sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, + sym_freeze, sym_decimal_class, sym_on_load, sym_allow_duplicate_key; static int binary_encindex; static int utf8_encindex; @@ -382,7 +382,7 @@ typedef struct json_frame_stack_struct { json_frame *ptr; } json_frame_stack; -enum duplicate_key_action { +enum deprecatable_action { JSON_DEPRECATED = 0, JSON_IGNORE, JSON_RAISE, @@ -392,7 +392,8 @@ typedef struct JSON_ParserStruct { VALUE on_load_proc; VALUE decimal_class; ID decimal_method_id; - enum duplicate_key_action on_duplicate_key; + enum deprecatable_action on_duplicate_key; + enum deprecatable_action on_comment; int max_nesting; bool allow_nan; bool allow_trailing_comma; @@ -590,6 +591,8 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum *column_out = column; } +static const unsigned int MAX_DEPRECATIONS = 5; + static void emit_parse_warning(const char *message, JSON_ParserState *state) { long line, column; @@ -707,9 +710,14 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const static const rb_data_type_t JSON_ParserConfig_type; +const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`"; NOINLINE(static) void -json_eat_comments(JSON_ParserState *state) +json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) { + if (config->on_comment == JSON_RAISE) { + raise_parse_error("unexpected token %s", state); + } + const char *start = state->cursor; state->cursor++; @@ -744,10 +752,15 @@ json_eat_comments(JSON_ParserState *state) raise_parse_error_at("unexpected token %s", state, start); break; } + + if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) { + state->emitted_deprecations++; + emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state); + } } ALWAYS_INLINE(static) void -json_eat_whitespace(JSON_ParserState *state) +json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config) { while (true) { switch (peek(state)) { @@ -778,7 +791,7 @@ json_eat_whitespace(JSON_ParserState *state) state->cursor++; break; case '/': - json_eat_comments(state); + json_eat_comments(state, config); break; default: @@ -1127,9 +1140,9 @@ NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_Parser case JSON_DEPRECATED: // Only emit the first few deprecations to avoid spamming. - if (state->emitted_deprecations < 5) { - emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); + if (state->emitted_deprecations < MAX_DEPRECATIONS) { state->emitted_deprecations++; + emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); } return; @@ -1498,7 +1511,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) } JSON_PHASE_VALUE: { - json_eat_whitespace(state); + json_eat_whitespace(state, config); VALUE value; switch (peek(state)) { @@ -1559,7 +1572,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) case '[': { state->cursor++; - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (peek(state) == ']') { state->cursor++; @@ -1585,7 +1598,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) const char *object_start_cursor = state->cursor; state->cursor++; - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (peek(state) == '}') { state->cursor++; @@ -1632,7 +1645,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_PHASE_OBJECT_KEY: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (RB_LIKELY(peek(state) == '"')) { json_push_value(state, config, json_parse_string(state, config, true)); @@ -1654,7 +1667,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_PHASE_OBJECT_COLON: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (RB_LIKELY(peek(state) == ':')) { state->cursor++; @@ -1675,14 +1688,14 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_PHASE_ARRAY_COMMA: { JSON_ASSERT(frame->type == JSON_FRAME_ARRAY); - json_eat_whitespace(state); + json_eat_whitespace(state, config); const char next_char = peek(state); if (RB_LIKELY(next_char == ',')) { state->cursor++; if (config->allow_trailing_comma) { - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (peek(state) == ']') { // Trailing comma: stay in COMMA to close on the next iteration. goto JSON_PHASE_ARRAY_COMMA; @@ -1717,14 +1730,14 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_PHASE_OBJECT_COMMA: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state); + json_eat_whitespace(state, config); const char next_char = peek(state); if (RB_LIKELY(next_char == ',')) { state->cursor++; if (config->allow_trailing_comma) { - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (peek(state) == '}') { // Trailing comma: stay in COMMA to close on the next iteration. goto JSON_PHASE_OBJECT_COMMA; @@ -1766,9 +1779,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) JSON_UNREACHABLE_RETURN(Qundef); } -static void json_ensure_eof(JSON_ParserState *state) +static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config) { - json_eat_whitespace(state); + json_eat_whitespace(state, config); if (!eos(state)) { raise_parse_error("unexpected token at end of stream %s", state); } @@ -1825,6 +1838,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; } else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } + else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); } else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } @@ -1977,7 +1991,7 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) RB_GC_GUARD(value_stack_handle); RB_GC_GUARD(frame_stack_handle); RB_GC_GUARD(Vsource); - json_ensure_eof(state); + json_ensure_eof(state, config); return result; } @@ -2079,6 +2093,7 @@ void Init_parser(void) sym_max_nesting = ID2SYM(rb_intern("max_nesting")); sym_allow_nan = ID2SYM(rb_intern("allow_nan")); sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma")); + sym_allow_comments = ID2SYM(rb_intern("allow_comments")); sym_allow_control_characters = ID2SYM(rb_intern("allow_control_characters")); sym_allow_invalid_escape = ID2SYM(rb_intern("allow_invalid_escape")); sym_symbolize_names = ID2SYM(rb_intern("symbolize_names")); diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java index 1fee0a596..562c0ab61 100644 --- a/java/src/json/ext/ParserConfig.java +++ b/java/src/json/ext/ParserConfig.java @@ -54,6 +54,8 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowComments; + private boolean deprecateComments; private boolean allowControlCharacters; private boolean allowInvalidEscape; private boolean allowDuplicateKey; @@ -180,6 +182,14 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { OptionsReader opts = new OptionsReader(context, options); this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); + if (opts.hasKey("allow_comments")) { + this.allowComments = opts.getBool("allow_comments", false); + this.deprecateComments = false; + } else { + this.allowComments = true; + this.deprecateComments = true; + } + this.allowControlCharacters = opts.getBool("allow_control_characters", false); this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); @@ -308,11 +318,11 @@ private RaiseException unexpectedToken(ThreadContext context, int absStart, int } -// line 334 "ParserConfig.rl" +// line 360 "ParserConfig.rl" -// line 316 "ParserConfig.java" +// line 326 "ParserConfig.java" private static byte[] init__JSON_value_actions_0() { return new byte [] { @@ -426,7 +436,7 @@ private static byte[] init__JSON_value_from_state_actions_0() static final int JSON_value_en_main = 1; -// line 440 "ParserConfig.rl" +// line 466 "ParserConfig.rl" void parseValue(ThreadContext context, ParserResult res, int p, int pe) { @@ -434,14 +444,14 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 438 "ParserConfig.java" +// line 448 "ParserConfig.java" { cs = JSON_value_start; } -// line 447 "ParserConfig.rl" +// line 473 "ParserConfig.rl" -// line 445 "ParserConfig.java" +// line 455 "ParserConfig.java" { int _klen; int _trans = 0; @@ -467,13 +477,13 @@ void parseValue(ThreadContext context, ParserResult res, int p, int pe) { while ( _nacts-- > 0 ) { switch ( _JSON_value_actions[_acts++] ) { case 9: -// line 425 "ParserConfig.rl" +// line 451 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 477 "ParserConfig.java" +// line 487 "ParserConfig.java" } } @@ -536,25 +546,25 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) switch ( _JSON_value_actions[_acts++] ) { case 0: -// line 342 "ParserConfig.rl" +// line 368 "ParserConfig.rl" { result = context.nil; } break; case 1: -// line 345 "ParserConfig.rl" +// line 371 "ParserConfig.rl" { result = context.fals; } break; case 2: -// line 348 "ParserConfig.rl" +// line 374 "ParserConfig.rl" { result = context.tru; } break; case 3: -// line 351 "ParserConfig.rl" +// line 377 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_NAN); @@ -564,7 +574,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 4: -// line 358 "ParserConfig.rl" +// line 384 "ParserConfig.rl" { if (config.allowNaN) { result = getConstant(CONST_INFINITY); @@ -574,7 +584,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 5: -// line 365 "ParserConfig.rl" +// line 391 "ParserConfig.rl" { if (pe > p + 8 && absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { @@ -603,7 +613,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 6: -// line 391 "ParserConfig.rl" +// line 417 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -616,7 +626,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 7: -// line 401 "ParserConfig.rl" +// line 427 "ParserConfig.rl" { currentNesting++; parseArray(context, res, p, pe); @@ -631,7 +641,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } break; case 8: -// line 413 "ParserConfig.rl" +// line 439 "ParserConfig.rl" { currentNesting++; parseObject(context, res, p, pe); @@ -645,7 +655,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } } break; -// line 649 "ParserConfig.java" +// line 659 "ParserConfig.java" } } } @@ -665,7 +675,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) break; } } -// line 448 "ParserConfig.rl" +// line 474 "ParserConfig.rl" if (cs >= JSON_value_first_final && result != null) { if (config.freeze) { @@ -678,7 +688,7 @@ else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) } -// line 682 "ParserConfig.java" +// line 692 "ParserConfig.java" private static byte[] init__JSON_integer_actions_0() { return new byte [] { @@ -777,7 +787,7 @@ private static byte[] init__JSON_integer_trans_actions_0() static final int JSON_integer_en_main = 1; -// line 470 "ParserConfig.rl" +// line 496 "ParserConfig.rl" void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { @@ -794,15 +804,15 @@ int parseIntegerInternal(int p, int pe) { int cs; -// line 798 "ParserConfig.java" +// line 808 "ParserConfig.java" { cs = JSON_integer_start; } -// line 486 "ParserConfig.rl" +// line 512 "ParserConfig.rl" int memo = p; -// line 806 "ParserConfig.java" +// line 816 "ParserConfig.java" { int _klen; int _trans = 0; @@ -883,13 +893,13 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) switch ( _JSON_integer_actions[_acts++] ) { case 0: -// line 464 "ParserConfig.rl" +// line 490 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 893 "ParserConfig.java" +// line 903 "ParserConfig.java" } } } @@ -909,7 +919,7 @@ else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) break; } } -// line 488 "ParserConfig.rl" +// line 514 "ParserConfig.rl" if (cs < JSON_integer_first_final) { return -1; @@ -929,7 +939,7 @@ RubyInteger bytesToInum(Ruby runtime, ByteList num) { } -// line 933 "ParserConfig.java" +// line 943 "ParserConfig.java" private static byte[] init__JSON_float_actions_0() { return new byte [] { @@ -1031,7 +1041,7 @@ private static byte[] init__JSON_float_trans_actions_0() static final int JSON_float_en_main = 1; -// line 521 "ParserConfig.rl" +// line 547 "ParserConfig.rl" void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { @@ -1050,15 +1060,15 @@ int parseFloatInternal(int p, int pe) { int cs; -// line 1054 "ParserConfig.java" +// line 1064 "ParserConfig.java" { cs = JSON_float_start; } -// line 539 "ParserConfig.rl" +// line 565 "ParserConfig.rl" int memo = p; -// line 1062 "ParserConfig.java" +// line 1072 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1139,13 +1149,13 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) switch ( _JSON_float_actions[_acts++] ) { case 0: -// line 512 "ParserConfig.rl" +// line 538 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1149 "ParserConfig.java" +// line 1159 "ParserConfig.java" } } } @@ -1165,7 +1175,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) break; } } -// line 541 "ParserConfig.rl" +// line 567 "ParserConfig.rl" if (cs < JSON_float_first_final) { return -1; @@ -1175,7 +1185,7 @@ else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) } -// line 1179 "ParserConfig.java" +// line 1189 "ParserConfig.java" private static byte[] init__JSON_string_actions_0() { return new byte [] { @@ -1277,7 +1287,7 @@ private static byte[] init__JSON_string_trans_actions_0() static final int JSON_string_en_main = 1; -// line 580 "ParserConfig.rl" +// line 606 "ParserConfig.rl" void parseString(ThreadContext context, ParserResult res, int p, int pe) { @@ -1285,15 +1295,15 @@ void parseString(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = null; -// line 1289 "ParserConfig.java" +// line 1299 "ParserConfig.java" { cs = JSON_string_start; } -// line 587 "ParserConfig.rl" +// line 613 "ParserConfig.rl" int memo = p; -// line 1297 "ParserConfig.java" +// line 1307 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1374,7 +1384,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) switch ( _JSON_string_actions[_acts++] ) { case 0: -// line 555 "ParserConfig.rl" +// line 581 "ParserConfig.rl" { int offset = byteList.begin(); ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, @@ -1389,13 +1399,13 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } break; case 1: -// line 568 "ParserConfig.rl" +// line 594 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1399 "ParserConfig.java" +// line 1409 "ParserConfig.java" } } } @@ -1415,7 +1425,7 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) break; } } -// line 589 "ParserConfig.rl" +// line 615 "ParserConfig.rl" if (cs >= JSON_string_first_final && result != null) { if (result instanceof RubyString) { @@ -1436,11 +1446,11 @@ else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) } -// line 1440 "ParserConfig.java" +// line 1450 "ParserConfig.java" private static byte[] init__JSON_array_actions_0() { return new byte [] { - 0, 1, 0, 1, 1 + 0, 1, 0, 1, 1, 1, 2 }; } @@ -1560,14 +1570,14 @@ private static byte[] init__JSON_array_indicies_0() return new byte [] { 0, 1, 0, 0, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 2, 0, 2, 1, 5, 5, 6, 4, 7, 8, - 5, 1, 9, 10, 1, 11, 9, 11, 5, 9, 5, 10, - 7, 7, 2, 2, 12, 2, 2, 2, 2, 2, 2, 2, - 7, 2, 1, 13, 14, 1, 15, 13, 15, 7, 13, 7, - 14, 2, 2, 2, 2, 2, 4, 2, 2, 2, 2, 0, - 0, 3, 8, 8, 16, 2, 0, 8, 1, 17, 18, 1, - 19, 17, 19, 0, 17, 0, 18, 17, 18, 20, 21, 1, - 19, 22, 17, 20, 1, 19, 0, 22, 8, 17, 20, 1, - 0, 8, 18, 21, 1, 1, 0 + 5, 1, 9, 10, 1, 11, 9, 11, 5, 9, 12, 10, + 7, 7, 2, 2, 13, 2, 2, 2, 2, 2, 2, 2, + 7, 2, 1, 14, 15, 1, 16, 14, 16, 7, 14, 17, + 15, 2, 2, 2, 2, 2, 4, 2, 2, 2, 2, 0, + 0, 3, 8, 8, 18, 2, 0, 8, 1, 19, 20, 1, + 21, 19, 21, 0, 19, 22, 20, 19, 20, 23, 24, 1, + 21, 25, 19, 23, 1, 21, 0, 25, 8, 19, 23, 1, + 22, 26, 20, 24, 1, 1, 0 }; } @@ -1578,7 +1588,8 @@ private static byte[] init__JSON_array_trans_targs_0() { return new byte [] { 2, 0, 3, 14, 22, 3, 4, 8, 13, 5, 7, 6, - 9, 10, 12, 11, 18, 15, 17, 16, 19, 21, 20 + 3, 9, 10, 12, 11, 8, 18, 15, 17, 16, 2, 19, + 21, 20, 13 }; } @@ -1588,8 +1599,9 @@ private static byte[] init__JSON_array_trans_targs_0() private static byte[] init__JSON_array_trans_actions_0() { return new byte [] { - 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 1, + 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, + 0, 1, 1 }; } @@ -1603,7 +1615,7 @@ private static byte[] init__JSON_array_trans_actions_0() static final int JSON_array_en_main = 1; -// line 643 "ParserConfig.rl" +// line 669 "ParserConfig.rl" void parseArray(ThreadContext context, ParserResult res, int p, int pe) { @@ -1617,14 +1629,14 @@ void parseArray(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyArray.newArray(context.runtime); -// line 1621 "ParserConfig.java" +// line 1633 "ParserConfig.java" { cs = JSON_array_start; } -// line 656 "ParserConfig.rl" +// line 682 "ParserConfig.rl" -// line 1628 "ParserConfig.java" +// line 1640 "ParserConfig.java" { int _klen; int _trans = 0; @@ -1667,7 +1679,7 @@ else if ( _widec > _JSON_array_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 614 "ParserConfig.rl" +// line 640 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -1737,7 +1749,24 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) switch ( _JSON_array_actions[_acts++] ) { case 0: -// line 616 "ParserConfig.rl" +// line 321 "ParserConfig.rl" + { + if (!config.allowComments) { + if (config.deprecateComments) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" + ); + } + } else { + throw unexpectedToken(context, p, pe); + } + } + } + break; + case 1: +// line 642 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -1749,14 +1778,14 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } } break; - case 1: -// line 627 "ParserConfig.rl" + case 2: +// line 653 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 1760 "ParserConfig.java" +// line 1789 "ParserConfig.java" } } } @@ -1776,7 +1805,7 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) break; } } -// line 657 "ParserConfig.rl" +// line 683 "ParserConfig.rl" if (cs >= JSON_array_first_final) { res.update(config.onLoad(context, result), p + 1); @@ -1786,11 +1815,11 @@ else if ( _widec > _JSON_array_trans_keys[_mid+1] ) } -// line 1790 "ParserConfig.java" +// line 1819 "ParserConfig.java" private static byte[] init__JSON_object_actions_0() { return new byte [] { - 0, 1, 0, 1, 1, 1, 2 + 0, 1, 0, 1, 1, 1, 2, 1, 3 }; } @@ -1916,16 +1945,16 @@ private static byte[] init__JSON_object_indicies_0() { return new byte [] { 0, 1, 0, 0, 2, 3, 4, 0, 1, 5, 5, 6, - 7, 5, 1, 8, 9, 1, 10, 8, 10, 5, 8, 5, - 9, 7, 7, 11, 11, 12, 11, 11, 11, 11, 11, 11, - 11, 7, 11, 1, 4, 13, 13, 14, 15, 16, 16, 0, - 17, 13, 16, 1, 13, 13, 14, 15, 4, 13, 1, 14, - 14, 2, 18, 14, 1, 19, 20, 1, 21, 19, 21, 14, - 19, 14, 20, 22, 23, 1, 24, 22, 24, 13, 22, 13, - 23, 22, 23, 25, 26, 1, 24, 27, 22, 25, 1, 24, - 13, 27, 16, 22, 25, 1, 13, 16, 23, 26, 1, 28, - 29, 1, 30, 28, 30, 7, 28, 7, 29, 31, 32, 1, - 33, 31, 33, 0, 31, 0, 32, 1, 0 + 7, 5, 1, 8, 9, 1, 10, 8, 10, 5, 8, 11, + 9, 7, 7, 12, 12, 13, 12, 12, 12, 12, 12, 12, + 12, 7, 12, 1, 4, 14, 14, 15, 16, 17, 17, 0, + 18, 14, 17, 1, 14, 14, 15, 16, 4, 14, 1, 15, + 15, 2, 19, 15, 1, 20, 21, 1, 22, 20, 22, 15, + 20, 23, 21, 24, 25, 1, 26, 24, 26, 14, 24, 27, + 25, 24, 25, 28, 29, 1, 26, 30, 24, 28, 1, 26, + 14, 30, 17, 24, 28, 1, 27, 31, 25, 29, 1, 32, + 33, 1, 34, 32, 34, 7, 32, 35, 33, 36, 37, 1, + 38, 36, 38, 0, 36, 39, 37, 1, 0 }; } @@ -1935,9 +1964,10 @@ private static byte[] init__JSON_object_indicies_0() private static byte[] init__JSON_object_trans_targs_0() { return new byte [] { - 2, 0, 3, 28, 32, 3, 4, 8, 5, 7, 6, 9, - 24, 10, 11, 16, 9, 20, 12, 13, 15, 14, 17, 19, - 18, 21, 23, 22, 25, 27, 26, 29, 31, 30 + 2, 0, 3, 28, 32, 3, 4, 8, 5, 7, 6, 3, + 9, 24, 10, 11, 16, 9, 20, 12, 13, 15, 14, 11, + 17, 19, 18, 10, 21, 23, 22, 9, 25, 27, 26, 8, + 29, 31, 30, 2 }; } @@ -1947,9 +1977,10 @@ private static byte[] init__JSON_object_trans_targs_0() private static byte[] init__JSON_object_trans_actions_0() { return new byte [] { - 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 1, 1, + 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, + 0, 0, 1, 1 }; } @@ -1963,7 +1994,7 @@ private static byte[] init__JSON_object_trans_actions_0() static final int JSON_object_en_main = 1; -// line 728 "ParserConfig.rl" +// line 754 "ParserConfig.rl" void parseObject(ThreadContext context, ParserResult res, int p, int pe) { @@ -1980,14 +2011,14 @@ void parseObject(ThreadContext context, ParserResult res, int p, int pe) { IRubyObject result = RubyHash.newHash(context.runtime); -// line 1984 "ParserConfig.java" +// line 2015 "ParserConfig.java" { cs = JSON_object_start; } -// line 744 "ParserConfig.rl" +// line 770 "ParserConfig.rl" -// line 1991 "ParserConfig.java" +// line 2022 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2030,7 +2061,7 @@ else if ( _widec > _JSON_object_cond_keys[_mid+1] ) case 0: { _widec = 65536 + (data[p] - 0); if ( -// line 671 "ParserConfig.rl" +// line 697 "ParserConfig.rl" config.allowTrailingComma ) _widec += 65536; break; } @@ -2100,7 +2131,24 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) switch ( _JSON_object_actions[_acts++] ) { case 0: -// line 673 "ParserConfig.rl" +// line 321 "ParserConfig.rl" + { + if (!config.allowComments) { + if (config.deprecateComments) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" + ); + } + } else { + throw unexpectedToken(context, p, pe); + } + } + } + break; + case 1: +// line 699 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2112,8 +2160,8 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } } break; - case 1: -// line 684 "ParserConfig.rl" + case 2: +// line 710 "ParserConfig.rl" { parseString(context, res, p, pe); if (res.result == null) { @@ -2144,14 +2192,14 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } } break; - case 2: -// line 714 "ParserConfig.rl" + case 3: +// line 740 "ParserConfig.rl" { p--; { p += 1; _goto_targ = 5; if (true) continue _goto;} } break; -// line 2155 "ParserConfig.java" +// line 2203 "ParserConfig.java" } } } @@ -2171,7 +2219,7 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) break; } } -// line 745 "ParserConfig.rl" +// line 771 "ParserConfig.rl" if (cs < JSON_object_first_final) { res.update(null, p + 1); @@ -2182,11 +2230,11 @@ else if ( _widec > _JSON_object_trans_keys[_mid+1] ) } -// line 2186 "ParserConfig.java" +// line 2234 "ParserConfig.java" private static byte[] init__JSON_actions_0() { return new byte [] { - 0, 1, 0 + 0, 1, 0, 1, 1 }; } @@ -2249,9 +2297,9 @@ private static byte[] init__JSON_indicies_0() { return new byte [] { 0, 0, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, - 0, 2, 1, 4, 5, 1, 6, 4, 6, 7, 4, 7, - 5, 8, 9, 1, 10, 8, 10, 0, 8, 0, 9, 7, - 7, 11, 7, 1, 0 + 0, 2, 1, 4, 5, 1, 6, 4, 6, 7, 4, 8, + 5, 9, 10, 1, 11, 9, 11, 0, 9, 12, 10, 7, + 7, 13, 7, 1, 0 }; } @@ -2261,7 +2309,8 @@ private static byte[] init__JSON_indicies_0() private static byte[] init__JSON_trans_targs_0() { return new byte [] { - 1, 0, 10, 6, 3, 5, 4, 10, 7, 9, 8, 2 + 1, 0, 10, 6, 3, 5, 4, 10, 10, 7, 9, 8, + 1, 2 }; } @@ -2271,7 +2320,8 @@ private static byte[] init__JSON_trans_targs_0() private static byte[] init__JSON_trans_actions_0() { return new byte [] { - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 3, 0, 0, 0, 1, 0, 1, 0, 0, 1, + 1, 0 }; } @@ -2285,7 +2335,7 @@ private static byte[] init__JSON_trans_actions_0() static final int JSON_en_main = 1; -// line 774 "ParserConfig.rl" +// line 800 "ParserConfig.rl" public IRubyObject parseImplementation(ThreadContext context) { @@ -2295,16 +2345,16 @@ public IRubyObject parseImplementation(ThreadContext context) { ParserResult res = new ParserResult(); -// line 2299 "ParserConfig.java" +// line 2349 "ParserConfig.java" { cs = JSON_start; } -// line 783 "ParserConfig.rl" +// line 809 "ParserConfig.rl" p = byteList.begin(); pe = p + byteList.length(); -// line 2308 "ParserConfig.java" +// line 2358 "ParserConfig.java" { int _klen; int _trans = 0; @@ -2385,7 +2435,24 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) switch ( _JSON_actions[_acts++] ) { case 0: -// line 760 "ParserConfig.rl" +// line 321 "ParserConfig.rl" + { + if (!config.allowComments) { + if (config.deprecateComments) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" + ); + } + } else { + throw unexpectedToken(context, p, pe); + } + } + } + break; + case 1: +// line 786 "ParserConfig.rl" { parseValue(context, res, p, pe); if (res.result == null) { @@ -2397,7 +2464,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) } } break; -// line 2401 "ParserConfig.java" +// line 2468 "ParserConfig.java" } } } @@ -2417,7 +2484,7 @@ else if ( data[p] > _JSON_trans_keys[_mid+1] ) break; } } -// line 786 "ParserConfig.rl" +// line 812 "ParserConfig.rl" if (cs >= JSON_first_final && p == pe) { return result; diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl index 2312823a6..e98c71f8c 100644 --- a/java/src/json/ext/ParserConfig.rl +++ b/java/src/json/ext/ParserConfig.rl @@ -52,6 +52,8 @@ public class ParserConfig extends RubyObject { private int maxNesting; private boolean allowNaN; private boolean allowTrailingComma; + private boolean allowComments; + private boolean deprecateComments; private boolean allowControlCharacters; private boolean allowInvalidEscape; private boolean allowDuplicateKey; @@ -178,6 +180,14 @@ public class ParserConfig extends RubyObject { OptionsReader opts = new OptionsReader(context, options); this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); + if (opts.hasKey("allow_comments")) { + this.allowComments = opts.getBool("allow_comments", false); + this.deprecateComments = false; + } else { + this.allowComments = true; + this.deprecateComments = true; + } + this.allowControlCharacters = opts.getBool("allow_control_characters", false); this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); @@ -308,11 +318,26 @@ public class ParserConfig extends RubyObject { %%{ machine JSON_common; + action parse_comment { + if (!config.allowComments) { + if (config.deprecateComments) { + if (config.deprecateDuplicateKey && emittedDeprecations < 5) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" + ); + } + } else { + throw unexpectedToken(context, p, pe); + } + } + } + cr = '\n'; cr_neg = [^\n]; ws = [ \t\r\n]; - c_comment = '/*' ( any* - (any* '*/' any* ) ) '*/'; - cpp_comment = '//' cr_neg* cr; + c_comment = '/*' ( any* - (any* '*/' any* ) ) '*/' >parse_comment; + cpp_comment = '//' cr_neg* cr >parse_comment; comment = c_comment | cpp_comment; ignore = ws | comment; name_separator = ':'; @@ -331,6 +356,7 @@ public class ParserConfig extends RubyObject { begin_string = '"'; begin_name = begin_string; begin_number = digit | '-'; + }%% %%{ diff --git a/lib/json.rb b/lib/json.rb index 26d601926..f8dc4ccc9 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -145,11 +145,11 @@ # # warning: detected duplicate keys in JSON object. # # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` # -# When set to `+true+` +# When set to +true+: # # The last value is used. # JSON.parse('{"a": 1, "a":2}') => {"a" => 2} # -# When set to `+false+`, the future default: +# When set to +false+, the future default: # JSON.parse('{"a": 1, "a":2}') => duplicate key at line 1 column 1 (JSON::ParserError) # # --- @@ -184,6 +184,20 @@ # # --- # +# Option +allow_comments+ (boolean) specifies whether to allow +# JavaScript style comments (either // comment or /* comment */); +# defaults to +false+. +# +# When not specified, a deprecation warning is emitted if a comment is encountered. +# +# When set to +true+, comments are ignored: +# JSON.parse('/* comment */ {"a": 1, "a":2}') # => {"a" => 2} +# +# When set to +false+, the future default: +# JSON.parse('/* comment */ {"a": 1, "a":2}') # unexpected character: '/' at line 1 column 1 (JSON::ParserError) +# +# --- +# # Option +allow_control_characters+ (boolean) specifies whether to allow # unescaped ASCII control characters, such as newlines, in strings; # defaults to +false+. diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 292ca1a67..943d93285 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -489,7 +489,7 @@ def test_parse_comments JSON assert_equal( { "key1" => "value1", "key2" => "value2", "key3" => "value3" }, - parse(json)) + parse(json, allow_comments: true)) json = <<~JSON { "key1":"value1" /* multi line @@ -498,7 +498,7 @@ def test_parse_comments * comment */ } JSON - assert_raise(ParserError) { parse(json) } + assert_raise(ParserError) { parse(json, allow_comments: true) } json = <<~JSON { "key1":"value1" /* multi line @@ -506,7 +506,7 @@ def test_parse_comments /* legal nested multi line comment start sequence */ } JSON - assert_equal({ "key1" => "value1" }, parse(json)) + assert_equal({ "key1" => "value1" }, parse(json, allow_comments: true)) json = <<~JSON { "key1":"value1" /* multi line @@ -515,18 +515,28 @@ def test_parse_comments and again, throw an Error */ } JSON - assert_raise(ParserError) { parse(json) } + assert_raise(ParserError) { parse(json, allow_comments: true) } json = <<~JSON { "key1":"value1" /*/*/ } JSON - assert_equal({ "key1" => "value1" }, parse(json)) - assert_equal({}, parse('{} /**/')) - assert_raise(ParserError) { parse('{} /* comment not closed') } - assert_raise(ParserError) { parse('{} /*/') } - assert_raise(ParserError) { parse('{} /x wrong comment') } - assert_raise(ParserError) { parse('{} /') } + assert_equal({ "key1" => "value1" }, parse(json, allow_comments: true)) + assert_equal({}, parse('{} /**/', allow_comments: true)) + assert_raise(ParserError) { parse('{} /* comment not closed', allow_comments: true) } + assert_raise(ParserError) { parse('{} /*/', allow_comments: true) } + assert_raise(ParserError) { parse('{} /x wrong comment', allow_comments: true) } + assert_raise(ParserError) { parse('{} /', allow_comments: true) } + end + + def test_parse_comments_deprecation + assert_equal({}, parse('/**/ {}', allow_comments: true)) + assert_raise(ParserError) { parse('/**/ {}', allow_comments: false) } + if RUBY_ENGINE == 'ruby' + assert_deprecated_warning(/Encountered comment in JSON/) do + parse('/**/ {}') + end + end end def test_nesting From 8972c6da6e5db480f399410fc82fef16619b56b0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 7 Jun 2026 20:43:42 +0200 Subject: [PATCH 346/506] parser.c: Precompute JSON::ParserError and ivar IDs --- ext/json/ext/parser/parser.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index dc76ca2cd..5559561e2 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2,10 +2,10 @@ #include "../vendor/ryu.h" #include "../simd/simd.h" -static VALUE mJSON, eNestingError, Encoding_UTF_8; +static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; -static ID i_new, i_try_convert, i_uminus, i_encode; +static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, @@ -645,9 +645,9 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta static VALUE parse_error_new(VALUE message, long line, long column) { - VALUE exc = rb_exc_new_str(rb_path2class("JSON::ParserError"), message); - rb_ivar_set(exc, rb_intern("@line"), LONG2NUM(line)); - rb_ivar_set(exc, rb_intern("@column"), LONG2NUM(column)); + VALUE exc = rb_exc_new_str(eParserError, message); + rb_ivar_set(exc, i_at_line, LONG2NUM(line)); + rb_ivar_set(exc, i_at_column, LONG2NUM(column)); return exc; } @@ -2069,8 +2069,13 @@ void Init_parser(void) mJSON = rb_define_module("JSON"); VALUE mExt = rb_define_module_under(mJSON, "Ext"); VALUE cParserConfig = rb_define_class_under(mExt, "ParserConfig", rb_cObject); + + rb_global_variable(&eParserError); + eParserError = rb_path2class("JSON::ParserError"); + + rb_global_variable(&eNestingError); eNestingError = rb_path2class("JSON::NestingError"); - rb_gc_register_mark_object(eNestingError); + rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate); rb_define_method(cParserConfig, "initialize", cParserConfig_initialize, 1); rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1); @@ -2078,14 +2083,14 @@ void Init_parser(void) VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject); rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2); + rb_global_variable(&CNaN); CNaN = rb_const_get(mJSON, rb_intern("NaN")); - rb_gc_register_mark_object(CNaN); + rb_global_variable(&CInfinity); CInfinity = rb_const_get(mJSON, rb_intern("Infinity")); - rb_gc_register_mark_object(CInfinity); + rb_global_variable(&CMinusInfinity); CMinusInfinity = rb_const_get(mJSON, rb_intern("MinusInfinity")); - rb_gc_register_mark_object(CMinusInfinity); rb_global_variable(&Encoding_UTF_8); Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8")); @@ -2106,6 +2111,8 @@ void Init_parser(void) i_try_convert = rb_intern("try_convert"); i_uminus = rb_intern("-@"); i_encode = rb_intern("encode"); + i_at_line = rb_intern("@line"); + i_at_column = rb_intern("@column"); binary_encindex = rb_ascii8bit_encindex(); utf8_encindex = rb_utf8_encindex(); From 8144d4cb3473b9b48f10716a6ea7fcc83300e826 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 8 Jun 2026 12:23:44 +0900 Subject: [PATCH 347/506] Suppres the warning for comment in JSON Suppres it for now because JRuby version does not support it yet. --- test/json/json_parser_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 943d93285..a4292871a 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -219,7 +219,9 @@ def test_parse_arrays def test_parse_json_primitive_values assert_raise(JSON::ParserError) { parse('') } assert_raise(TypeError) { parse(nil) } - assert_raise(JSON::ParserError) { parse(' /* foo */ ') } + EnvUtil.suppress_warning do # still no warning in JRuby verions + assert_raise(JSON::ParserError) { parse(' /* foo */ ') } + end assert_equal nil, parse('null') assert_equal false, parse('false') assert_equal true, parse('true') From 996bac686d64e4e3aaeae03b14a7f9ee9695ebdb Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 8 Jun 2026 09:11:20 +0200 Subject: [PATCH 348/506] generator.c: don't start with a stack buffer in IO case It was never intended to be initialized with a stack buffer in that case, as when dumping into an IO we want larger chunks. --- ext/json/ext/fbuffer/fbuffer.h | 43 +++++++++++++++--------------- ext/json/ext/generator/generator.c | 12 +++------ 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index b4f5266ca..cc44ea030 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -37,13 +37,17 @@ static void fbuffer_append_long(FBuffer *fb, long number); static inline void fbuffer_append_char(FBuffer *fb, char newchr); static VALUE fbuffer_finalize(FBuffer *fb); -static void fbuffer_stack_init(FBuffer *fb, size_t initial_length, char *stack_buffer, size_t stack_buffer_size) +static void fbuffer_init(FBuffer *fb, size_t initial_length, VALUE io, char *stack_buffer, size_t stack_buffer_size) { - fb->initial_length = (initial_length > 0) ? initial_length : FBUFFER_INITIAL_LENGTH_DEFAULT; - if (stack_buffer) { + if (RTEST(io)) { + JSON_ASSERT(fb->type == FBUFFER_HEAP_ALLOCATED); + fb->io = io; + fb->initial_length = (initial_length > 0) ? initial_length : FBUFFER_IO_BUFFER_SIZE; + } else { fb->type = FBUFFER_STACK_ALLOCATED; fb->ptr = stack_buffer; fb->capa = stack_buffer_size; + fb->initial_length = (initial_length > 0) ? initial_length : FBUFFER_INITIAL_LENGTH_DEFAULT; } #if JSON_DEBUG fb->requested = 0; @@ -79,45 +83,40 @@ static void fbuffer_flush(FBuffer *fb) fbuffer_clear(fb); } -static void fbuffer_realloc(FBuffer *fb, size_t required) +static void fbuffer_realloc(FBuffer *fb, size_t new_capa) { - if (required > fb->capa) { + if (new_capa > fb->capa) { if (fb->type == FBUFFER_STACK_ALLOCATED) { const char *old_buffer = fb->ptr; - fb->ptr = ALLOC_N(char, required); + fb->ptr = ALLOC_N(char, new_capa); fb->type = FBUFFER_HEAP_ALLOCATED; MEMCPY(fb->ptr, old_buffer, char, fb->len); } else { - JSON_SIZED_REALLOC_N(fb->ptr, char, required, fb->capa); + JSON_SIZED_REALLOC_N(fb->ptr, char, new_capa, fb->capa); } - fb->capa = required; + fb->capa = new_capa; } } static void fbuffer_do_inc_capa(FBuffer *fb, size_t requested) { if (RB_UNLIKELY(fb->io)) { - if (fb->capa < FBUFFER_IO_BUFFER_SIZE) { - fbuffer_realloc(fb, FBUFFER_IO_BUFFER_SIZE); - } else { + if (fb->capa != 0) { fbuffer_flush(fb); - } - - if (RB_LIKELY(requested < fb->capa)) { - return; + if (RB_LIKELY(requested < fb->capa)) { + return; + } } } - size_t required; + size_t new_capa = fb->capa ? fb->capa : fb->initial_length; + size_t needed_capa = requested + fb->len; - if (RB_UNLIKELY(!fb->ptr)) { - fb->ptr = ALLOC_N(char, fb->initial_length); - fb->capa = fb->initial_length; + while (new_capa < needed_capa) { + new_capa *= 2; } - for (required = fb->capa; requested > required - fb->len; required <<= 1); - - fbuffer_realloc(fb, required); + fbuffer_realloc(fb, new_capa); } static inline void fbuffer_inc_capa(FBuffer *fb, size_t requested) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 82853633b..08c1fd0db 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1308,10 +1308,8 @@ static inline VALUE cState_partial_generate(VALUE self, VALUE obj, generator_fun GET_STATE(self); char stack_buffer[FBUFFER_STACK_SIZE]; - FBuffer buffer = { - .io = RTEST(io) ? io : Qfalse, - }; - fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE); + FBuffer buffer = { 0 }; + fbuffer_init(&buffer, state->buffer_initial_length, io, stack_buffer, FBUFFER_STACK_SIZE); struct generate_json_data data = { .buffer = &buffer, @@ -1870,10 +1868,8 @@ static VALUE cState_m_do_generate(VALUE klass, VALUE obj, VALUE opts, VALUE io, configure_state(&state, Qfalse, opts); char stack_buffer[FBUFFER_STACK_SIZE]; - FBuffer buffer = { - .io = RTEST(io) ? io : Qfalse, - }; - fbuffer_stack_init(&buffer, state.buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE); + FBuffer buffer = { 0 }; + fbuffer_init(&buffer, state.buffer_initial_length, io, stack_buffer, FBUFFER_STACK_SIZE); struct generate_json_data data = { .buffer = &buffer, From 53f9d990bb627914baeda347bb3a705b880f1b9a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 11 Jun 2026 20:47:34 +0200 Subject: [PATCH 349/506] Merge 2.19.9 changes --- CHANGES.md | 5 +++++ lib/json/version.rb | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 69bdb076d..1d76b37e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,11 @@ * Deprecate default support of JavaScript comments in the parser and add `allow_comments: true` parsing option. * Integrate with Ruby 4.1 `ruby_sized_xfree`. +### 2026-06-11 (2.19.9) + +* Fix buffer overflow that could lead to a crash when writing JSON directly into an IO + with `JSON.generate(object, io)`. [CVE-PENDING]. + ### 2026-06-03 (2.19.8) * Fix 1-byte buffer overread on EOS errors. diff --git a/lib/json/version.rb b/lib/json/version.rb index 30c0a71d2..8efca3d4a 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.8' + VERSION = '2.19.9' end From 2b908f6f267a6719d705f0a8254c23537d71eb45 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 15 Jun 2026 11:47:58 +0200 Subject: [PATCH 350/506] Implement JSON::ResumableParser Only available on MRI for now. --- CHANGES.md | 3 + ext/json/ext/json.h | 5 + ext/json/ext/parser/parser.c | 804 ++++++++++++++++++++++++----- test/json/resumable_parser_test.rb | 271 ++++++++++ 4 files changed, 954 insertions(+), 129 deletions(-) create mode 100644 test/json/resumable_parser_test.rb diff --git a/CHANGES.md b/CHANGES.md index 1d76b37e6..949e3fff1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ### Unreleased +* Changed the C parser to no longer be recursive, so parsing very deep documents with `max_nesting: false` will no longer + result in `SystemStackError stack level too deep` errors. +* Added `JSON::ResumableParser` to parse streams of JSON documents. Not yet available on JRuby. * Deprecate default support of JavaScript comments in the parser and add `allow_comments: true` parsing option. * Integrate with Ruby 4.1 `ruby_sized_xfree`. diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index 78b0eee0a..a29d37bee 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -28,8 +28,13 @@ #else # define JSON_UNREACHABLE_RETURN UNREACHABLE_RETURN #endif + /* shims */ +#ifndef UNDEF_P +#define UNDEF_P(val) (val == Qundef) +#endif + #if SIZEOF_UINT64_T == SIZEOF_LONG_LONG # define INT64T2NUM(x) LL2NUM(x) # define UINT64T2NUM(x) ULL2NUM(x) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 5559561e2..15e79b24e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -5,7 +5,7 @@ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; -static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column; +static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column, i_at_eos; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, @@ -58,6 +58,20 @@ typedef struct rvalue_cache_struct { VALUE entries[JSON_RVALUE_CACHE_CAPA]; } rvalue_cache; +static void rvalue_cache_mark(rvalue_cache *cache) +{ + for (int index = 0; index < cache->length; index++) { + rb_gc_mark_movable(cache->entries[index]); + } +} + +static void rvalue_cache_compact(rvalue_cache *cache) +{ + for (int index = 0; index < cache->length; index++) { + cache->entries[index] = rb_gc_location(cache->entries[index]); + } +} + static rb_encoding *enc_utf8; #define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55 @@ -206,7 +220,7 @@ static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalue_stack **stack_ref) { - long required = stack->capa * 2; + long required = stack->capa ? stack->capa * 2 : RVALUE_STACK_INITIAL_CAPA; if (stack->type == RVALUE_STACK_STACK_ALLOCATED) { stack = rvalue_stack_spill(stack, handle, stack_ref); @@ -222,8 +236,10 @@ static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, if (RB_UNLIKELY(stack->head >= stack->capa)) { stack = rvalue_stack_grow(stack, handle, stack_ref); } + stack->ptr[stack->head] = value; stack->head++; + return value; } @@ -372,7 +388,7 @@ typedef struct json_frame_struct { enum json_frame_type type; enum json_frame_phase phase; long value_stack_head; // rvalue_stack->head when this container opened - const char *start_cursor; // object frames only (the '{'); NULL otherwise + size_t start_offset; // object frames only (the '{'); NULL otherwise } json_frame; typedef struct json_frame_stack_struct { @@ -421,7 +437,7 @@ static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VAL static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *handle, json_frame_stack **stack_ref) { - long required = stack->capa * 2; + long required = stack->capa ? stack->capa * 2 : JSON_FRAME_STACK_INITIAL_CAPA; if (stack->type == RVALUE_STACK_STACK_ALLOCATED) { stack = json_frame_stack_spill(stack, handle, stack_ref); @@ -643,26 +659,49 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta return message; } -static VALUE parse_error_new(VALUE message, long line, long column) +static VALUE parse_error_new(VALUE message, long line, long column, bool eos) { VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); rb_ivar_set(exc, i_at_column, LONG2NUM(column)); + if (eos) { + rb_ivar_set(exc, i_at_eos, Qtrue); + } return exc; } -NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state) +NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos) { long line, column; cursor_position(state, &line, &column); VALUE message = build_parse_error_message(format, state, line, column); - rb_exc_raise(parse_error_new(message, line, column)); + rb_exc_raise(parse_error_new(message, line, column, eos)); +} + +NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state) +{ + raise_parse_error(format, state, true); +} + +NORETURN(static) void raise_syntax_error(const char *format, JSON_ParserState *state) +{ + raise_parse_error(format, state, false); } -NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at) +NORETURN(static) void raise_parse_error_at(const char *format, JSON_ParserState *state, const char *at, bool eos) { state->cursor = at; - raise_parse_error(format, state); + raise_parse_error(format, state, eos); +} + +NORETURN(static) void raise_eos_error_at(const char *format, JSON_ParserState *state, const char *at) +{ + raise_parse_error_at(format, state, at, true); +} + +NORETURN(static) void raise_syntax_error_at(const char *format, JSON_ParserState *state, const char *at) +{ + raise_parse_error_at(format, state, at, false); } /* unicode */ @@ -687,7 +726,7 @@ static const signed char digit_values[256] = { static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe) { if (RB_UNLIKELY(sp > spe - 4)) { - raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); + raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); } const unsigned char *p = (const unsigned char *)sp; @@ -698,7 +737,7 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const const signed char b3 = digit_values[p[3]]; if (RB_UNLIKELY((signed char)(b0 | b1 | b2 | b3) < 0)) { - raise_parse_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); + raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); } return ((uint32_t)b0 << 12) | ((uint32_t)b1 << 8) | ((uint32_t)b2 << 4) | (uint32_t)b3; @@ -715,7 +754,7 @@ NOINLINE(static) void json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) { if (config->on_comment == JSON_RAISE) { - raise_parse_error("unexpected token %s", state); + raise_syntax_error("unexpected token %s", state); } const char *start = state->cursor; @@ -737,7 +776,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) while (true) { const char *next_match = memchr(state->cursor, '*', state->end - state->cursor); if (!next_match) { - raise_parse_error_at("unterminated comment, expected closing '*/'", state, start); + raise_eos_error_at("unterminated comment, expected closing '*/'", state, start); } state->cursor = next_match + 1; @@ -749,7 +788,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) break; } default: - raise_parse_error_at("unexpected token %s", state, start); + raise_parse_error_at("unexpected token %s", state, start, eos(state)); break; } @@ -945,13 +984,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser uint32_t sur = unescape_unicode(state, pe + 2, stringEnd); if (RB_UNLIKELY((sur & 0xFC00) != 0xDC00)) { - raise_parse_error_at("invalid surrogate pair at %s", state, p); + raise_syntax_error_at("invalid surrogate pair at %s", state, p); } ch = (((ch & 0x3F) << 10) | ((((ch >> 6) & 0xF) + 1) << 16) | (sur & 0x3FF)); pe += 5; } else { - raise_parse_error_at("incomplete surrogate pair at %s", state, p); + raise_syntax_error_at("incomplete surrogate pair at %s", state, p); break; } } @@ -961,20 +1000,22 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser p = ++pe; break; } + case 0: + return Qundef; default: if ((unsigned char)*pe < 0x20) { if (!config->allow_control_characters) { if (*pe == '\n') { - raise_parse_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); + raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); } - raise_parse_error_at("invalid ASCII control character in string: %s", state, pe - 1); + raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1); } } if (config->allow_invalid_escape) { APPEND_CHAR(*pe); } else { - raise_parse_error_at("invalid escape character in string: %s", state, pe - 1); + raise_syntax_error_at("invalid escape character in string: %s", state, pe - 1); } break; } @@ -1129,7 +1170,7 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d long line, column; cursor_position(state, &line, &column); rb_str_concat(message, build_parse_error_message("", state, line, column)) ; - rb_exc_raise(parse_error_new(message, line, column)); + rb_exc_raise(parse_error_new(message, line, column, false)); } NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) @@ -1265,7 +1306,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi } default: if (!config->allow_control_characters) { - raise_parse_error("invalid ASCII control character in string: %s", state); + raise_syntax_error("invalid ASCII control character in string: %s", state); } break; } @@ -1273,8 +1314,7 @@ static VALUE json_parse_escaped_string(JSON_ParserState *state, JSON_ParserConfi state->cursor++; } while (string_scan(state)); - raise_parse_error("unexpected end of input, expected closing \"", state); - return Qfalse; + return Qundef; } ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_ParserConfig *config, bool is_name) @@ -1283,7 +1323,7 @@ ALWAYS_INLINE(static) VALUE json_parse_string(JSON_ParserState *state, JSON_Pars const char *start = state->cursor; if (RB_UNLIKELY(!string_scan(state))) { - raise_parse_error("unexpected end of input, expected closing \"", state); + return Qundef; } VALUE string; @@ -1382,7 +1422,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig int mantissa_digits = json_parse_digits(state, &mantissa); if (RB_UNLIKELY((first_digit == '0' && mantissa_digits > 1) || (negative && mantissa_digits == 0))) { - raise_parse_error_at("invalid number: %s", state, start); + return Qundef; } // Parse fractional part @@ -1395,7 +1435,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig mantissa_digits += fractional_digits; if (RB_UNLIKELY(!fractional_digits)) { - raise_parse_error_at("invalid number: %s", state, start); + return Qundef; } } @@ -1415,7 +1455,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig int exponent_digits = json_parse_digits(state, &abs_exponent); if (RB_UNLIKELY(!exponent_digits)) { - raise_parse_error_at("invalid number: %s", state, start); + return Qundef; } if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) { @@ -1437,16 +1477,6 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig return json_decode_float(config, mantissa, mantissa_digits, exponent, negative, start, state->cursor); } -static inline VALUE json_parse_positive_number(JSON_ParserState *state, JSON_ParserConfig *config) -{ - return json_parse_number(state, config, false, state->cursor); -} - -static inline VALUE json_parse_negative_number(JSON_ParserState *state, JSON_ParserConfig *config) -{ - return json_parse_number(state, config, true, state->cursor - 1); -} - // How many values (array elements, or interleaved object keys+values) have been // pushed onto the rvalue stack since this container opened. Used to size the // bulk decode on close, and to tell the first key/colon from later ones. @@ -1460,19 +1490,19 @@ static inline long json_frame_entry_count(const json_frame *frame, const rvalue_ // moves on to expecting a ',' or its closing bracket. The caller passes the // frame it already has in hand -- the one that was expecting the value -- which // after a container close is the freshly re-exposed parent. -static inline void json_value_completed(json_frame *frame) +static inline enum json_frame_phase json_value_completed(json_frame *frame) { JSON_ASSERT((int)JSON_PHASE_DONE == (int)JSON_FRAME_ROOT); JSON_ASSERT((int)JSON_PHASE_ARRAY_COMMA == (int)JSON_FRAME_ARRAY); JSON_ASSERT((int)JSON_PHASE_OBJECT_COMMA == (int)JSON_FRAME_OBJECT); - frame->phase = (enum json_frame_phase) frame->type; + return frame->phase = (enum json_frame_phase) frame->type; } -ALWAYS_INLINE(static) bool json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset) +ALWAYS_INLINE(static) void json_match_keyword(JSON_ParserState *state, const char *keyword, size_t offset) { // It is assumed that since `keyword` is always a literal, the compiler is able to constantize this - // `strlen` and several other computations in that routine, such as eliminating the `if (resumable)` branch. + // `strlen` and several other computations in that routine. size_t len = strlen(keyword); @@ -1480,9 +1510,11 @@ ALWAYS_INLINE(static) bool json_match_keyword(JSON_ParserState *state, const cha // That's why we sometime compare starting from the first byte and sometimes from the second. if (rest(state) >= len && (memcmp(state->cursor + offset, keyword + offset, len - offset) == 0)) { state->cursor += len; - return true; + return; } - return false; + + bool eos = rest(state) < len && memcmp(state->cursor, keyword, rest(state)) == 0; + raise_parse_error("unexpected token %s", state, eos); } // Parse an arbitrary JSON value iteratively. This is a state machine driven @@ -1490,25 +1522,22 @@ ALWAYS_INLINE(static) bool json_match_keyword(JSON_ParserState *state, const cha // resume purely from the frame stack. A JSON_FRAME_ROOT frame sits at the // bottom of the stack, so the stack is never empty mid-parse and the document // itself is just another frame whose value, once parsed, leaves its phase DONE. -static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) +// When invoked in resumable mode, it returns true after parsing a complete document. +// If reaching EOS without having parsed a complete document, either returns false +// of raise a JSON::ParserError tagged with `@eos=true`. +ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config, bool resumable) { json_frame *frame = json_frame_stack_peek(state->frames); switch (frame->phase) { - case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_DONE: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; case JSON_PHASE_OBJECT_KEY: goto JSON_PHASE_OBJECT_KEY; case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - JSON_UNREACHABLE_RETURN(Qundef); - - JSON_PHASE_DONE: { - // The root document value is parsed; it is the lone survivor on - // the rvalue stack. - return *rvalue_stack_peek(state->value_stack, 1); - } + JSON_UNREACHABLE_RETURN(false); JSON_PHASE_VALUE: { json_eat_whitespace(state, config); @@ -1516,68 +1545,109 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) VALUE value; switch (peek(state)) { case 'n': - if (json_match_keyword(state, "null", 0)) { - value = Qnil; - break; - } - raise_parse_error("unexpected token %s", state); + json_match_keyword(state, "null", 0); + value = Qnil; + break; case 't': - if (json_match_keyword(state, "true", 0)) { - value = Qtrue; - break; - } - raise_parse_error("unexpected token %s", state); + json_match_keyword(state, "true", 0); + value = Qtrue; + break; case 'f': - if (json_match_keyword(state, "false", 1)) { - value = Qfalse; - break; - } - raise_parse_error("unexpected token %s", state); + json_match_keyword(state, "false", 1); + value = Qfalse; + break; case 'N': - // Note: memcmp with a small power of two compile to an integer comparison - if (config->allow_nan && json_match_keyword(state, "NaN", 1)) { - value = CNaN; - break; + if (!config->allow_nan) { + raise_syntax_error("unexpected token %s", state); } - raise_parse_error("unexpected token %s", state); + + json_match_keyword(state, "NaN", 1); + value = CNaN; + break; case 'I': - if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { - value = CInfinity; - break; + if (!config->allow_nan) { + raise_syntax_error("unexpected token %s", state); } - raise_parse_error("unexpected token %s", state); + + json_match_keyword(state, "Infinity", 0); + value = CInfinity; + break; case '-': { + const char *start = state->cursor; state->cursor++; - if (config->allow_nan && json_match_keyword(state, "Infinity", 0)) { + + value = json_parse_number(state, config, true, start); + + if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) { + state->cursor = start; + json_match_keyword(state, "-Infinity", 1); value = CMinusInfinity; - } else { - value = json_parse_negative_number(state, config); + break; + } + + // Top level numbers are ambiguous when parsing streams, we can't + // know if we parsed all the digits if we hit EOS. + if (RB_UNLIKELY(resumable && eos(state))) { + state->cursor = start; + return false; + } + + if (RB_UNLIKELY(UNDEF_P(value))) { + raise_syntax_error_at("invalid number: %s", state, start); } break; } - case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - value = json_parse_positive_number(state, config); + case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { + const char *start = state->cursor; + + value = json_parse_number(state, config, false, start); + + // Top level numbers are ambiguous when parsing streams, we can't + // know if we parsed all the digits if we hit EOS. + if (RB_UNLIKELY(resumable && eos(state))) { + state->cursor = start; + return false; + } + + if (RB_UNLIKELY(UNDEF_P(value))) { + raise_syntax_error_at("invalid number: %s", state, start); + } break; + } - case '"': + case '"': { // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} + const char *start = state->cursor; value = json_parse_string(state, config, false); + + if (RB_UNLIKELY(UNDEF_P(value))) { + bool is_eos = eos(state); + if (resumable) { + state->cursor = start; + } + raise_parse_error("unexpected end of input, expected closing \"", state, is_eos); + } break; + } case '[': { - state->cursor++; + const char *start = state->cursor++; json_eat_whitespace(state, config); - if (peek(state) == ']') { + const char next = peek(state); + if (next == ']') { state->cursor++; value = json_decode_array(state, config, 0); break; + } else if (resumable && next == 0) { + state->cursor = start; + return false; } state->current_nesting++; @@ -1594,6 +1664,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) }); goto JSON_PHASE_VALUE; } + case '{': { const char *object_start_cursor = state->cursor; @@ -1604,6 +1675,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; value = json_decode_object(state, config, 0); break; + } else if (resumable && eos(state)) { + state->cursor = object_start_cursor; + return false; } state->current_nesting++; @@ -1616,30 +1690,30 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) .type = JSON_FRAME_OBJECT, .phase = JSON_PHASE_OBJECT_KEY, .value_stack_head = state->value_stack->head, - .start_cursor = object_start_cursor, + .start_offset = object_start_cursor - state->start, }); goto JSON_PHASE_OBJECT_KEY; } case 0: - raise_parse_error("unexpected end of input", state); + return false; default: - raise_parse_error("unexpected character: %s", state); + raise_syntax_error("unexpected character: %s", state); } json_push_value(state, config, value); json_value_completed(frame); switch (frame->phase) { - case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_DONE: return true; case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } JSON_PHASE_OBJECT_KEY: { @@ -1647,21 +1721,34 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_eat_whitespace(state, config); + const char *start = state->cursor; + if (RB_LIKELY(peek(state) == '"')) { - json_push_value(state, config, json_parse_string(state, config, true)); + VALUE string = json_parse_string(state, config, true); + if (UNDEF_P(string)) { + if (resumable) { + state->cursor = start; + return false; + } else { + raise_syntax_error("unexpected end of input, expected closing \"", state); + } + } + json_push_value(state, config, string); frame->phase = JSON_PHASE_OBJECT_COLON; goto JSON_PHASE_OBJECT_COLON; + } else if (resumable && eos(state)) { + return false; } else { // The message differs for the first key vs. a key after a // ',': the first is the only one reached with nothing pushed // for this object yet. if (json_frame_entry_count(frame, state->value_stack) == 0) { - raise_parse_error("expected object key, got %s", state); + raise_syntax_error("expected object key, got %s", state); } else { - raise_parse_error("expected object key, got: %s", state); + raise_syntax_error("expected object key, got: %s", state); } } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } JSON_PHASE_OBJECT_COLON: { @@ -1673,16 +1760,18 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; frame->phase = JSON_PHASE_VALUE; goto JSON_PHASE_VALUE; + } else if (resumable && eos(state)) { + return false; } else { // First colon (only the first pair's key is pushed, nothing // else) vs. a later one. if (json_frame_entry_count(frame, state->value_stack) == 1) { - raise_parse_error("expected ':' after object key", state); + raise_syntax_error("expected ':' after object key", state); } else { - raise_parse_error("expected ':' after object key, got: %s", state); + raise_syntax_error("expected ':' after object key, got: %s", state); } } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } JSON_PHASE_ARRAY_COMMA: { @@ -1708,23 +1797,27 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) long count = json_frame_entry_count(frame, state->value_stack); state->current_nesting--; state->in_array--; - json_frame_stack_pop(state->frames); + json_push_value(state, config, json_decode_array(state, config, count)); + json_frame_stack_pop(state->frames); frame = json_frame_stack_peek(state->frames); + json_value_completed(frame); switch (frame->phase) { - case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_DONE: return true; case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } + } else if (resumable && next_char == 0) { + return false; } else { - raise_parse_error("expected ',' or ']' after array value", state); + raise_syntax_error("expected ',' or ']' after array value", state); } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } JSON_PHASE_OBJECT_COMMA: { @@ -1735,9 +1828,9 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) if (RB_LIKELY(next_char == ',')) { state->cursor++; + json_eat_whitespace(state, config); if (config->allow_trailing_comma) { - json_eat_whitespace(state, config); if (peek(state) == '}') { // Trailing comma: stay in COMMA to close on the next iteration. goto JSON_PHASE_OBJECT_COMMA; @@ -1753,7 +1846,7 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) // Temporary rewind cursor in case an error is raised const char *final_cursor = state->cursor; - state->cursor = frame->start_cursor; + state->cursor = state->start + frame->start_offset; VALUE object = json_decode_object(state, config, count); state->cursor = final_cursor; @@ -1763,27 +1856,29 @@ static VALUE json_parse_any(JSON_ParserState *state, JSON_ParserConfig *config) json_value_completed(frame); switch (frame->phase) { - case JSON_PHASE_DONE: goto JSON_PHASE_DONE; + case JSON_PHASE_DONE: return true; case JSON_PHASE_ARRAY_COMMA: goto JSON_PHASE_ARRAY_COMMA; case JSON_PHASE_OBJECT_COMMA: goto JSON_PHASE_OBJECT_COMMA; case JSON_PHASE_VALUE: goto JSON_PHASE_VALUE; - case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(Qundef); + case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } + } else if (resumable && next_char == 0) { + return false; } else { - raise_parse_error("expected ',' or '}' after object value, got: %s", state); + raise_syntax_error("expected ',' or '}' after object value, got: %s", state); } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } - JSON_UNREACHABLE_RETURN(Qundef); + JSON_UNREACHABLE_RETURN(false); } static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config) { json_eat_whitespace(state, config); if (!eos(state)) { - raise_parse_error("unexpected token at end of stream %s", state); + raise_syntax_error("unexpected token at end of stream %s", state); } } @@ -1903,23 +1998,9 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self * * Creates a new JSON::Ext::ParserConfig instance. * - * It will be configured by the _opts_ hash. _opts_ can have the following - * keys: + * Argument +opts+, if given, contains a \Hash of options for the parsing. + * See {Parsing Options}[#module-JSON-label-Parsing+Options]. * - * _opts_ can have the following keys: - * * *max_nesting*: The maximum depth of nesting allowed in the parsed data - * structures. Disable depth checking with :max_nesting => false|nil|0, it - * defaults to 100. - * * *allow_nan*: If set to true, allow NaN, Infinity and -Infinity in - * defiance of RFC 4627 to be parsed by the Parser. This option defaults to - * false. - * * *symbolize_names*: If set to true, returns symbols for the names - * (keys) in a JSON object. Otherwise strings are returned, which is - * also the default. It's not possible to use this option in - * conjunction with the *create_additions* option. - * * *decimal_class*: Specifies which class to use instead of the default - * (Float) when parsing decimal numbers. This class must accept a single - * string argument in its constructor. */ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) { @@ -1982,7 +2063,11 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) }; JSON_ParserState *state = &_state; - VALUE result = json_parse_any(state, config); + bool complete = json_parse_any(state, config, false); + + // The root document value is parsed; it is the lone survivor on + // the rvalue stack. + VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef; // This may be skipped in case of exception, but // it won't cause a leak. @@ -1991,7 +2076,12 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) RB_GC_GUARD(value_stack_handle); RB_GC_GUARD(frame_stack_handle); RB_GC_GUARD(Vsource); - json_ensure_eof(state, config); + + if (complete) { + json_ensure_eof(state, config); + } else { + raise_eos_error("unexpected end of input", state); + } return result; } @@ -2058,6 +2148,449 @@ static VALUE cJSON_parser_s_allocate(VALUE klass) return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config); } +typedef struct JSON_ResumableParserStruct { + JSON_ParserConfig config; + JSON_ParserState state; + rvalue_stack value_stack; + json_frame_stack frames; + VALUE buffer; + bool in_use; +} JSON_ResumableParser; + +static void JSON_ResumableParser_mark(void *ptr) +{ + JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr; + JSON_ParserConfig_mark(&parser->config); + rvalue_stack_mark(&parser->value_stack); + rvalue_cache_mark(&parser->state.name_cache); + rb_gc_mark(parser->buffer); // pin the buffer +} + +static void JSON_ResumableParser_free(void *ptr) +{ + JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr; + rvalue_stack_free_buffer(&parser->value_stack); + json_frame_stack_free_buffer(&parser->frames); +} + +static size_t JSON_ResumableParser_memsize(const void *ptr) +{ + const JSON_ResumableParser *parser = (const JSON_ResumableParser *)ptr; + size_t memsize = JSON_ParserConfig_memsize(&parser->config); + memsize += rvalue_stack_memsize(&parser->value_stack); + memsize += json_frame_stack_memsize(&parser->frames); +#ifndef HAVE_RUBY_TYPED_EMBEDDABLE + memsize += ( + sizeof(JSON_ResumableParser) + - sizeof(JSON_ParserState) + - sizeof(JSON_ParserConfig) + - sizeof(rvalue_stack) + - sizeof(json_frame_stack) + ); +#endif + return memsize; +} + +static void JSON_ResumableParser_compact(void *ptr) +{ + JSON_ResumableParser *parser = (JSON_ResumableParser *)ptr; + JSON_ParserConfig_compact(&parser->config); + rvalue_stack_compact(&parser->value_stack); + rvalue_cache_compact(&parser->state.name_cache); + parser->buffer = rb_gc_location(parser->buffer); +} + +static const rb_data_type_t JSON_ResumableParser_type = { + .wrap_struct_name = "JSON::Ext::ResumableParser", + .function = { + JSON_ResumableParser_mark, + JSON_ResumableParser_free, + JSON_ResumableParser_memsize, + JSON_ResumableParser_compact, + }, + // RUBY_TYPED_WB_PROTECTED is deliberately not declared because + // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need + // to trigger a lot of write barriers. + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, +}; + +static VALUE cResumableParser_allocate(VALUE klass) +{ + JSON_ResumableParser *parser; + VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser); + parser->state.in_array++; + return obj; +} + +static inline JSON_ResumableParser *cResumableParser_get(VALUE self) +{ + JSON_ResumableParser *parser; + TypedData_Get_Struct(self, JSON_ResumableParser, &JSON_ResumableParser_type, parser); + return parser; +} + +/* + * call-seq: new(opts => {}) + * + * Creates a new JSON::ResumableParser instance. + * + * Argument +opts+, if given, contains a \Hash of options for the parsing. + * See {Parsing Options}[#module-JSON-label-Parsing+Options]. + * + * A ResumableParser is able to parse partial documents and resume parsing later + * when more of the document is provided: + * + * parser = JSON::ResumableParser.new + * parser << '{"user": "george", "role": "ad' + * parser.parse # => false + * parser.eos? # => true + * parser.partial_value # => { "user" => "george", "role" => nil } + * parser.rest # => '"ad' + * + * parser << 'min" }[1, 2, 3]' + * parser.parse # => true + * parser.value # => { "user" => "george", "role" => "admin" } + * + * parser.parse # => true + * parser.value # => [1, 2, 3] + * + * === Limitations + * + * While ResumableParser is able to parse streams of documents without any + * explicit separators between them, it is highly recommended to separate documents + * by either spaces or newlines, as otherwise the \JSON syntax for numbers may be ambiguous. + * When parsing a number, ResumableParser will not consider the number complete until something follows: + * + * parser << '123' + * parser.parse # => false + * parser << ' ' + * parser.parse # => true + * parser.value # => 123 + */ +static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) +{ + rb_check_arity(argc, 0, 1); + rb_check_frozen(self); + JSON_ResumableParser *parser = cResumableParser_get(self); + + VALUE opts = argc > 0 ? argv[0] : Qnil; + parser_config_init(&parser->config, opts, self); + + return self; +} + +/* + * call-seq: self << string -> self + * + * Appends the given string to the parser's buffer. + */ +static VALUE cResumableParser_feed(VALUE self, VALUE str) +{ + rb_check_frozen(self); + str = convert_encoding(str); + if (!RSTRING_LEN(str)) { + return self; + } + + JSON_ResumableParser *parser = cResumableParser_get(self); + + size_t offset = parser->state.cursor - parser->state.start; + const size_t remaining = parser->state.end - parser->state.cursor; + + if (!remaining) { + parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str)); + offset = 0; + } else { + JSON_ASSERT(parser->buffer); + + const size_t size = parser->state.end - parser->state.start; + const size_t consumed = size - remaining; + char *old_ptr = RSTRING_PTR(parser->buffer); + + if (RB_OBJ_FROZEN_RAW(parser->buffer)) { + VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str))); + MEMCPY(RSTRING_PTR(new_buffer), old_ptr + consumed, char, remaining); + rb_str_set_len(new_buffer, remaining); + offset = 0; + parser->buffer = new_buffer; + } else if (consumed > (size / 2) && size >= 512) { + MEMMOVE(old_ptr, old_ptr + consumed, char, remaining); + rb_str_set_len(parser->buffer, remaining); + offset = 0; + } + rb_str_append(parser->buffer, str); + } + + long len; + const char *start; + RSTRING_GETMEM(parser->buffer, start, len); + parser->state.start = start; + parser->state.end = start + len; + parser->state.cursor = parser->state.start + offset; + + return self; +} + +struct json_parse_any_args { + JSON_ParserState *state; + JSON_ParserConfig *config; +}; + +static VALUE json_parse_any_resumable_safe(VALUE _args) +{ + struct json_parse_any_args *args = (struct json_parse_any_args *)_args; + return (VALUE)json_parse_any(args->state, args->config, true); +} + +static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + + if (parser->in_use) { + rb_raise(rb_eArgError, "ResumableParser can't be used recursively"); + } + + if (lock) { + parser->in_use = true; + } + + // self may have moved, so we need to update all pointers + // Investigate: We might be better off keeping JSON_ParserState on the stack + // and only persist what we need. + parser->state.value_stack_handle = &self; + parser->state.frame_stack_handle = &self; + parser->state.value_stack = &parser->value_stack; + parser->state.frames = &parser->frames; + + return parser; +} + +/* + * call-seq: parse -> true or false + * + * Attemps to parse a JSON document from the internal buffer. + * Returns whether a complete document could be parsed. + * + * It does raise +JSON::ParserError+ when encountering invalid \JSON syntax. + * + * The parsed object can be retrieved by calling #value + */ +static VALUE cResumableParser_parse(VALUE self) +{ + JSON_ResumableParser *parser = ResumableParser_acquire(self, true); + if (!parser->buffer) { + return Qfalse; + } + + if (parser->frames.head == 0) { + json_frame_stack_push(&parser->state, (json_frame){ + .type = JSON_FRAME_ROOT, + .phase = JSON_PHASE_VALUE, + }); + } + + VALUE Vsource = parser->buffer; // Prevent compaction + + json_frame *frame = json_frame_stack_peek(&parser->frames); + + if (frame->phase == JSON_PHASE_DONE) { + JSON_ASSERT(parser->value_stack.head == 1); + JSON_ASSERT(parser->frames.head == 1); + + frame->phase = JSON_PHASE_VALUE; + rvalue_stack_pop(parser->state.value_stack, 1); + } + + struct json_parse_any_args args = { + .state = &parser->state, + .config = &parser->config, + }; + int status; + bool complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status); + parser->in_use = false; + if (status) { + complete = false; + if (RTEST(rb_ivar_get(rb_errinfo(), rb_intern("@eos")))) { + complete = false; // is an EOS error + } else { + rb_jump_tag(status); // reraise + } + } + RB_GC_GUARD(Vsource); + return complete ? Qtrue : Qfalse; +} + +/* + * call-seq: value? -> true or false + * + * Returns whether a parsed value is available. + */ +static VALUE cResumableParser_value_p(VALUE self) +{ + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + + if (parser->value_stack.head > 0) { + json_frame *frame = json_frame_stack_peek(&parser->frames); + if (frame->phase == JSON_PHASE_DONE) { + return Qtrue; + } + } + return Qfalse; +} + +/* + * call-seq: value -> object + * + * Returns and consume the last parsed value. + * Raises ArgumentError if there is no parsed value or if it was already retrieved: + * parser << '[1][2]' + * parser.value # ArgumentError no ready value + * parser.parse # => true + * parser.value # => [1] + * parser.value # ArgumentError no ready value + */ +static VALUE cResumableParser_value(VALUE self) +{ + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + + if (parser->frames.head > 0) { + json_frame *frame = json_frame_stack_peek(&parser->frames); + + if (frame->phase == JSON_PHASE_DONE) { + VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1); + rvalue_stack_pop(parser->state.value_stack, 1); + json_frame_stack_pop(parser->state.frames); + return result; + } + } + rb_raise(rb_eArgError, "no ready value"); +} + +/* + * call-seq: clear -> self + * + * Entirely reset the parser state and buffer. + */ +static VALUE cResumableParser_clear(VALUE self) +{ + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + parser->buffer = 0; + parser->frames.head = 0; + parser->value_stack.head = 0; + parser->state.name_cache.length = 0; + parser->state.start = parser->state.cursor = parser->state.end = NULL; + return self; +} + +/* + * call-seq: partial_value -> object + * + * Returns the Ruby objects parsed up to this point: + * parser << '[1, [2, 3,' + * parser.parse # => false + * parser.value # ArgumentError no ready value + * parser.partial_value # => [1, [2, 3]] + */ +static VALUE cResumableParser_partial_value(VALUE self) +{ + JSON_ResumableParser *original_parser = ResumableParser_acquire(self, false); + JSON_ResumableParser parser = *original_parser; + + if (parser.value_stack.head == 0) { + return Qnil; + } + + json_frame *frame = json_frame_stack_peek(parser.state.frames); + long missing_object_value = 0; + if (frame->type == JSON_FRAME_OBJECT && (frame->phase == JSON_PHASE_VALUE || frame->phase == JSON_PHASE_OBJECT_COLON)) { + missing_object_value = 1; + } + + // Copy the value stack as we need to mutate it. + long capa = parser.value_stack.head; + parser.value_stack.capa = (capa + missing_object_value); + VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value); + MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa); + parser.value_stack.ptr = value_stack_buffer; + + JSON_ParserState *state = &parser.state; + JSON_ParserConfig *config = &parser.config; + + if (missing_object_value) { + rvalue_stack_push(state->value_stack, Qnil, NULL, &state->value_stack); + } + + VALUE partial_result = Qundef; + + while (UNDEF_P(partial_result)) { + frame = json_frame_stack_peek(state->frames); + + switch (frame->type) { + case JSON_FRAME_ROOT: { + partial_result = *rvalue_stack_peek(state->value_stack, 1); + break; + } + + case JSON_FRAME_ARRAY: { + long count = json_frame_entry_count(frame, state->value_stack); + json_push_value(state, config, json_decode_array(state, config, count)); + json_frame_stack_pop(state->frames); + + break; + } + + case JSON_FRAME_OBJECT: { + long count = json_frame_entry_count(frame, state->value_stack); + json_push_value(state, config, json_decode_object(state, config, count)); + json_frame_stack_pop(state->frames); + break; + } + + default: { + JSON_UNREACHABLE_RETURN(Qundef); + break; + } + } + } + + ALLOCV_END(tmpbuf); + return partial_result; +} + +/* + * call-seq: rest -> string + * + * Returns a string containing what remains to be parsed in the buffer + * parser << '{ "message": "unterminated message' + * parser.parse # => false + * parser.rest # => '"unterminated message"' + */ +static VALUE cResumableParser_rest(VALUE self) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + + if (!parser->buffer) { + return rb_utf8_str_new("", 0); + } + + size_t offset = parser->state.cursor - parser->state.start; + const char *ptr; + long len; + RSTRING_GETMEM(parser->buffer, ptr, len); + return rb_utf8_str_new(ptr + offset, len - offset); +} + +/* + * call-seq: value? -> true or false + * + * Returns whether the internal buffer has been entirely consumed. + */ +static VALUE cResumableParser_eos_p(VALUE self) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + return eos(&parser->state) ? Qtrue : Qfalse; +} + void Init_parser(void) { #ifdef HAVE_RB_EXT_RACTOR_SAFE @@ -2077,12 +2610,24 @@ void Init_parser(void) eNestingError = rb_path2class("JSON::NestingError"); rb_define_alloc_func(cParserConfig, cJSON_parser_s_allocate); - rb_define_method(cParserConfig, "initialize", cParserConfig_initialize, 1); + rb_define_private_method(cParserConfig, "initialize", cParserConfig_initialize, 1); rb_define_method(cParserConfig, "parse", cParserConfig_parse, 1); VALUE cParser = rb_define_class_under(mExt, "Parser", rb_cObject); rb_define_singleton_method(cParser, "parse", cParser_m_parse, 2); + VALUE cResumableParser = rb_define_class_under(mJSON, "ResumableParser", rb_cObject); + rb_define_alloc_func(cResumableParser, cResumableParser_allocate); + rb_define_private_method(cResumableParser, "initialize", cResumableParser_initialize, -1); + rb_define_method(cResumableParser, "<<", cResumableParser_feed, 1); + rb_define_method(cResumableParser, "parse", cResumableParser_parse, 0); + rb_define_method(cResumableParser, "value", cResumableParser_value, 0); + rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0); + rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0); + rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0); + rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0); + rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0); + rb_global_variable(&CNaN); CNaN = rb_const_get(mJSON, rb_intern("NaN")); @@ -2113,6 +2658,7 @@ void Init_parser(void) i_encode = rb_intern("encode"); i_at_line = rb_intern("@line"); i_at_column = rb_intern("@column"); + i_at_eos = rb_intern("@eos"); binary_encindex = rb_ascii8bit_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb new file mode 100644 index 000000000..bab92c8de --- /dev/null +++ b/test/json/resumable_parser_test.rb @@ -0,0 +1,271 @@ +# frozen_string_literal: true +require_relative 'test_helper' + +class JSONResumageParserTest < Test::Unit::TestCase + include JSON + + def setup + omit "JRuby not supported" if RUBY_ENGINE == "jruby" + @parser = new_parser + end + + def test_value + refute_predicate @parser, :value? + assert_raise(ArgumentError) { @parser.value } + + @parser << '[]' + + refute_predicate @parser, :value? + assert_raise(ArgumentError) { @parser.value } + + assert @parser.parse + + assert_predicate @parser, :value? + assert_equal [], @parser.value + + refute_predicate @parser, :value? + assert_raise(ArgumentError) { @parser.value } + end + + def test_clear + @parser << '[1, 2, 3' + refute @parser.parse + assert_equal '3', @parser.rest + + @parser.clear + + assert_equal '', @parser.rest + refute_predicate @parser, :value? + + @parser << '[1, 2, 3][' + assert @parser.parse + assert_predicate @parser, :value? + assert_equal '[', @parser.rest + + @parser.clear + + assert_equal '', @parser.rest + refute_predicate @parser, :value? + end + + def test_parse_document_direct + @parser << '[true]' + assert_equal true, @parser.parse + assert_equal [true], @parser.value + end + + def test_parse_multiple_documents_direct + @parser << '[true]{}[1, 2, 3]' + + assert_equal true, @parser.parse + assert_equal [true], @parser.value + + assert_equal true, @parser.parse + assert_equal({}, @parser.value) + + assert_equal true, @parser.parse + assert_equal [1, 2, 3], @parser.value + end + + def test_parse_top_level_keywords + assert_resumed_parsing('true') + assert_resumed_parsing('false') + assert_resumed_parsing('null') + + assert_parse_stream([true, false, nil], 'truefalsenull') + end + + def test_parse_top_level_numbers + assert_parse_stream([1, 2, 3], '1 2 3 ') + assert_parse_stream([1, 2], '1 2 3') # Parser can't know if the number is terminated + assert_parse_stream([3], ' ') + assert_parse_stream([1, 2, 3, true], '1 2 3true') + assert_parse_stream([-1, 2.34, 5.0e67], '-1 2.34 5e67 ') + end + + def test_parse_byte_by_byte_array + assert_resumed_parsing('[]') + assert_resumed_parsing('[ ]') + assert_resumed_parsing('[true]') + assert_resumed_parsing('[12]') + assert_resumed_parsing('[ 12 ]') + assert_resumed_parsing('[ 12.3 ]') + assert_resumed_parsing('[ 12.3e12 ]') + assert_resumed_parsing('[ 1e12 ]') + assert_resumed_parsing('[-12]') + assert_resumed_parsing('[ -12 ]') + assert_resumed_parsing('[ -12.3 ]') + assert_resumed_parsing('[ -12.3e12 ]') + assert_resumed_parsing('[ -1e12 ]') + end + + def test_parse_byte_by_byte_object + assert_resumed_parsing('{}') + assert_resumed_parsing('{ }') + assert_resumed_parsing('{"test" : true}') + assert_resumed_parsing('{ "test":12, "value" : { "key": 42} }') + end + + def test_parse_byte_by_byte_string + assert_resumed_parsing(JSON.generate('test')) + assert_resumed_parsing(JSON.generate('te\\st')) + assert_resumed_parsing('"te\\u2028st"') + assert_resumed_parsing(JSON.generate("te\u2028st")) + assert_resumed_parsing(JSON.generate("te \u2028 st")) + end + + def test_parse_byte_by_byte_numbers + assert_resumed_parsing('123 ') + end + + def test_rest + @parser << '[1, 2, 3, "unterminated string' + refute @parser.parse + assert_equal '"unterminated string', @parser.rest + end + + def test_eos + assert_predicate @parser, :eos? + + @parser << '[1, 2, 3]' + refute_predicate @parser, :eos? + + assert @parser.parse + assert_predicate @parser, :eos? + + @parser << '123' + refute_predicate @parser, :eos? + + refute @parser.parse + refute_predicate @parser, :eos? + + @parser << ' ' + assert @parser.parse + assert_equal 123, @parser.value + refute_predicate @parser, :eos? + + refute @parser.parse + assert_predicate @parser, :eos? + end + + def test_partial_value + assert_nil @parser.partial_value + assert_partial_value [1, 2, 3], '[1, 2, 3, "unterminated string' + assert_partial_value({ "a" => 1, "b" => { "c" => nil } }, '{ "a": 1, "b": { "c": "unterminated string') + assert_partial_value({ "a" => 1, "b" => { "c" => nil } }, '{ "a": 1, "b": { "c"') + assert_partial_value([1, { "a" => 1, "b" => { "c" => nil } }], '[1, { "a": 1, "b": { "c"') + end + + def test_partial_value_missing + assert_nil @parser.partial_value + end + + def test_reentrency_prevented + called = false + parser = nil + callback = ->(o) do + unless called + called = true + parser.parse + end + o + end + parser = new_parser(on_load: callback) + parser << '[]' + error = assert_raise ArgumentError do + parser.parse + end + assert_equal "ResumableParser can't be used recursively", error.message + + called = false + parser = nil + callback = ->(o) do + unless called + called = true + parser.partial_value + end + o + end + parser = new_parser(on_load: callback) + parser << '[]' + error = assert_raise ArgumentError do + parser.parse + end + assert_equal "ResumableParser can't be used recursively", error.message + end + + def test_exception_unlock_parser + called = false + parser = nil + callback = ->(o) do + unless called + called = true + raise "oops" + end + o + end + parser = new_parser(on_load: callback) + parser << '[][1]' + assert_raise RuntimeError do + parser.parse + end + + assert parser.parse + assert_equal [1], parser.value + end + + def test_spill_rvalue_stack + expected = [1] * 1000 + @parser << JSON.dump(expected) + assert @parser.parse + assert_equal expected, @parser.value + end + + def test_spill_frames_stack + json = '[' * 1000 + ']' * 1000 + expected = JSON.parse(json, max_nesting: 1000) + @parser = new_parser(max_nesting: 1000) + @parser << json + assert @parser.parse + assert_equal expected, @parser.value + end + + private + + def assert_partial_value(expected, json) + parser = new_parser + parser << json + refute parser.parse + 2.times do + assert_equal expected, parser.partial_value + end + end + + def assert_resumed_parsing(json, parser = @parser) + expected = JSON.parse(json) + + last_parsed_byte_index = 0 + json.each_byte do |byte| + parser << byte.chr + last_parsed_byte_index += 1 + break if parser.parse + end + actual = parser.value + assert_equal expected, actual + remaining_bytes = (json.bytesize - last_parsed_byte_index) + assert_equal 0, remaining_bytes, "unconsumed bytes: #{actual.inspect}, remaining: #{json.byteslice(-1, remaining_bytes).inspect}" + end + + def assert_parse_stream(expected, json, parser = @parser) + actual = [] + parser << json + while parser.parse + actual << parser.value + end + assert_equal(expected, actual) + end + + def new_parser(options = nil) + JSON::ResumableParser.new(options) + end +end From 02346d82c756cde1b7892e26982d9fe1f2021014 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 15 Jun 2026 16:05:03 +0200 Subject: [PATCH 351/506] ResumableParser#partial_value avoid mutating the stacks Fix: https://github.com/ruby/json/issues/1005 `parser.state.frames` and `parser.state.value_stack` need to be updated to point to the copy. --- ext/json/ext/parser/parser.c | 3 +++ test/json/resumable_parser_test.rb | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 15e79b24e..e722a1a72 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2496,6 +2496,9 @@ static VALUE cResumableParser_partial_value(VALUE self) JSON_ResumableParser *original_parser = ResumableParser_acquire(self, false); JSON_ResumableParser parser = *original_parser; + parser.state.frames = &parser.frames; + parser.state.value_stack = &parser.value_stack; + if (parser.value_stack.head == 0) { return Qnil; } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index bab92c8de..600cd33e5 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -156,6 +156,20 @@ def test_partial_value assert_partial_value([1, { "a" => 1, "b" => { "c" => nil } }], '[1, { "a": 1, "b": { "c"') end + def test_partial_value_issue_1005 + data = <<~JSON + [ + [] + ] + JSON + data.each_line do |line| + @parser << line + @parser.parse + @parser.partial_value # This unexpected parse error doesn't happen if we comment this out + end + assert_equal [[]], @parser.value + end + def test_partial_value_missing assert_nil @parser.partial_value end From 68bcf605eb2a588184e12c95848cc603ff6afb04 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 07:55:59 +0200 Subject: [PATCH 352/506] Update Changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 949e3fff1..21c8fb54e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,7 +11,7 @@ ### 2026-06-11 (2.19.9) * Fix buffer overflow that could lead to a crash when writing JSON directly into an IO - with `JSON.generate(object, io)`. [CVE-PENDING]. + with `JSON.generate(object, io)`. [CVE-2026-54696]. ### 2026-06-03 (2.19.8) From 24a2b083d72ed1d6d0bf3b703cb99b7998b6d310 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 08:09:31 +0200 Subject: [PATCH 353/506] ResumableParser: Reset err_info on EOS errors Fix: https://github.com/ruby/json/issues/1008 I'm unfortunately unable to turn the reproduction script into a test case there, I don't understand what is causing `err_info` to be re-raised in the repro that doesn't work in test-unit. ```ruby require 'json' parser = JSON::ResumableParser.new({}) ['{"message": "hello ', 'world"}', '[1,2]'].each do |chunk| parser << chunk begin while parser.parse p parser.value end rescue JSON::ParserError => e p e parser.clear end end ``` --- ext/json/ext/parser/parser.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e722a1a72..fa66d0847 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2412,6 +2412,7 @@ static VALUE cResumableParser_parse(VALUE self) complete = false; if (RTEST(rb_ivar_get(rb_errinfo(), rb_intern("@eos")))) { complete = false; // is an EOS error + rb_set_errinfo(Qnil); } else { rb_jump_tag(status); // reraise } From 27009f624f57c3cd55044dbd3bc77c4954c6ee69 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 09:05:47 +0200 Subject: [PATCH 354/506] parser.c: Refactor `start` pointers --- ext/json/ext/parser/parser.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index fa66d0847..d7a9f22b7 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1543,6 +1543,8 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo json_eat_whitespace(state, config); VALUE value; + const char *value_start = state->cursor; + switch (peek(state)) { case 'n': json_match_keyword(state, "null", 0); @@ -1578,13 +1580,12 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo break; case '-': { - const char *start = state->cursor; state->cursor++; - value = json_parse_number(state, config, true, start); + value = json_parse_number(state, config, true, value_start); if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) { - state->cursor = start; + state->cursor = value_start; json_match_keyword(state, "-Infinity", 1); value = CMinusInfinity; break; @@ -1593,43 +1594,41 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo // Top level numbers are ambiguous when parsing streams, we can't // know if we parsed all the digits if we hit EOS. if (RB_UNLIKELY(resumable && eos(state))) { - state->cursor = start; + state->cursor = value_start; return false; } if (RB_UNLIKELY(UNDEF_P(value))) { - raise_syntax_error_at("invalid number: %s", state, start); + raise_syntax_error_at("invalid number: %s", state, value_start); } break; } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - const char *start = state->cursor; - - value = json_parse_number(state, config, false, start); + value = json_parse_number(state, config, false, value_start); // Top level numbers are ambiguous when parsing streams, we can't // know if we parsed all the digits if we hit EOS. if (RB_UNLIKELY(resumable && eos(state))) { - state->cursor = start; + state->cursor = value_start; return false; } if (RB_UNLIKELY(UNDEF_P(value))) { - raise_syntax_error_at("invalid number: %s", state, start); + raise_syntax_error_at("invalid number: %s", state, value_start); } break; } case '"': { // %r{\A"[^"\\\t\n\x00]*(?:\\[bfnrtu\\/"][^"\\]*)*"} - const char *start = state->cursor; value = json_parse_string(state, config, false); if (RB_UNLIKELY(UNDEF_P(value))) { bool is_eos = eos(state); - if (resumable) { - state->cursor = start; + if (resumable && is_eos) { + state->cursor = value_start; + return false; } raise_parse_error("unexpected end of input, expected closing \"", state, is_eos); } @@ -1637,7 +1636,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case '[': { - const char *start = state->cursor++; + state->cursor++; json_eat_whitespace(state, config); const char next = peek(state); @@ -1646,7 +1645,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo value = json_decode_array(state, config, 0); break; } else if (resumable && next == 0) { - state->cursor = start; + state->cursor = value_start; return false; } @@ -1666,8 +1665,6 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case '{': { - const char *object_start_cursor = state->cursor; - state->cursor++; json_eat_whitespace(state, config); @@ -1676,7 +1673,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo value = json_decode_object(state, config, 0); break; } else if (resumable && eos(state)) { - state->cursor = object_start_cursor; + state->cursor = value_start; return false; } @@ -1690,7 +1687,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo .type = JSON_FRAME_OBJECT, .phase = JSON_PHASE_OBJECT_KEY, .value_stack_head = state->value_stack->head, - .start_offset = object_start_cursor - state->start, + .start_offset = value_start - state->start, }); goto JSON_PHASE_OBJECT_KEY; } From 4185ef1a7b034e065b6667d7bbc8fadd0667f982 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 15 Jun 2026 14:53:16 +0200 Subject: [PATCH 355/506] Replace Ryu with a C99 port of fast_float MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contrary to Ryu, it remains correct up to 18 mantissa digits, meaning we don't have to fallback as much to Ruby's much slower `rb_cstr_to_dbl`. On `canada.json`, the most number heavy benchmark, `ffp_s2d` is less than 5% of total runtime. Most of the time is actually sent parsing, not decoding. There is a more complete C99 port of `fast_float` at https://github.com/kolemannix/ffc.h which also does parsing and could be worth considering, however it might make it much harder to detect when we're outside the safe range and to fallback, so unclear if it would be a win. Credit to Tilo Sloboda for the C99 port (sligthly reworked). ``` == Parsing citm_catalog.json (500124 bytes) ruby 4.0.5 (2026-05-20 revision 64336ffd0e) +YJIT +PRISM [arm64-darwin25] Warming up -------------------------------------- after 73.000 i/100ms Calculating ------------------------------------- after 733.301 (± 1.0%) i/s (1.36 ms/i) - 3.723k in 5.077044s Comparison: before: 716.8 i/s after: 733.3 i/s - 1.02x faster == Parsing float parsing (2090303 bytes) ruby 4.0.5 (2026-05-20 revision 64336ffd0e) +YJIT +PRISM [arm64-darwin25] Warming up -------------------------------------- after 35.000 i/100ms Calculating ------------------------------------- after 353.130 (± 0.3%) i/s (2.83 ms/i) - 1.785k in 5.054791s Comparison: before: 323.1 i/s after: 353.1 i/s - 1.09x faster ``` --- LEGAL | 6 +- ext/json/ext/parser/depend | 2 +- ext/json/ext/parser/parser.c | 8 +- ext/json/ext/vendor/fast_float_parser.h | 774 +++++++++++++++++ ext/json/ext/vendor/ryu.h | 819 ------------------ ....rb => json_float_parser_fallback_test.rb} | 42 +- test/json/json_parser_test.rb | 3 +- 7 files changed, 804 insertions(+), 850 deletions(-) create mode 100644 ext/json/ext/vendor/fast_float_parser.h delete mode 100644 ext/json/ext/vendor/ryu.h rename test/json/{json_ryu_fallback_test.rb => json_float_parser_fallback_test.rb} (85%) diff --git a/LEGAL b/LEGAL index dfc7189b0..3d3e0935a 100644 --- a/LEGAL +++ b/LEGAL @@ -15,6 +15,6 @@ ext/json/ext/vendor/jeaiii-ltoa.h:: This file is adapted from https://github.com/jeaiii/itoa It is licensed under the MIT License -ext/json/ext/vendor/ryu.h:: - This file is adapted from the Ryu algorithm by Ulf Adams https://github.com/ulfjack/ryu. - It is dual-licensed under Apache License 2.0 OR Boost Software License 1.0. +ext/json/ext/vendor/fast_float_parser.h:: + This file is adapted from the Fast Float C++ library by The fast_float authors https://github.com/fastfloat/fast_float + It is licensed under the MIT License diff --git a/ext/json/ext/parser/depend b/ext/json/ext/parser/depend index 5d75ef248..53aa89b70 100644 --- a/ext/json/ext/parser/depend +++ b/ext/json/ext/parser/depend @@ -2,6 +2,6 @@ parser.o: $(srcdir)/../fbuffer/fbuffer.h parser.o: $(srcdir)/../json.h parser.o: $(srcdir)/../simd/simd.h -parser.o: $(srcdir)/../vendor/ryu.h +parser.o: $(srcdir)/../vendor/fast_float_parser.h parser.o: parser.c # AUTOGENERATED DEPENDENCIES END diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d7a9f22b7..c216faf4b 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1,5 +1,5 @@ #include "../json.h" -#include "../vendor/ryu.h" +#include "../vendor/fast_float_parser.h" #include "../simd/simd.h" static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; @@ -1111,13 +1111,11 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis return rb_float_new(negative ? -0.0 : 0.0); } - // Fall back to rb_cstr_to_dbl for potential subnormals (rare edge case) - // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308) - if (RB_UNLIKELY(mantissa_digits > 17 || mantissa_digits + exponent < -307)) { + if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) { return json_decode_large_float(start, end - start); } - return DBL2NUM(ryu_s2d_from_parts(mantissa, mantissa_digits, (int32_t)exponent, negative)); + return DBL2NUM(ffp_s2d(exponent, mantissa, negative)); } static inline VALUE json_decode_array(JSON_ParserState *state, JSON_ParserConfig *config, long count) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h new file mode 100644 index 000000000..6bbb77dae --- /dev/null +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -0,0 +1,774 @@ +/* Eisel-Lemire decimal->double, ported from fast_float: + * include/fast_float/decimal_to_binary.h, the compute_float> + * + compute_product_approximation routines. + * + * Algorithm authors: Michael Eisel (original approach) and Daniel Lemire + * (formalization, proof, and the fast_float implementation) — hence "Eisel-Lemire". + * + * Copyright (c) 2021 The fast_float authors. Tri-licensed Apache-2.0 / MIT / BSL-1.0; + * used here under MIT — see LICENSE-fast_float-MIT in this directory. + * + * C99 Port by Tilo Sloboda. + */ + +#ifndef FFP_H +#define FFP_H +#include +#include +#include + +#define FFP_SMALLEST_POWER_OF_FIVE (-342) +#define FFP_LARGEST_POWER_OF_FIVE (308) +static const uint64_t ffp_power_of_five_128[2 * (308 - (-342) + 1)] = { + 0xeef453d6923bd65a, 0x113faa2906a13b3f, + 0x9558b4661b6565f8, 0x4ac7ca59a424c507, + 0xbaaee17fa23ebf76, 0x5d79bcf00d2df649, + 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, + 0x91d8a02bb6c10594, 0x79071b9b8a4be869, + 0xb64ec836a47146f9, 0x9748e2826cdee284, + 0xe3e27a444d8d98b7, 0xfd1b1b2308169b25, + 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, + 0xb208ef855c969f4f, 0xbdbd2d335e51a935, + 0xde8b2b66b3bc4723, 0xad2c788035e61382, + 0x8b16fb203055ac76, 0x4c3bcb5021afcc31, + 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, + 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, + 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, + 0xa9c98d8ccb009506, 0x680efdaf511f18c2, + 0xd43bf0effdc0ba48, 0x212bd1b2566def2, + 0x84a57695fe98746d, 0x14bb630f7604b57, + 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, + 0xcf42894a5dce35ea, 0x52064cac828675b9, + 0x818995ce7aa0e1b2, 0x7343efebd1940993, + 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, + 0xca66fa129f9b60a6, 0xd41a26e077774ef6, + 0xfd00b897478238d0, 0x8920b098955522b4, + 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, + 0xc5a890362fddbc62, 0xeb2189f734aa831d, + 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, + 0x9a6bb0aa55653b2d, 0x47b233c92125366e, + 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, + 0xf148440a256e2c76, 0xc00670ea43ca250d, + 0x96cd2a865764dbca, 0x380406926a5e5728, + 0xbc807527ed3e12bc, 0xc605083704f5ecf2, + 0xeba09271e88d976b, 0xf7864a44c633682e, + 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, + 0xb8157268fdae9e4c, 0x5960ea05bad82964, + 0xe61acf033d1a45df, 0x6fb92487298e33bd, + 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, + 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, + 0xe0b62e2929aba83c, 0x331acdabfe94de87, + 0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14, + 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, + 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, + 0x892731ac9faf056e, 0xbe311c083a225cd2, + 0xab70fe17c79ac6ca, 0x6dbd630a48aaf406, + 0xd64d3d9db981787d, 0x92cbbccdad5b108, + 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, + 0xa76c582338ed2621, 0xaf2af2b80af6f24e, + 0xd1476e2c07286faa, 0x1af5af660db4aee1, + 0x82cca4db847945ca, 0x50d98d9fc890ed4d, + 0xa37fce126597973c, 0xe50ff107bab528a0, + 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, + 0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a, + 0x9faacf3df73609b1, 0x77b191618c54e9ac, + 0xc795830d75038c1d, 0xd59df5b9ef6a2417, + 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, + 0x9becce62836ac577, 0x4ee367f9430aec32, + 0xc2e801fb244576d5, 0x229c41f793cda73f, + 0xf3a20279ed56d48a, 0x6b43527578c1110f, + 0x9845418c345644d6, 0x830a13896b78aaa9, + 0xbe5691ef416bd60c, 0x23cc986bc656d553, + 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, + 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, + 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, + 0xe858ad248f5c22c9, 0xd1b3400f8f9cff68, + 0x91376c36d99995be, 0x23100809b9c21fa1, + 0xb58547448ffffb2d, 0xabd40a0c2832a78a, + 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, + 0x8dd01fad907ffc3b, 0xae3da7d97f6792e3, + 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, + 0xdd95317f31c7fa1d, 0x40405643d711d583, + 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, + 0xad1c8eab5ee43b66, 0xda3243650005eecf, + 0xd863b256369d4a40, 0x90bed43e40076a82, + 0x873e4f75e2224e68, 0x5a7744a6e804a291, + 0xa90de3535aaae202, 0x711515d0a205cb36, + 0xd3515c2831559a83, 0xd5a5b44ca873e03, + 0x8412d9991ed58091, 0xe858790afe9486c2, + 0xa5178fff668ae0b6, 0x626e974dbe39a872, + 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, + 0x80fa687f881c7f8e, 0x7ce66634bc9d0b99, + 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, + 0xc987434744ac874e, 0xa327ffb266b56220, + 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, + 0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9, + 0xc4ce17b399107c22, 0xcb550fb4384d21d3, + 0xf6019da07f549b2b, 0x7e2a53a146606a48, + 0x99c102844f94e0fb, 0x2eda7444cbfc426d, + 0xc0314325637a1939, 0xfa911155fefb5308, + 0xf03d93eebc589f88, 0x793555ab7eba27ca, + 0x96267c7535b763b5, 0x4bc1558b2f3458de, + 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, + 0xea9c227723ee8bcb, 0x465e15a979c1cadc, + 0x92a1958a7675175f, 0xbfacd89ec191ec9, + 0xb749faed14125d36, 0xcef980ec671f667b, + 0xe51c79a85916f484, 0x82b7e12780e7401a, + 0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810, + 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, + 0xdfbdcece67006ac9, 0x67a791e093e1d49a, + 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, + 0xaecc49914078536d, 0x58fae9f773886e18, + 0xda7f5bf590966848, 0xaf39a475506a899e, + 0x888f99797a5e012d, 0x6d8406c952429603, + 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, + 0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64, + 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, + 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, + 0xd0601d8efc57b08b, 0xf13b94daf124da26, + 0x823c12795db6ce57, 0x76c53d08d6b70858, + 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, + 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, + 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, + 0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf, + 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, + 0xf867241c8cc6d4c0, 0xc30163d203c94b62, + 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, + 0xc21094364dfb5636, 0x985915fc12f542e4, + 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, + 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, + 0xbd8430bd08277231, 0x50c6ff782a838353, + 0xece53cec4a314ebd, 0xa4f8bf5635246428, + 0x940f4613ae5ed136, 0x871b7795e136be99, + 0xb913179899f68584, 0x28e2557b59846e3f, + 0xe757dd7ec07426e5, 0x331aeada2fe589cf, + 0x9096ea6f3848984f, 0x3ff0d2c85def7621, + 0xb4bca50b065abe63, 0xfed077a756b53a9, + 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, + 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, + 0xb080392cc4349dec, 0xbd8d794d96aacfb3, + 0xdca04777f541c567, 0xecf0d7a0fc5583a0, + 0x89e42caaf9491b60, 0xf41686c49db57244, + 0xac5d37d5b79b6239, 0x311c2875c522ced5, + 0xd77485cb25823ac7, 0x7d633293366b828b, + 0x86a8d39ef77164bc, 0xae5dff9c02033197, + 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, + 0xd267caa862a12d66, 0xd072df63c324fd7b, + 0x8380dea93da4bc60, 0x4247cb9e59f71e6d, + 0xa46116538d0deb78, 0x52d9be85f074e608, + 0xcd795be870516656, 0x67902e276c921f8b, + 0x806bd9714632dff6, 0xba1cd8a3db53b6, + 0xa086cfcd97bf97f3, 0x80e8a40eccd228a4, + 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, + 0xfad2a4b13d1b5d6c, 0x796b805720085f81, + 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, + 0xc3f490aa77bd60fc, 0xbedbfc4411068a9c, + 0xf4f1b4d515acb93b, 0xee92fb5515482d44, + 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, + 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, + 0xef340a98172aace4, 0x86fb897116c87c34, + 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, + 0xbae0a846d2195712, 0x8974836059cca109, + 0xe998d258869facd7, 0x2bd1a438703fc94b, + 0x91ff83775423cc06, 0x7b6306a34627ddcf, + 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, + 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, + 0x8e938662882af53e, 0x547eb47b7282ee9c, + 0xb23867fb2a35b28d, 0xe99e619a4f23aa43, + 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, + 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, + 0xae0b158b4738705e, 0x9624ab50b148d445, + 0xd98ddaee19068c76, 0x3badd624dd9b0957, + 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, + 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, + 0xd47487cc8470652b, 0x7647c3200069671f, + 0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073, + 0xa5fb0a17c777cf09, 0xf468107100525890, + 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, + 0x81ac1fe293d599bf, 0xc6f14cd848405530, + 0xa21727db38cb002f, 0xb8ada00e5a506a7c, + 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, + 0xfd442e4688bd304a, 0x908f4a166d1da663, + 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, + 0xc5dd44271ad3cdba, 0x40eff1e1853f29fd, + 0xf7549530e188c128, 0xd12bee59e68ef47c, + 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, + 0xc13a148e3032d6e7, 0xe36a52363c1faf01, + 0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1, + 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, + 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, + 0xebdf661791d60f56, 0x111b495b3464ad21, + 0x936b9fcebb25c995, 0xcab10dd900beec34, + 0xb84687c269ef3bfb, 0x3d5d514f40eea742, + 0xe65829b3046b0afa, 0xcb4a5a3112a5112, + 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, + 0xb3f4e093db73a093, 0x59ed216765690f56, + 0xe0f218b8d25088b8, 0x306869c13ec3532c, + 0x8c974f7383725573, 0x1e414218c73a13fb, + 0xafbd2350644eeacf, 0xe5d1929ef90898fa, + 0xdbac6c247d62a583, 0xdf45f746b74abf39, + 0x894bc396ce5da772, 0x6b8bba8c328eb783, + 0xab9eb47c81f5114f, 0x66ea92f3f326564, + 0xd686619ba27255a2, 0xc80a537b0efefebd, + 0x8613fd0145877585, 0xbd06742ce95f5f36, + 0xa798fc4196e952e7, 0x2c48113823b73704, + 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, + 0x82ef85133de648c4, 0x9a984d73dbe722fb, + 0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba, + 0xcc963fee10b7d1b3, 0x318df905079926a8, + 0xffbbcfe994e5c61f, 0xfdf17746497f7052, + 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, + 0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0, + 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, + 0x9c1661a651213e2d, 0x6bea10ca65c084e, + 0xc31bfa0fe5698db8, 0x486e494fcff30a62, + 0xf3e2f893dec3f126, 0x5a89dba3c3efccfa, + 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, + 0xbe89523386091465, 0xf6bbb397f1135823, + 0xee2ba6c0678b597f, 0x746aa07ded582e2c, + 0x94db483840b717ef, 0xa8c2a44eb4571cdc, + 0xba121a4650e4ddeb, 0x92f34d62616ce413, + 0xe896a0d7e51e1566, 0x77b020baf9c81d17, + 0x915e2486ef32cd60, 0xace1474dc1d122e, + 0xb5b5ada8aaff80b8, 0xd819992132456ba, + 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, + 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, + 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, + 0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde, + 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, + 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, + 0xd89d64d57a607744, 0xe871c7bf077ba8b7, + 0x87625f056c7c4a8b, 0x11471cd764ad4972, + 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, + 0xd389b47879823479, 0x4aff1d108d4ec2c3, + 0x843610cb4bf160cb, 0xcedf722a585139ba, + 0xa54394fe1eedb8fe, 0xc2974eb4ee658828, + 0xce947a3da6a9273e, 0x733d226229feea32, + 0x811ccc668829b887, 0x806357d5a3f525f, + 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, + 0xc9bcff6034c13052, 0xfc89b393dd02f0b5, + 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, + 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, + 0xc5029163f384a931, 0xa9e795e65d4df11, + 0xf64335bcf065d37d, 0x4d4617b5ff4a16d5, + 0x99ea0196163fa42e, 0x504bced1bf8e4e45, + 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, + 0xf07da27a82c37088, 0x5d767327bb4e5a4c, + 0x964e858c91ba2655, 0x3a6a07f8d510f86f, + 0xbbe226efb628afea, 0x890489f70a55368b, + 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, + 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, + 0xb77ada0617e3bbcb, 0x9ce6ebb40173744, + 0xe55990879ddcaabd, 0xcc420a6a101d0515, + 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, + 0xb32df8e9f3546564, 0x47939822dc96abf9, + 0xdff9772470297ebd, 0x59787e2b93bc56f7, + 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, + 0xaefae51477a06b03, 0xede622920b6b23f1, + 0xdab99e59958885c4, 0xe95fab368e45eced, + 0x88b402f7fd75539b, 0x11dbcb0218ebb414, + 0xaae103b5fcd2a881, 0xd652bdc29f26a119, + 0xd59944a37c0752a2, 0x4be76d3346f0495f, + 0x857fcae62d8493a5, 0x6f70a4400c562ddb, + 0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952, + 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, + 0x825ecc24c873782f, 0x8ed400668c0c28c8, + 0xa2f67f2dfa90563b, 0x728900802f0f32fa, + 0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9, + 0xfea126b7d78186bc, 0xe2f610c84987bfa8, + 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, + 0xc6ede63fa05d3143, 0x91503d1c79720dbb, + 0xf8a95fcf88747d94, 0x75a44c6397ce912a, + 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, + 0xc24452da229b021b, 0xfbe85badce996168, + 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, + 0x97c560ba6b0919a5, 0xdccd879fc967d41a, + 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, + 0xed246723473e3813, 0x290123e9aab23b68, + 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, + 0xb94470938fa89bce, 0xf808e40e8d5b3e69, + 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, + 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, + 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, + 0xe2280b6c20dd5232, 0x25c6da63c38de1b0, + 0x8d590723948a535f, 0x579c487e5a38ad0e, + 0xb0af48ec79ace837, 0x2d835a9df0c6d851, + 0xdcdb1b2798182244, 0xf8e431456cf88e65, + 0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff, + 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, + 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, + 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, + 0xa87fea27a539e9a5, 0x3f2398d747b36224, + 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, + 0x83a3eeeef9153e89, 0x1953cf68300424ac, + 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, + 0xcdb02555653131b6, 0x3792f412cb06794d, + 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, + 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, + 0xc8de047564d20a8b, 0xf245825a5a445275, + 0xfb158592be068d2e, 0xeed6e2f0f0d56712, + 0x9ced737bb6c4183d, 0x55464dd69685606b, + 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, + 0xf53304714d9265df, 0xd53dd99f4b3066a8, + 0x993fe2c6d07b7fab, 0xe546a8038efe4029, + 0xbf8fdb78849a5f96, 0xde98520472bdd033, + 0xef73d256a5c0f77c, 0x963e66858f6d4440, + 0x95a8637627989aad, 0xdde7001379a44aa8, + 0xbb127c53b17ec159, 0x5560c018580d5d52, + 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, + 0x9226712162ab070d, 0xcab3961304ca70e8, + 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, + 0xe45c10c42a2b3b05, 0x8cb89a7db77c506a, + 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, + 0xb267ed1940f1c61c, 0x55f038b237591ed3, + 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, + 0x8b61313bbabce2c6, 0x2323ac4b3b3da015, + 0xae397d8aa96c1b77, 0xabec975e0a0d081a, + 0xd9c7dced53c72255, 0x96e7bd358c904a21, + 0x881cea14545c7575, 0x7e50d64177da2e54, + 0xaa242499697392d2, 0xdde50bd1d5d0b9e9, + 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, + 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, + 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, + 0xcfb11ead453994ba, 0x67de18eda5814af2, + 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, + 0xa2425ff75e14fc31, 0xa1258379a94d028d, + 0xcad2f7f5359a3b3e, 0x96ee45813a04330, + 0xfd87b5f28300ca0d, 0x8bca9d6e188853fc, + 0x9e74d1b791e07e48, 0x775ea264cf55347e, + 0xc612062576589dda, 0x95364afe032a819e, + 0xf79687aed3eec551, 0x3a83ddbd83f52205, + 0x9abe14cd44753b52, 0xc4926a9672793543, + 0xc16d9a0095928a27, 0x75b7053c0f178294, + 0xf1c90080baf72cb1, 0x5324c68b12dd6339, + 0x971da05074da7bee, 0xd3f6fc16ebca5e04, + 0xbce5086492111aea, 0x88f4bb1ca6bcf585, + 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, + 0x9392ee8e921d5d07, 0x3aff322e62439fd0, + 0xb877aa3236a4b449, 0x9befeb9fad487c3, + 0xe69594bec44de15b, 0x4c2ebe687989a9b4, + 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, + 0xb424dc35095cd80f, 0x538484c19ef38c95, + 0xe12e13424bb40e13, 0x2865a5f206b06fba, + 0x8cbccc096f5088cb, 0xf93f87b7442e45d4, + 0xafebff0bcb24aafe, 0xf78f69a51539d749, + 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, + 0x89705f4136b4a597, 0x31680a88f8953031, + 0xabcc77118461cefc, 0xfdc20d2b36ba7c3e, + 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, + 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, + 0xa7c5ac471b478423, 0xfcf80dc33721d54, + 0xd1b71758e219652b, 0xd3c36113404ea4a9, + 0x83126e978d4fdf3b, 0x645a1cac083126ea, + 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, + 0xcccccccccccccccc, 0xcccccccccccccccd, + 0x8000000000000000, 0x0, + 0xa000000000000000, 0x0, + 0xc800000000000000, 0x0, + 0xfa00000000000000, 0x0, + 0x9c40000000000000, 0x0, + 0xc350000000000000, 0x0, + 0xf424000000000000, 0x0, + 0x9896800000000000, 0x0, + 0xbebc200000000000, 0x0, + 0xee6b280000000000, 0x0, + 0x9502f90000000000, 0x0, + 0xba43b74000000000, 0x0, + 0xe8d4a51000000000, 0x0, + 0x9184e72a00000000, 0x0, + 0xb5e620f480000000, 0x0, + 0xe35fa931a0000000, 0x0, + 0x8e1bc9bf04000000, 0x0, + 0xb1a2bc2ec5000000, 0x0, + 0xde0b6b3a76400000, 0x0, + 0x8ac7230489e80000, 0x0, + 0xad78ebc5ac620000, 0x0, + 0xd8d726b7177a8000, 0x0, + 0x878678326eac9000, 0x0, + 0xa968163f0a57b400, 0x0, + 0xd3c21bcecceda100, 0x0, + 0x84595161401484a0, 0x0, + 0xa56fa5b99019a5c8, 0x0, + 0xcecb8f27f4200f3a, 0x0, + 0x813f3978f8940984, 0x4000000000000000, + 0xa18f07d736b90be5, 0x5000000000000000, + 0xc9f2c9cd04674ede, 0xa400000000000000, + 0xfc6f7c4045812296, 0x4d00000000000000, + 0x9dc5ada82b70b59d, 0xf020000000000000, + 0xc5371912364ce305, 0x6c28000000000000, + 0xf684df56c3e01bc6, 0xc732000000000000, + 0x9a130b963a6c115c, 0x3c7f400000000000, + 0xc097ce7bc90715b3, 0x4b9f100000000000, + 0xf0bdc21abb48db20, 0x1e86d40000000000, + 0x96769950b50d88f4, 0x1314448000000000, + 0xbc143fa4e250eb31, 0x17d955a000000000, + 0xeb194f8e1ae525fd, 0x5dcfab0800000000, + 0x92efd1b8d0cf37be, 0x5aa1cae500000000, + 0xb7abc627050305ad, 0xf14a3d9e40000000, + 0xe596b7b0c643c719, 0x6d9ccd05d0000000, + 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, + 0xb35dbf821ae4f38b, 0xdda2802c8a800000, + 0xe0352f62a19e306e, 0xd50b2037ad200000, + 0x8c213d9da502de45, 0x4526f422cc340000, + 0xaf298d050e4395d6, 0x9670b12b7f410000, + 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, + 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, + 0xab0e93b6efee0053, 0x8eea0d047a457a00, + 0xd5d238a4abe98068, 0x72a4904598d6d880, + 0x85a36366eb71f041, 0x47a6da2b7f864750, + 0xa70c3c40a64e6c51, 0x999090b65f67d924, + 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, + 0x82818f1281ed449f, 0xbff8f10e7a8921a4, + 0xa321f2d7226895c7, 0xaff72d52192b6a0d, + 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, + 0xfee50b7025c36a08, 0x2f236d04753d5b4, + 0x9f4f2726179a2245, 0x1d762422c946590, + 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, + 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, + 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, + 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, + 0xf316271c7fc3908a, 0x8bef464e3945ef7a, + 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, + 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, + 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, + 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, + 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, + 0xe7d34c64a9c85d44, 0x60dbbca87196b616, + 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, + 0xb51d13aea4a488dd, 0x6babab6398bdbe41, + 0xe264589a4dcdab14, 0xc696963c7eed2dd1, + 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, + 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, + 0xdd15fe86affad912, 0x49ef0eb713f39ebe, + 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, + 0xacb92ed9397bf996, 0x49c2c37f07965404, + 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, + 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, + 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, + 0xd2d80db02aabd62b, 0xf50a3fa490c30190, + 0x83c7088e1aab65db, 0x792667c6da79e0fa, + 0xa4b8cab1a1563f52, 0x577001b891185938, + 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, + 0x80b05e5ac60b6178, 0x544f8158315b05b4, + 0xa0dc75f1778e39d6, 0x696361ae3db1c721, + 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, + 0xfb5878494ace3a5f, 0x4ab48a04065c723, + 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, + 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, + 0xf5746577930d6500, 0xca8f44ec7ee36479, + 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, + 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, + 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, + 0x95d04aee3b80ece5, 0xbba1f1d158724a12, + 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, + 0xea1575143cf97226, 0xf52d09d71a3293bd, + 0x924d692ca61be758, 0x593c2626705f9c56, + 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, + 0xe498f455c38b997a, 0xb6dfb9c0f956447, + 0x8edf98b59a373fec, 0x4724bd4189bd5eac, + 0xb2977ee300c50fe7, 0x58edec91ec2cb657, + 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, + 0x8b865b215899f46c, 0xbd79e0d20082ee74, + 0xae67f1e9aec07187, 0xecd8590680a3aa11, + 0xda01ee641a708de9, 0xe80e6f4820cc9495, + 0x884134fe908658b2, 0x3109058d147fdcdd, + 0xaa51823e34a7eede, 0xbd4b46f0599fd415, + 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, + 0x850fadc09923329e, 0x3e2cf6bc604ddb0, + 0xa6539930bf6bff45, 0x84db8346b786151c, + 0xcfe87f7cef46ff16, 0xe612641865679a63, + 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, + 0xa26da3999aef7749, 0xe3be5e330f38f09d, + 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, + 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, + 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, + 0xc646d63501a1511d, 0xb281e1fd541501b8, + 0xf7d88bc24209a565, 0x1f225a7ca91a4226, + 0x9ae757596946075f, 0x3375788de9b06958, + 0xc1a12d2fc3978937, 0x52d6b1641c83ae, + 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, + 0x9745eb4d50ce6332, 0xf840b7ba963646e0, + 0xbd176620a501fbff, 0xb650e5a93bc3d898, + 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, + 0x93ba47c980e98cdf, 0xc66f336c36b10137, + 0xb8a8d9bbe123f017, 0xb80b0047445d4184, + 0xe6d3102ad96cec1d, 0xa60dc059157491e5, + 0x9043ea1ac7e41392, 0x87c89837ad68db2f, + 0xb454e4a179dd1877, 0x29babe4598c311fb, + 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, + 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, + 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, + 0xdc21a1171d42645d, 0x76707543f4fa1f73, + 0x899504ae72497eba, 0x6a06494a791c53a8, + 0xabfa45da0edbde69, 0x487db9d17636892, + 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, + 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, + 0xa7f26836f282b732, 0x8e6cac7768d7141e, + 0xd1ef0244af2364ff, 0x3207d795430cd926, + 0x8335616aed761f1f, 0x7f44e6bd49e807b8, + 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, + 0xcd036837130890a1, 0x36dba887c37a8c0f, + 0x802221226be55a64, 0xc2494954da2c9789, + 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, + 0xc83553c5c8965d3d, 0x6f92829494e5acc7, + 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, + 0x9c69a97284b578d7, 0xff2a760414536efb, + 0xc38413cf25e2d70d, 0xfef5138519684aba, + 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, + 0x98bf2f79d5993802, 0xef2f773ffbd97a61, + 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, + 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, + 0x952ab45cfa97a0b2, 0xdd945a747bf26183, + 0xba756174393d88df, 0x94f971119aeef9e4, + 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, + 0x91abb422ccb812ee, 0xac62e055c10ab33a, + 0xb616a12b7fe617aa, 0x577b986b314d6009, + 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, + 0x8e41ade9fbebc27d, 0x14588f13be847307, + 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, + 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, + 0x8aec23d680043bee, 0x25de7bb9480d5854, + 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, + 0xd910f7ff28069da4, 0x1b2ba1518094da04, + 0x87aa9aff79042286, 0x90fb44d2f05d0842, + 0xa99541bf57452b28, 0x353a1607ac744a53, + 0xd3fa922f2d1675f2, 0x42889b8997915ce8, + 0x847c9b5d7c2e09b7, 0x69956135febada11, + 0xa59bc234db398c25, 0x43fab9837e699095, + 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, + 0x8161afb94b44f57d, 0x1d1be0eebac278f5, + 0xa1ba1ba79e1632dc, 0x6462d92a69731732, + 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, + 0xfcb2cb35e702af78, 0x5cda735244c3d43e, + 0x9defbf01b061adab, 0x3a0888136afa64a7, + 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, + 0xf6c69a72a3989f5b, 0x8aad549e57273d45, + 0x9a3c2087a63f6399, 0x36ac54e2f678864b, + 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, + 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, + 0x969eb7c47859e743, 0x9f644ae5a4b1b325, + 0xbc4665b596706114, 0x873d5d9f0dde1fee, + 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, + 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, + 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, + 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, + 0x8fa475791a569d10, 0xf96e017d694487bc, + 0xb38d92d760ec4455, 0x37c981dcc395a9ac, + 0xe070f78d3927556a, 0x85bbe253f47b1417, + 0x8c469ab843b89562, 0x93956d7478ccec8e, + 0xaf58416654a6babb, 0x387ac8d1970027b2, + 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, + 0x88fcf317f22241e2, 0x441fece3bdf81f03, + 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, + 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, + 0x85c7056562757456, 0xf6872d5667844e49, + 0xa738c6bebb12d16c, 0xb428f8ac016561db, + 0xd106f86e69d785c7, 0xe13336d701beba52, + 0x82a45b450226b39c, 0xecc0024661173473, + 0xa34d721642b06084, 0x27f002d7f95d0190, + 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, + 0xff290242c83396ce, 0x7e67047175a15271, + 0x9f79a169bd203e41, 0xf0062c6e984d386, + 0xc75809c42c684dd1, 0x52c07b78a3e60868, + 0xf92e0c3537826145, 0xa7709a56ccdf8a82, + 0x9bbcc7a142b17ccb, 0x88a66076400bb691, + 0xc2abf989935ddbfe, 0x6acff893d00ea435, + 0xf356f7ebf83552fe, 0x583f6b8c4124d43, + 0x98165af37b2153de, 0xc3727a337a8b704a, + 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, + 0xeda2ee1c7064130c, 0x1162def06f79df73, + 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, + 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, + 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, + 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, + 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, + 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, + 0x8da471a9de737e24, 0x5ceaecfed289e5d2, + 0xb10d8e1456105dad, 0x7425a83e872c5f47, + 0xdd50f1996b947518, 0xd12f124e28f77719, + 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, + 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, + 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, + 0x8714a775e3e95c78, 0x65acfaec34810a71, + 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, + 0xd31045a8341ca07c, 0x1ede48111209a050, + 0x83ea2b892091e44d, 0x934aed0aab460432, + 0xa4e4b66b68b65d60, 0xf81da84d5617853f, + 0xce1de40642e3f4b9, 0x36251260ab9d668e, + 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, + 0xa1075a24e4421730, 0xb24cf65b8612f81f, + 0xc94930ae1d529cfc, 0xdee033f26797b627, + 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, + 0x9d412e0806e88aa5, 0x8e1f289560ee864e, + 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, + 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, + 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, + 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, + 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, + 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, + 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, + 0xea53df5fd18d5513, 0x84c86189216dc5ed, + 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, + 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, + 0xe4d5e82392a40515, 0xfabaf3feaa5334a, + 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, + 0xb2c71d5bca9023f8, 0x743e20e9ef511012, + 0xdf78e4b2bd342cf6, 0x914da9246b255416, + 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, + 0xae9672aba3d0c320, 0xa184ac2473b529b1, + 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, + 0x8865899617fb1871, 0x7e2fa67c7a658892, + 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, + 0xd51ea6fa85785631, 0x552a74227f3ea565, + 0x8533285c936b35de, 0xd53a88958f87275f, + 0xa67ff273b8460356, 0x8a892abaf368f137, + 0xd01fef10a657842c, 0x2d2b7569b0432d85, + 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, + 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, + 0xcb3f2f7642717713, 0x241c70a936219a73, + 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, + 0x9ec95d1463e8a506, 0xf4363804324a40aa, + 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, + 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, + 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, + 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, + 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, + 0x976e41088617ca01, 0xd5be0503e085d813, + 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, + 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, + 0x93e1ab8252f33b45, 0xcabb90e5c942b503, + 0xb8da1662e7b00a17, 0x3d6a751f3b936243, + 0xe7109bfba19c0c9d, 0xcc512670a783ad4, + 0x906a617d450187e2, 0x27fb2b80668b24c5, + 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, + 0xe1a63853bbd26451, 0x5e7873f8a0396973, + 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, + 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, + 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, + 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, + 0xac2820d9623bf429, 0x546345fa9fbdcd44, + 0xd732290fbacaf133, 0xa97c177947ad4095, + 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, + 0xa81f301449ee8c70, 0x5c68f256bfff5a74, + 0xd226fc195c6a2f8c, 0x73832eec6fff3111, + 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, + 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, + 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, + 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, + 0xa0555e361951c366, 0xd7e105bcc332621f, + 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, + 0xfa856334878fc150, 0xb14f98f6f0feb951, + 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, + 0xc3b8358109e84f07, 0xa862f80ec4700c8, + 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, + 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, + 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, + 0xeeea5d5004981478, 0x1858ccfce06cac74, + 0x95527a5202df0ccb, 0xf37801e0c43ebc8, + 0xbaa718e68396cffd, 0xd30560258f54e6ba, + 0xe950df20247c83fd, 0x47c6b82ef32a2069, + 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, + 0xb6472e511c81471d, 0xe0133fe4adf8e952, + 0xe3d8f9e563a198e5, 0x58180fddd97723a6, + 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648, +}; + +/* binary_format constants from fast_float. */ +#define FFP_MANTISSA_BITS 52 +#define FFP_MIN_EXPONENT (-1023) +#define FFP_INFINITE_POWER 0x7FF +#define FFP_SMALLEST_POW10 (-342) +#define FFP_LARGEST_POW10 308 +#define FFP_MIN_RTE (-4) /* min_exponent_round_to_even */ +#define FFP_MAX_RTE 23 /* max_exponent_round_to_even */ + +/* (((152170 + 65536) * q) >> 16) + 63 == floor(log2(10^q)) + q + 63, see paper. */ +static inline int32_t ffp_power(int32_t q) { + return (((152170 + 65536) * q) >> 16) + 63; +} + +static inline void ffp_mul128(uint64_t a, uint64_t b, uint64_t *hi, uint64_t *lo) { +#if defined(__SIZEOF_INT128__) + __uint128_t p = (__uint128_t)a * (__uint128_t)b; + *lo = (uint64_t)p; + *hi = (uint64_t)(p >> 64); +#else + uint64_t a0 = (uint32_t)a, a1 = a >> 32, b0 = (uint32_t)b, b1 = b >> 32; + uint64_t p00 = a0 * b0, p01 = a0 * b1, p10 = a1 * b0, p11 = a1 * b1; + uint64_t mid = p10 + (p00 >> 32) + (uint32_t)p01; + *hi = p11 + (mid >> 32) + (p01 >> 32); + *lo = (mid << 32) | (uint32_t)p00; +#endif +} + +static inline double ffp_bits2double(uint64_t bits) { + double d; + memcpy(&d, &bits, sizeof(d)); + return d; +} + +/* q = power of ten, w = mantissa (exact, fits in uint64). neg = sign. */ +static inline double ffp_s2d(int64_t q, uint64_t w, bool neg) { + if (w == 0) { + return neg ? -0.0 : 0.0; + } + + const uint64_t sign = (uint64_t)(neg != 0) << 63; + uint64_t mantissa, prod_hi, prod_lo, sp_hi, sp_lo; + int32_t power2; + int lz, upperbit, shift, index; + + if (q < FFP_SMALLEST_POW10) return ffp_bits2double(sign); /* underflow -> 0 */ + if (q > FFP_LARGEST_POW10) { + return ffp_bits2double(sign | ((uint64_t)FFP_INFINITE_POWER << FFP_MANTISSA_BITS)); + } + + lz = __builtin_clzll(w); + w <<= lz; + + /* compute_product_approximation: precision_mask = 0x1FF. */ + index = 2 * (int)(q - FFP_SMALLEST_POWER_OF_FIVE); + ffp_mul128(w, ffp_power_of_five_128[index], &prod_hi, &prod_lo); + if ((prod_hi & 0x1FF) == 0x1FF) { + ffp_mul128(w, ffp_power_of_five_128[index + 1], &sp_hi, &sp_lo); + prod_lo += sp_hi; + if (sp_hi > prod_lo) { + prod_hi++; + } + } + + upperbit = (int)(prod_hi >> 63); + shift = upperbit + 64 - FFP_MANTISSA_BITS - 3; /* upperbit + 9 */ + mantissa = prod_hi >> shift; + power2 = (int32_t)(ffp_power((int32_t)q) + upperbit - lz - FFP_MIN_EXPONENT); + + if (power2 <= 0) { /* subnormal */ + if (-power2 + 1 >= 64) return ffp_bits2double(sign); /* far below min -> 0 */ + mantissa >>= (-power2 + 1); + mantissa += (mantissa & 1); + mantissa >>= 1; + power2 = (mantissa < ((uint64_t)1 << FFP_MANTISSA_BITS)) ? 0 : 1; + return ffp_bits2double(sign | ((uint64_t)power2 << FFP_MANTISSA_BITS) | mantissa); + } + + /* round-to-even: if we land exactly between two doubles, round down. */ + if ((prod_lo <= 1) && (q >= FFP_MIN_RTE) && (q <= FFP_MAX_RTE) && + ((mantissa & 3) == 1) && ((mantissa << shift) == prod_hi)) { + mantissa &= ~(uint64_t)1; + } + + mantissa += (mantissa & 1); + mantissa >>= 1; + if (mantissa >= ((uint64_t)2 << FFP_MANTISSA_BITS)) { + mantissa = (uint64_t)1 << FFP_MANTISSA_BITS; + power2++; + } + + mantissa &= ~((uint64_t)1 << FFP_MANTISSA_BITS); /* drop implicit bit */ + if (power2 >= FFP_INFINITE_POWER) { + return ffp_bits2double(sign | ((uint64_t)FFP_INFINITE_POWER << FFP_MANTISSA_BITS)); + } + return ffp_bits2double(sign | ((uint64_t)power2 << FFP_MANTISSA_BITS) | mantissa); +} + +#endif // FFP_H diff --git a/ext/json/ext/vendor/ryu.h b/ext/json/ext/vendor/ryu.h deleted file mode 100644 index f06ec814b..000000000 --- a/ext/json/ext/vendor/ryu.h +++ /dev/null @@ -1,819 +0,0 @@ -// Copyright 2018 Ulf Adams -// -// The contents of this file may be used under the terms of the Apache License, -// Version 2.0. -// -// Alternatively, the contents of this file may be used under the terms of -// the Boost Software License, Version 1.0. -// -// Unless required by applicable law or agreed to in writing, this software -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. -// -// --- -// -// Apache License -// Version 2.0, January 2004 -// http://www.apache.org/licenses/ -// -// TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -// -// 1. Definitions. -// -// "License" shall mean the terms and conditions for use, reproduction, -// and distribution as defined by Sections 1 through 9 of this document. -// -// "Licensor" shall mean the copyright owner or entity authorized by -// the copyright owner that is granting the License. -// -// "Legal Entity" shall mean the union of the acting entity and all -// other entities that control, are controlled by, or are under common -// control with that entity. For the purposes of this definition, -// "control" means (i) the power, direct or indirect, to cause the -// direction or management of such entity, whether by contract or -// otherwise, or (ii) ownership of fifty percent (50%) or more of the -// outstanding shares, or (iii) beneficial ownership of such entity. -// -// "You" (or "Your") shall mean an individual or Legal Entity -// exercising permissions granted by this License. -// -// "Source" form shall mean the preferred form for making modifications, -// including but not limited to software source code, documentation -// source, and configuration files. -// -// "Object" form shall mean any form resulting from mechanical -// transformation or translation of a Source form, including but -// not limited to compiled object code, generated documentation, -// and conversions to other media types. -// -// "Work" shall mean the work of authorship, whether in Source or -// Object form, made available under the License, as indicated by a -// copyright notice that is included in or attached to the work -// (an example is provided in the Appendix below). -// -// "Derivative Works" shall mean any work, whether in Source or Object -// form, that is based on (or derived from) the Work and for which the -// editorial revisions, annotations, elaborations, or other modifications -// represent, as a whole, an original work of authorship. For the purposes -// of this License, Derivative Works shall not include works that remain -// separable from, or merely link (or bind by name) to the interfaces of, -// the Work and Derivative Works thereof. -// -// "Contribution" shall mean any work of authorship, including -// the original version of the Work and any modifications or additions -// to that Work or Derivative Works thereof, that is intentionally -// submitted to Licensor for inclusion in the Work by the copyright owner -// or by an individual or Legal Entity authorized to submit on behalf of -// the copyright owner. For the purposes of this definition, "submitted" -// means any form of electronic, verbal, or written communication sent -// to the Licensor or its representatives, including but not limited to -// communication on electronic mailing lists, source code control systems, -// and issue tracking systems that are managed by, or on behalf of, the -// Licensor for the purpose of discussing and improving the Work, but -// excluding communication that is conspicuously marked or otherwise -// designated in writing by the copyright owner as "Not a Contribution." -// -// "Contributor" shall mean Licensor and any individual or Legal Entity -// on behalf of whom a Contribution has been received by Licensor and -// subsequently incorporated within the Work. -// -// 2. Grant of Copyright License. Subject to the terms and conditions of -// this License, each Contributor hereby grants to You a perpetual, -// worldwide, non-exclusive, no-charge, royalty-free, irrevocable -// copyright license to reproduce, prepare Derivative Works of, -// publicly display, publicly perform, sublicense, and distribute the -// Work and such Derivative Works in Source or Object form. -// -// 3. Grant of Patent License. Subject to the terms and conditions of -// this License, each Contributor hereby grants to You a perpetual, -// worldwide, non-exclusive, no-charge, royalty-free, irrevocable -// (except as stated in this section) patent license to make, have made, -// use, offer to sell, sell, import, and otherwise transfer the Work, -// where such license applies only to those patent claims licensable -// by such Contributor that are necessarily infringed by their -// Contribution(s) alone or by combination of their Contribution(s) -// with the Work to which such Contribution(s) was submitted. If You -// institute patent litigation against any entity (including a -// cross-claim or counterclaim in a lawsuit) alleging that the Work -// or a Contribution incorporated within the Work constitutes direct -// or contributory patent infringement, then any patent licenses -// granted to You under this License for that Work shall terminate -// as of the date such litigation is filed. -// -// 4. Redistribution. You may reproduce and distribute copies of the -// Work or Derivative Works thereof in any medium, with or without -// modifications, and in Source or Object form, provided that You -// meet the following conditions: -// -// (a) You must give any other recipients of the Work or -// Derivative Works a copy of this License; and -// -// (b) You must cause any modified files to carry prominent notices -// stating that You changed the files; and -// -// (c) You must retain, in the Source form of any Derivative Works -// that You distribute, all copyright, patent, trademark, and -// attribution notices from the Source form of the Work, -// excluding those notices that do not pertain to any part of -// the Derivative Works; and -// -// (d) If the Work includes a "NOTICE" text file as part of its -// distribution, then any Derivative Works that You distribute must -// include a readable copy of the attribution notices contained -// within such NOTICE file, excluding those notices that do not -// pertain to any part of the Derivative Works, in at least one -// of the following places: within a NOTICE text file distributed -// as part of the Derivative Works; within the Source form or -// documentation, if provided along with the Derivative Works; or, -// within a display generated by the Derivative Works, if and -// wherever such third-party notices normally appear. The contents -// of the NOTICE file are for informational purposes only and -// do not modify the License. You may add Your own attribution -// notices within Derivative Works that You distribute, alongside -// or as an addendum to the NOTICE text from the Work, provided -// that such additional attribution notices cannot be construed -// as modifying the License. -// -// You may add Your own copyright statement to Your modifications and -// may provide additional or different license terms and conditions -// for use, reproduction, or distribution of Your modifications, or -// for any such Derivative Works as a whole, provided Your use, -// reproduction, and distribution of the Work otherwise complies with -// the conditions stated in this License. -// -// 5. Submission of Contributions. Unless You explicitly state otherwise, -// any Contribution intentionally submitted for inclusion in the Work -// by You to the Licensor shall be under the terms and conditions of -// this License, without any additional terms or conditions. -// Notwithstanding the above, nothing herein shall supersede or modify -// the terms of any separate license agreement you may have executed -// with Licensor regarding such Contributions. -// -// 6. Trademarks. This License does not grant permission to use the trade -// names, trademarks, service marks, or product names of the Licensor, -// except as required for reasonable and customary use in describing the -// origin of the Work and reproducing the content of the NOTICE file. -// -// 7. Disclaimer of Warranty. Unless required by applicable law or -// agreed to in writing, Licensor provides the Work (and each -// Contributor provides its Contributions) on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied, including, without limitation, any warranties or conditions -// of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -// PARTICULAR PURPOSE. You are solely responsible for determining the -// appropriateness of using or redistributing the Work and assume any -// risks associated with Your exercise of permissions under this License. -// -// 8. Limitation of Liability. In no event and under no legal theory, -// whether in tort (including negligence), contract, or otherwise, -// unless required by applicable law (such as deliberate and grossly -// negligent acts) or agreed to in writing, shall any Contributor be -// liable to You for damages, including any direct, indirect, special, -// incidental, or consequential damages of any character arising as a -// result of this License or out of the use or inability to use the -// Work (including but not limited to damages for loss of goodwill, -// work stoppage, computer failure or malfunction, or any and all -// other commercial damages or losses), even if such Contributor -// has been advised of the possibility of such damages. -// -// 9. Accepting Warranty or Additional Liability. While redistributing -// the Work or Derivative Works thereof, You may choose to offer, -// and charge a fee for, acceptance of support, warranty, indemnity, -// or other liability obligations and/or rights consistent with this -// License. However, in accepting such obligations, You may act only -// on Your own behalf and on Your sole responsibility, not on behalf -// of any other Contributor, and only if You agree to indemnify, -// defend, and hold each Contributor harmless for any liability -// incurred by, or claims asserted against, such Contributor by reason -// of your accepting any such warranty or additional liability. -// -// END OF TERMS AND CONDITIONS -// -// APPENDIX: How to apply the Apache License to your work. -// -// To apply the Apache License to your work, attach the following -// boilerplate notice, with the fields enclosed by brackets "[]" -// replaced with your own identifying information. (Don't include -// the brackets!) The text should be enclosed in the appropriate -// comment syntax for the file format. We also recommend that a -// file or class name and description of purpose be included on the -// same "printed page" as the copyright notice for easier -// identification within third-party archives. -// -// Copyright [yyyy] [name of copyright owner] -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// --- -// -// Boost Software License - Version 1.0 - August 17th, 2003 -// -// Permission is hereby granted, free of charge, to any person or organization -// obtaining a copy of the software and accompanying documentation covered by -// this license (the "Software") to use, reproduce, display, distribute, -// execute, and transmit the Software, and to prepare derivative works of the -// Software, and to permit third-parties to whom the Software is furnished to -// do so, all subject to the following: -// -// The copyright notices in the Software and this entire statement, including -// the above license grant, this restriction and the following disclaimer, -// must be included in all copies of the Software, in whole or in part, and -// all derivative works of the Software, unless such copies or derivative -// works are solely in the form of machine-executable object code generated by -// a source language processor. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -// -// --- -// Minimal Ryu implementation adapted for Ruby JSON gem by Josef Šimánek -// Optimized for pre-extracted mantissa/exponent from JSON parsing -// This is a stripped-down version containing only what's needed for -// converting decimal mantissa+exponent to IEEE 754 double precision. - -#ifndef RYU_H -#define RYU_H - -#include -#include -#include - -// Detect __builtin_clzll availability (for floor_log2) -// Note: MSVC doesn't have __builtin_clzll, so we provide a fallback -#ifdef __clang__ - #if __has_builtin(__builtin_clzll) - #define RYU_HAVE_BUILTIN_CLZLL 1 - #else - #define RYU_HAVE_BUILTIN_CLZLL 0 - #endif -#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) - #define RYU_HAVE_BUILTIN_CLZLL 1 -#else - #define RYU_HAVE_BUILTIN_CLZLL 0 -#endif - -// Count leading zeros (for floor_log2) -static inline uint32_t ryu_leading_zeros64(uint64_t input) -{ -#if RYU_HAVE_BUILTIN_CLZLL - return __builtin_clzll(input); -#else - // Fallback: binary search for the highest set bit - // This works on MSVC and other compilers without __builtin_clzll - if (input == 0) return 64; - uint32_t n = 0; - if (input <= 0x00000000FFFFFFFFULL) { n += 32; input <<= 32; } - if (input <= 0x0000FFFFFFFFFFFFULL) { n += 16; input <<= 16; } - if (input <= 0x00FFFFFFFFFFFFFFULL) { n += 8; input <<= 8; } - if (input <= 0x0FFFFFFFFFFFFFFFULL) { n += 4; input <<= 4; } - if (input <= 0x3FFFFFFFFFFFFFFFULL) { n += 2; input <<= 2; } - if (input <= 0x7FFFFFFFFFFFFFFFULL) { n += 1; } - return n; -#endif -} - -// These tables are generated by PrintDoubleLookupTable. -#define DOUBLE_POW5_INV_BITCOUNT 125 -#define DOUBLE_POW5_BITCOUNT 125 - -#define DOUBLE_POW5_INV_TABLE_SIZE 342 -#define DOUBLE_POW5_TABLE_SIZE 326 - -static const uint64_t DOUBLE_POW5_INV_SPLIT[DOUBLE_POW5_INV_TABLE_SIZE][2] = { - { 1u, 2305843009213693952u }, { 11068046444225730970u, 1844674407370955161u }, - { 5165088340638674453u, 1475739525896764129u }, { 7821419487252849886u, 1180591620717411303u }, - { 8824922364862649494u, 1888946593147858085u }, { 7059937891890119595u, 1511157274518286468u }, - { 13026647942995916322u, 1208925819614629174u }, { 9774590264567735146u, 1934281311383406679u }, - { 11509021026396098440u, 1547425049106725343u }, { 16585914450600699399u, 1237940039285380274u }, - { 15469416676735388068u, 1980704062856608439u }, { 16064882156130220778u, 1584563250285286751u }, - { 9162556910162266299u, 1267650600228229401u }, { 7281393426775805432u, 2028240960365167042u }, - { 16893161185646375315u, 1622592768292133633u }, { 2446482504291369283u, 1298074214633706907u }, - { 7603720821608101175u, 2076918743413931051u }, { 2393627842544570617u, 1661534994731144841u }, - { 16672297533003297786u, 1329227995784915872u }, { 11918280793837635165u, 2126764793255865396u }, - { 5845275820328197809u, 1701411834604692317u }, { 15744267100488289217u, 1361129467683753853u }, - { 3054734472329800808u, 2177807148294006166u }, { 17201182836831481939u, 1742245718635204932u }, - { 6382248639981364905u, 1393796574908163946u }, { 2832900194486363201u, 2230074519853062314u }, - { 5955668970331000884u, 1784059615882449851u }, { 1075186361522890384u, 1427247692705959881u }, - { 12788344622662355584u, 2283596308329535809u }, { 13920024512871794791u, 1826877046663628647u }, - { 3757321980813615186u, 1461501637330902918u }, { 10384555214134712795u, 1169201309864722334u }, - { 5547241898389809503u, 1870722095783555735u }, { 4437793518711847602u, 1496577676626844588u }, - { 10928932444453298728u, 1197262141301475670u }, { 17486291911125277965u, 1915619426082361072u }, - { 6610335899416401726u, 1532495540865888858u }, { 12666966349016942027u, 1225996432692711086u }, - { 12888448528943286597u, 1961594292308337738u }, { 17689456452638449924u, 1569275433846670190u }, - { 14151565162110759939u, 1255420347077336152u }, { 7885109000409574610u, 2008672555323737844u }, - { 9997436015069570011u, 1606938044258990275u }, { 7997948812055656009u, 1285550435407192220u }, - { 12796718099289049614u, 2056880696651507552u }, { 2858676849947419045u, 1645504557321206042u }, - { 13354987924183666206u, 1316403645856964833u }, { 17678631863951955605u, 2106245833371143733u }, - { 3074859046935833515u, 1684996666696914987u }, { 13527933681774397782u, 1347997333357531989u }, - { 10576647446613305481u, 2156795733372051183u }, { 15840015586774465031u, 1725436586697640946u }, - { 8982663654677661702u, 1380349269358112757u }, { 18061610662226169046u, 2208558830972980411u }, - { 10759939715039024913u, 1766847064778384329u }, { 12297300586773130254u, 1413477651822707463u }, - { 15986332124095098083u, 2261564242916331941u }, { 9099716884534168143u, 1809251394333065553u }, - { 14658471137111155161u, 1447401115466452442u }, { 4348079280205103483u, 1157920892373161954u }, - { 14335624477811986218u, 1852673427797059126u }, { 7779150767507678651u, 1482138742237647301u }, - { 2533971799264232598u, 1185710993790117841u }, { 15122401323048503126u, 1897137590064188545u }, - { 12097921058438802501u, 1517710072051350836u }, { 5988988032009131678u, 1214168057641080669u }, - { 16961078480698431330u, 1942668892225729070u }, { 13568862784558745064u, 1554135113780583256u }, - { 7165741412905085728u, 1243308091024466605u }, { 11465186260648137165u, 1989292945639146568u }, - { 16550846638002330379u, 1591434356511317254u }, { 16930026125143774626u, 1273147485209053803u }, - { 4951948911778577463u, 2037035976334486086u }, { 272210314680951647u, 1629628781067588869u }, - { 3907117066486671641u, 1303703024854071095u }, { 6251387306378674625u, 2085924839766513752u }, - { 16069156289328670670u, 1668739871813211001u }, { 9165976216721026213u, 1334991897450568801u }, - { 7286864317269821294u, 2135987035920910082u }, { 16897537898041588005u, 1708789628736728065u }, - { 13518030318433270404u, 1367031702989382452u }, { 6871453250525591353u, 2187250724783011924u }, - { 9186511415162383406u, 1749800579826409539u }, { 11038557946871817048u, 1399840463861127631u }, - { 10282995085511086630u, 2239744742177804210u }, { 8226396068408869304u, 1791795793742243368u }, - { 13959814484210916090u, 1433436634993794694u }, { 11267656730511734774u, 2293498615990071511u }, - { 5324776569667477496u, 1834798892792057209u }, { 7949170070475892320u, 1467839114233645767u }, - { 17427382500606444826u, 1174271291386916613u }, { 5747719112518849781u, 1878834066219066582u }, - { 15666221734240810795u, 1503067252975253265u }, { 12532977387392648636u, 1202453802380202612u }, - { 5295368560860596524u, 1923926083808324180u }, { 4236294848688477220u, 1539140867046659344u }, - { 7078384693692692099u, 1231312693637327475u }, { 11325415509908307358u, 1970100309819723960u }, - { 9060332407926645887u, 1576080247855779168u }, { 14626963555825137356u, 1260864198284623334u }, - { 12335095245094488799u, 2017382717255397335u }, { 9868076196075591040u, 1613906173804317868u }, - { 15273158586344293478u, 1291124939043454294u }, { 13369007293925138595u, 2065799902469526871u }, - { 7005857020398200553u, 1652639921975621497u }, { 16672732060544291412u, 1322111937580497197u }, - { 11918976037903224966u, 2115379100128795516u }, { 5845832015580669650u, 1692303280103036413u }, - { 12055363241948356366u, 1353842624082429130u }, { 841837113407818570u, 2166148198531886609u }, - { 4362818505468165179u, 1732918558825509287u }, { 14558301248600263113u, 1386334847060407429u }, - { 12225235553534690011u, 2218135755296651887u }, { 2401490813343931363u, 1774508604237321510u }, - { 1921192650675145090u, 1419606883389857208u }, { 17831303500047873437u, 2271371013423771532u }, - { 6886345170554478103u, 1817096810739017226u }, { 1819727321701672159u, 1453677448591213781u }, - { 16213177116328979020u, 1162941958872971024u }, { 14873036941900635463u, 1860707134196753639u }, - { 15587778368262418694u, 1488565707357402911u }, { 8780873879868024632u, 1190852565885922329u }, - { 2981351763563108441u, 1905364105417475727u }, { 13453127855076217722u, 1524291284333980581u }, - { 7073153469319063855u, 1219433027467184465u }, { 11317045550910502167u, 1951092843947495144u }, - { 12742985255470312057u, 1560874275157996115u }, { 10194388204376249646u, 1248699420126396892u }, - { 1553625868034358140u, 1997919072202235028u }, { 8621598323911307159u, 1598335257761788022u }, - { 17965325103354776697u, 1278668206209430417u }, { 13987124906400001422u, 2045869129935088668u }, - { 121653480894270168u, 1636695303948070935u }, { 97322784715416134u, 1309356243158456748u }, - { 14913111714512307107u, 2094969989053530796u }, { 8241140556867935363u, 1675975991242824637u }, - { 17660958889720079260u, 1340780792994259709u }, { 17189487779326395846u, 2145249268790815535u }, - { 13751590223461116677u, 1716199415032652428u }, { 18379969808252713988u, 1372959532026121942u }, - { 14650556434236701088u, 2196735251241795108u }, { 652398703163629901u, 1757388200993436087u }, - { 11589965406756634890u, 1405910560794748869u }, { 7475898206584884855u, 2249456897271598191u }, - { 2291369750525997561u, 1799565517817278553u }, { 9211793429904618695u, 1439652414253822842u }, - { 18428218302589300235u, 2303443862806116547u }, { 7363877012587619542u, 1842755090244893238u }, - { 13269799239553916280u, 1474204072195914590u }, { 10615839391643133024u, 1179363257756731672u }, - { 2227947767661371545u, 1886981212410770676u }, { 16539753473096738529u, 1509584969928616540u }, - { 13231802778477390823u, 1207667975942893232u }, { 6413489186596184024u, 1932268761508629172u }, - { 16198837793502678189u, 1545815009206903337u }, { 5580372605318321905u, 1236652007365522670u }, - { 8928596168509315048u, 1978643211784836272u }, { 18210923379033183008u, 1582914569427869017u }, - { 7190041073742725760u, 1266331655542295214u }, { 436019273762630246u, 2026130648867672343u }, - { 7727513048493924843u, 1620904519094137874u }, { 9871359253537050198u, 1296723615275310299u }, - { 4726128361433549347u, 2074757784440496479u }, { 7470251503888749801u, 1659806227552397183u }, - { 13354898832594820487u, 1327844982041917746u }, { 13989140502667892133u, 2124551971267068394u }, - { 14880661216876224029u, 1699641577013654715u }, { 11904528973500979224u, 1359713261610923772u }, - { 4289851098633925465u, 2175541218577478036u }, { 18189276137874781665u, 1740432974861982428u }, - { 3483374466074094362u, 1392346379889585943u }, { 1884050330976640656u, 2227754207823337509u }, - { 5196589079523222848u, 1782203366258670007u }, { 15225317707844309248u, 1425762693006936005u }, - { 5913764258841343181u, 2281220308811097609u }, { 8420360221814984868u, 1824976247048878087u }, - { 17804334621677718864u, 1459980997639102469u }, { 17932816512084085415u, 1167984798111281975u }, - { 10245762345624985047u, 1868775676978051161u }, { 4507261061758077715u, 1495020541582440929u }, - { 7295157664148372495u, 1196016433265952743u }, { 7982903447895485668u, 1913626293225524389u }, - { 10075671573058298858u, 1530901034580419511u }, { 4371188443704728763u, 1224720827664335609u }, - { 14372599139411386667u, 1959553324262936974u }, { 15187428126271019657u, 1567642659410349579u }, - { 15839291315758726049u, 1254114127528279663u }, { 3206773216762499739u, 2006582604045247462u }, - { 13633465017635730761u, 1605266083236197969u }, { 14596120828850494932u, 1284212866588958375u }, - { 4907049252451240275u, 2054740586542333401u }, { 236290587219081897u, 1643792469233866721u }, - { 14946427728742906810u, 1315033975387093376u }, { 16535586736504830250u, 2104054360619349402u }, - { 5849771759720043554u, 1683243488495479522u }, { 15747863852001765813u, 1346594790796383617u }, - { 10439186904235184007u, 2154551665274213788u }, { 15730047152871967852u, 1723641332219371030u }, - { 12584037722297574282u, 1378913065775496824u }, { 9066413911450387881u, 2206260905240794919u }, - { 10942479943902220628u, 1765008724192635935u }, { 8753983955121776503u, 1412006979354108748u }, - { 10317025513452932081u, 2259211166966573997u }, { 874922781278525018u, 1807368933573259198u }, - { 8078635854506640661u, 1445895146858607358u }, { 13841606313089133175u, 1156716117486885886u }, - { 14767872471458792434u, 1850745787979017418u }, { 746251532941302978u, 1480596630383213935u }, - { 597001226353042382u, 1184477304306571148u }, { 15712597221132509104u, 1895163686890513836u }, - { 8880728962164096960u, 1516130949512411069u }, { 10793931984473187891u, 1212904759609928855u }, - { 17270291175157100626u, 1940647615375886168u }, { 2748186495899949531u, 1552518092300708935u }, - { 2198549196719959625u, 1242014473840567148u }, { 18275073973719576693u, 1987223158144907436u }, - { 10930710364233751031u, 1589778526515925949u }, { 12433917106128911148u, 1271822821212740759u }, - { 8826220925580526867u, 2034916513940385215u }, { 7060976740464421494u, 1627933211152308172u }, - { 16716827836597268165u, 1302346568921846537u }, { 11989529279587987770u, 2083754510274954460u }, - { 9591623423670390216u, 1667003608219963568u }, { 15051996368420132820u, 1333602886575970854u }, - { 13015147745246481542u, 2133764618521553367u }, { 3033420566713364587u, 1707011694817242694u }, - { 6116085268112601993u, 1365609355853794155u }, { 9785736428980163188u, 2184974969366070648u }, - { 15207286772667951197u, 1747979975492856518u }, { 1097782973908629988u, 1398383980394285215u }, - { 1756452758253807981u, 2237414368630856344u }, { 5094511021344956708u, 1789931494904685075u }, - { 4075608817075965366u, 1431945195923748060u }, { 6520974107321544586u, 2291112313477996896u }, - { 1527430471115325346u, 1832889850782397517u }, { 12289990821117991246u, 1466311880625918013u }, - { 17210690286378213644u, 1173049504500734410u }, { 9090360384495590213u, 1876879207201175057u }, - { 18340334751822203140u, 1501503365760940045u }, { 14672267801457762512u, 1201202692608752036u }, - { 16096930852848599373u, 1921924308174003258u }, { 1809498238053148529u, 1537539446539202607u }, - { 12515645034668249793u, 1230031557231362085u }, { 1578287981759648052u, 1968050491570179337u }, - { 12330676829633449412u, 1574440393256143469u }, { 13553890278448669853u, 1259552314604914775u }, - { 3239480371808320148u, 2015283703367863641u }, { 17348979556414297411u, 1612226962694290912u }, - { 6500486015647617283u, 1289781570155432730u }, { 10400777625036187652u, 2063650512248692368u }, - { 15699319729512770768u, 1650920409798953894u }, { 16248804598352126938u, 1320736327839163115u }, - { 7551343283653851484u, 2113178124542660985u }, { 6041074626923081187u, 1690542499634128788u }, - { 12211557331022285596u, 1352433999707303030u }, { 1091747655926105338u, 2163894399531684849u }, - { 4562746939482794594u, 1731115519625347879u }, { 7339546366328145998u, 1384892415700278303u }, - { 8053925371383123274u, 2215827865120445285u }, { 6443140297106498619u, 1772662292096356228u }, - { 12533209867169019542u, 1418129833677084982u }, { 5295740528502789974u, 2269007733883335972u }, - { 15304638867027962949u, 1815206187106668777u }, { 4865013464138549713u, 1452164949685335022u }, - { 14960057215536570740u, 1161731959748268017u }, { 9178696285890871890u, 1858771135597228828u }, - { 14721654658196518159u, 1487016908477783062u }, { 4398626097073393881u, 1189613526782226450u }, - { 7037801755317430209u, 1903381642851562320u }, { 5630241404253944167u, 1522705314281249856u }, - { 814844308661245011u, 1218164251424999885u }, { 1303750893857992017u, 1949062802279999816u }, - { 15800395974054034906u, 1559250241823999852u }, { 5261619149759407279u, 1247400193459199882u }, - { 12107939454356961969u, 1995840309534719811u }, { 5997002748743659252u, 1596672247627775849u }, - { 8486951013736837725u, 1277337798102220679u }, { 2511075177753209390u, 2043740476963553087u }, - { 13076906586428298482u, 1634992381570842469u }, { 14150874083884549109u, 1307993905256673975u }, - { 4194654460505726958u, 2092790248410678361u }, { 18113118827372222859u, 1674232198728542688u }, - { 3422448617672047318u, 1339385758982834151u }, { 16543964232501006678u, 2143017214372534641u }, - { 9545822571258895019u, 1714413771498027713u }, { 15015355686490936662u, 1371531017198422170u }, - { 5577825024675947042u, 2194449627517475473u }, { 11840957649224578280u, 1755559702013980378u }, - { 16851463748863483271u, 1404447761611184302u }, { 12204946739213931940u, 2247116418577894884u }, - { 13453306206113055875u, 1797693134862315907u }, { 3383947335406624054u, 1438154507889852726u }, - { 16482362180876329456u, 2301047212623764361u }, { 9496540929959153242u, 1840837770099011489u }, - { 11286581558709232917u, 1472670216079209191u }, { 5339916432225476010u, 1178136172863367353u }, - { 4854517476818851293u, 1885017876581387765u }, { 3883613981455081034u, 1508014301265110212u }, - { 14174937629389795797u, 1206411441012088169u }, { 11611853762797942306u, 1930258305619341071u }, - { 5600134195496443521u, 1544206644495472857u }, { 15548153800622885787u, 1235365315596378285u }, - { 6430302007287065643u, 1976584504954205257u }, { 16212288050055383484u, 1581267603963364205u }, - { 12969830440044306787u, 1265014083170691364u }, { 9683682259845159889u, 2024022533073106183u }, - { 15125643437359948558u, 1619218026458484946u }, { 8411165935146048523u, 1295374421166787957u }, - { 17147214310975587960u, 2072599073866860731u }, { 10028422634038560045u, 1658079259093488585u }, - { 8022738107230848036u, 1326463407274790868u }, { 9147032156827446534u, 2122341451639665389u }, - { 11006974540203867551u, 1697873161311732311u }, { 5116230817421183718u, 1358298529049385849u }, - { 15564666937357714594u, 2173277646479017358u }, { 1383687105660440706u, 1738622117183213887u }, - { 12174996128754083534u, 1390897693746571109u }, { 8411947361780802685u, 2225436309994513775u }, - { 6729557889424642148u, 1780349047995611020u }, { 5383646311539713719u, 1424279238396488816u }, - { 1235136468979721303u, 2278846781434382106u }, { 15745504434151418335u, 1823077425147505684u }, - { 16285752362063044992u, 1458461940118004547u }, { 5649904260166615347u, 1166769552094403638u }, - { 5350498001524674232u, 1866831283351045821u }, { 591049586477829062u, 1493465026680836657u }, - { 11540886113407994219u, 1194772021344669325u }, { 18673707743239135u, 1911635234151470921u }, - { 14772334225162232601u, 1529308187321176736u }, { 8128518565387875758u, 1223446549856941389u }, - { 1937583260394870242u, 1957514479771106223u }, { 8928764237799716840u, 1566011583816884978u }, - { 14521709019723594119u, 1252809267053507982u }, { 8477339172590109297u, 2004494827285612772u }, - { 17849917782297818407u, 1603595861828490217u }, { 6901236596354434079u, 1282876689462792174u }, - { 18420676183650915173u, 2052602703140467478u }, { 3668494502695001169u, 1642082162512373983u }, - { 10313493231639821582u, 1313665730009899186u }, { 9122891541139893884u, 2101865168015838698u }, - { 14677010862395735754u, 1681492134412670958u }, { 673562245690857633u, 1345193707530136767u } -}; - -static const uint64_t DOUBLE_POW5_SPLIT[DOUBLE_POW5_TABLE_SIZE][2] = { - { 0u, 1152921504606846976u }, { 0u, 1441151880758558720u }, - { 0u, 1801439850948198400u }, { 0u, 2251799813685248000u }, - { 0u, 1407374883553280000u }, { 0u, 1759218604441600000u }, - { 0u, 2199023255552000000u }, { 0u, 1374389534720000000u }, - { 0u, 1717986918400000000u }, { 0u, 2147483648000000000u }, - { 0u, 1342177280000000000u }, { 0u, 1677721600000000000u }, - { 0u, 2097152000000000000u }, { 0u, 1310720000000000000u }, - { 0u, 1638400000000000000u }, { 0u, 2048000000000000000u }, - { 0u, 1280000000000000000u }, { 0u, 1600000000000000000u }, - { 0u, 2000000000000000000u }, { 0u, 1250000000000000000u }, - { 0u, 1562500000000000000u }, { 0u, 1953125000000000000u }, - { 0u, 1220703125000000000u }, { 0u, 1525878906250000000u }, - { 0u, 1907348632812500000u }, { 0u, 1192092895507812500u }, - { 0u, 1490116119384765625u }, { 4611686018427387904u, 1862645149230957031u }, - { 9799832789158199296u, 1164153218269348144u }, { 12249790986447749120u, 1455191522836685180u }, - { 15312238733059686400u, 1818989403545856475u }, { 14528612397897220096u, 2273736754432320594u }, - { 13692068767113150464u, 1421085471520200371u }, { 12503399940464050176u, 1776356839400250464u }, - { 15629249925580062720u, 2220446049250313080u }, { 9768281203487539200u, 1387778780781445675u }, - { 7598665485932036096u, 1734723475976807094u }, { 274959820560269312u, 2168404344971008868u }, - { 9395221924704944128u, 1355252715606880542u }, { 2520655369026404352u, 1694065894508600678u }, - { 12374191248137781248u, 2117582368135750847u }, { 14651398557727195136u, 1323488980084844279u }, - { 13702562178731606016u, 1654361225106055349u }, { 3293144668132343808u, 2067951531382569187u }, - { 18199116482078572544u, 1292469707114105741u }, { 8913837547316051968u, 1615587133892632177u }, - { 15753982952572452864u, 2019483917365790221u }, { 12152082354571476992u, 1262177448353618888u }, - { 15190102943214346240u, 1577721810442023610u }, { 9764256642163156992u, 1972152263052529513u }, - { 17631875447420442880u, 1232595164407830945u }, { 8204786253993389888u, 1540743955509788682u }, - { 1032610780636961552u, 1925929944387235853u }, { 2951224747111794922u, 1203706215242022408u }, - { 3689030933889743652u, 1504632769052528010u }, { 13834660704216955373u, 1880790961315660012u }, - { 17870034976990372916u, 1175494350822287507u }, { 17725857702810578241u, 1469367938527859384u }, - { 3710578054803671186u, 1836709923159824231u }, { 26536550077201078u, 2295887403949780289u }, - { 11545800389866720434u, 1434929627468612680u }, { 14432250487333400542u, 1793662034335765850u }, - { 8816941072311974870u, 2242077542919707313u }, { 17039803216263454053u, 1401298464324817070u }, - { 12076381983474541759u, 1751623080406021338u }, { 5872105442488401391u, 2189528850507526673u }, - { 15199280947623720629u, 1368455531567204170u }, { 9775729147674874978u, 1710569414459005213u }, - { 16831347453020981627u, 2138211768073756516u }, { 1296220121283337709u, 1336382355046097823u }, - { 15455333206886335848u, 1670477943807622278u }, { 10095794471753144002u, 2088097429759527848u }, - { 6309871544845715001u, 1305060893599704905u }, { 12499025449484531656u, 1631326116999631131u }, - { 11012095793428276666u, 2039157646249538914u }, { 11494245889320060820u, 1274473528905961821u }, - { 532749306367912313u, 1593091911132452277u }, { 5277622651387278295u, 1991364888915565346u }, - { 7910200175544436838u, 1244603055572228341u }, { 14499436237857933952u, 1555753819465285426u }, - { 8900923260467641632u, 1944692274331606783u }, { 12480606065433357876u, 1215432671457254239u }, - { 10989071563364309441u, 1519290839321567799u }, { 9124653435777998898u, 1899113549151959749u }, - { 8008751406574943263u, 1186945968219974843u }, { 5399253239791291175u, 1483682460274968554u }, - { 15972438586593889776u, 1854603075343710692u }, { 759402079766405302u, 1159126922089819183u }, - { 14784310654990170340u, 1448908652612273978u }, { 9257016281882937117u, 1811135815765342473u }, - { 16182956370781059300u, 2263919769706678091u }, { 7808504722524468110u, 1414949856066673807u }, - { 5148944884728197234u, 1768687320083342259u }, { 1824495087482858639u, 2210859150104177824u }, - { 1140309429676786649u, 1381786968815111140u }, { 1425386787095983311u, 1727233711018888925u }, - { 6393419502297367043u, 2159042138773611156u }, { 13219259225790630210u, 1349401336733506972u }, - { 16524074032238287762u, 1686751670916883715u }, { 16043406521870471799u, 2108439588646104644u }, - { 803757039314269066u, 1317774742903815403u }, { 14839754354425000045u, 1647218428629769253u }, - { 4714634887749086344u, 2059023035787211567u }, { 9864175832484260821u, 1286889397367007229u }, - { 16941905809032713930u, 1608611746708759036u }, { 2730638187581340797u, 2010764683385948796u }, - { 10930020904093113806u, 1256727927116217997u }, { 18274212148543780162u, 1570909908895272496u }, - { 4396021111970173586u, 1963637386119090621u }, { 5053356204195052443u, 1227273366324431638u }, - { 15540067292098591362u, 1534091707905539547u }, { 14813398096695851299u, 1917614634881924434u }, - { 13870059828862294966u, 1198509146801202771u }, { 12725888767650480803u, 1498136433501503464u }, - { 15907360959563101004u, 1872670541876879330u }, { 14553786618154326031u, 1170419088673049581u }, - { 4357175217410743827u, 1463023860841311977u }, { 10058155040190817688u, 1828779826051639971u }, - { 7961007781811134206u, 2285974782564549964u }, { 14199001900486734687u, 1428734239102843727u }, - { 13137066357181030455u, 1785917798878554659u }, { 11809646928048900164u, 2232397248598193324u }, - { 16604401366885338411u, 1395248280373870827u }, { 16143815690179285109u, 1744060350467338534u }, - { 10956397575869330579u, 2180075438084173168u }, { 6847748484918331612u, 1362547148802608230u }, - { 17783057643002690323u, 1703183936003260287u }, { 17617136035325974999u, 2128979920004075359u }, - { 17928239049719816230u, 1330612450002547099u }, { 17798612793722382384u, 1663265562503183874u }, - { 13024893955298202172u, 2079081953128979843u }, { 5834715712847682405u, 1299426220705612402u }, - { 16516766677914378815u, 1624282775882015502u }, { 11422586310538197711u, 2030353469852519378u }, - { 11750802462513761473u, 1268970918657824611u }, { 10076817059714813937u, 1586213648322280764u }, - { 12596021324643517422u, 1982767060402850955u }, { 5566670318688504437u, 1239229412751781847u }, - { 2346651879933242642u, 1549036765939727309u }, { 7545000868343941206u, 1936295957424659136u }, - { 4715625542714963254u, 1210184973390411960u }, { 5894531928393704067u, 1512731216738014950u }, - { 16591536947346905892u, 1890914020922518687u }, { 17287239619732898039u, 1181821263076574179u }, - { 16997363506238734644u, 1477276578845717724u }, { 2799960309088866689u, 1846595723557147156u }, - { 10973347230035317489u, 1154122327223216972u }, { 13716684037544146861u, 1442652909029021215u }, - { 12534169028502795672u, 1803316136286276519u }, { 11056025267201106687u, 2254145170357845649u }, - { 18439230838069161439u, 1408840731473653530u }, { 13825666510731675991u, 1761050914342066913u }, - { 3447025083132431277u, 2201313642927583642u }, { 6766076695385157452u, 1375821026829739776u }, - { 8457595869231446815u, 1719776283537174720u }, { 10571994836539308519u, 2149720354421468400u }, - { 6607496772837067824u, 1343575221513417750u }, { 17482743002901110588u, 1679469026891772187u }, - { 17241742735199000331u, 2099336283614715234u }, { 15387775227926763111u, 1312085177259197021u }, - { 5399660979626290177u, 1640106471573996277u }, { 11361262242960250625u, 2050133089467495346u }, - { 11712474920277544544u, 1281333180917184591u }, { 10028907631919542777u, 1601666476146480739u }, - { 7924448521472040567u, 2002083095183100924u }, { 14176152362774801162u, 1251301934489438077u }, - { 3885132398186337741u, 1564127418111797597u }, { 9468101516160310080u, 1955159272639746996u }, - { 15140935484454969608u, 1221974545399841872u }, { 479425281859160394u, 1527468181749802341u }, - { 5210967620751338397u, 1909335227187252926u }, { 17091912818251750210u, 1193334516992033078u }, - { 12141518985959911954u, 1491668146240041348u }, { 15176898732449889943u, 1864585182800051685u }, - { 11791404716994875166u, 1165365739250032303u }, { 10127569877816206054u, 1456707174062540379u }, - { 8047776328842869663u, 1820883967578175474u }, { 836348374198811271u, 2276104959472719343u }, - { 7440246761515338900u, 1422565599670449589u }, { 13911994470321561530u, 1778206999588061986u }, - { 8166621051047176104u, 2222758749485077483u }, { 2798295147690791113u, 1389224218428173427u }, - { 17332926989895652603u, 1736530273035216783u }, { 17054472718942177850u, 2170662841294020979u }, - { 8353202440125167204u, 1356664275808763112u }, { 10441503050156459005u, 1695830344760953890u }, - { 3828506775840797949u, 2119787930951192363u }, { 86973725686804766u, 1324867456844495227u }, - { 13943775212390669669u, 1656084321055619033u }, { 3594660960206173375u, 2070105401319523792u }, - { 2246663100128858359u, 1293815875824702370u }, { 12031700912015848757u, 1617269844780877962u }, - { 5816254103165035138u, 2021587305976097453u }, { 5941001823691840913u, 1263492066235060908u }, - { 7426252279614801142u, 1579365082793826135u }, { 4671129331091113523u, 1974206353492282669u }, - { 5225298841145639904u, 1233878970932676668u }, { 6531623551432049880u, 1542348713665845835u }, - { 3552843420862674446u, 1927935892082307294u }, { 16055585193321335241u, 1204959932551442058u }, - { 10846109454796893243u, 1506199915689302573u }, { 18169322836923504458u, 1882749894611628216u }, - { 11355826773077190286u, 1176718684132267635u }, { 9583097447919099954u, 1470898355165334544u }, - { 11978871809898874942u, 1838622943956668180u }, { 14973589762373593678u, 2298278679945835225u }, - { 2440964573842414192u, 1436424174966147016u }, { 3051205717303017741u, 1795530218707683770u }, - { 13037379183483547984u, 2244412773384604712u }, { 8148361989677217490u, 1402757983365377945u }, - { 14797138505523909766u, 1753447479206722431u }, { 13884737113477499304u, 2191809349008403039u }, - { 15595489723564518921u, 1369880843130251899u }, { 14882676136028260747u, 1712351053912814874u }, - { 9379973133180550126u, 2140438817391018593u }, { 17391698254306313589u, 1337774260869386620u }, - { 3292878744173340370u, 1672217826086733276u }, { 4116098430216675462u, 2090272282608416595u }, - { 266718509671728212u, 1306420176630260372u }, { 333398137089660265u, 1633025220787825465u }, - { 5028433689789463235u, 2041281525984781831u }, { 10060300083759496378u, 1275800953740488644u }, - { 12575375104699370472u, 1594751192175610805u }, { 1884160825592049379u, 1993438990219513507u }, - { 17318501580490888525u, 1245899368887195941u }, { 7813068920331446945u, 1557374211108994927u }, - { 5154650131986920777u, 1946717763886243659u }, { 915813323278131534u, 1216698602428902287u }, - { 14979824709379828129u, 1520873253036127858u }, { 9501408849870009354u, 1901091566295159823u }, - { 12855909558809837702u, 1188182228934474889u }, { 2234828893230133415u, 1485227786168093612u }, - { 2793536116537666769u, 1856534732710117015u }, { 8663489100477123587u, 1160334207943823134u }, - { 1605989338741628675u, 1450417759929778918u }, { 11230858710281811652u, 1813022199912223647u }, - { 9426887369424876662u, 2266277749890279559u }, { 12809333633531629769u, 1416423593681424724u }, - { 16011667041914537212u, 1770529492101780905u }, { 6179525747111007803u, 2213161865127226132u }, - { 13085575628799155685u, 1383226165704516332u }, { 16356969535998944606u, 1729032707130645415u }, - { 15834525901571292854u, 2161290883913306769u }, { 2979049660840976177u, 1350806802445816731u }, - { 17558870131333383934u, 1688508503057270913u }, { 8113529608884566205u, 2110635628821588642u }, - { 9682642023980241782u, 1319147268013492901u }, { 16714988548402690132u, 1648934085016866126u }, - { 11670363648648586857u, 2061167606271082658u }, { 11905663298832754689u, 1288229753919426661u }, - { 1047021068258779650u, 1610287192399283327u }, { 15143834390605638274u, 2012858990499104158u }, - { 4853210475701136017u, 1258036869061940099u }, { 1454827076199032118u, 1572546086327425124u }, - { 1818533845248790147u, 1965682607909281405u }, { 3442426662494187794u, 1228551629943300878u }, - { 13526405364972510550u, 1535689537429126097u }, { 3072948650933474476u, 1919611921786407622u }, - { 15755650962115585259u, 1199757451116504763u }, { 15082877684217093670u, 1499696813895630954u }, - { 9630225068416591280u, 1874621017369538693u }, { 8324733676974063502u, 1171638135855961683u }, - { 5794231077790191473u, 1464547669819952104u }, { 7242788847237739342u, 1830684587274940130u }, - { 18276858095901949986u, 2288355734093675162u }, { 16034722328366106645u, 1430222333808546976u }, - { 1596658836748081690u, 1787777917260683721u }, { 6607509564362490017u, 2234722396575854651u }, - { 1823850468512862308u, 1396701497859909157u }, { 6891499104068465790u, 1745876872324886446u }, - { 17837745916940358045u, 2182346090406108057u }, { 4231062170446641922u, 1363966306503817536u }, - { 5288827713058302403u, 1704957883129771920u }, { 6611034641322878003u, 2131197353912214900u }, - { 13355268687681574560u, 1331998346195134312u }, { 16694085859601968200u, 1664997932743917890u }, - { 11644235287647684442u, 2081247415929897363u }, { 4971804045566108824u, 1300779634956185852u }, - { 6214755056957636030u, 1625974543695232315u }, { 3156757802769657134u, 2032468179619040394u }, - { 6584659645158423613u, 1270292612261900246u }, { 17454196593302805324u, 1587865765327375307u }, - { 17206059723201118751u, 1984832206659219134u }, { 6142101308573311315u, 1240520129162011959u }, - { 3065940617289251240u, 1550650161452514949u }, { 8444111790038951954u, 1938312701815643686u }, - { 665883850346957067u, 1211445438634777304u }, { 832354812933696334u, 1514306798293471630u }, - { 10263815553021896226u, 1892883497866839537u }, { 17944099766707154901u, 1183052186166774710u }, - { 13206752671529167818u, 1478815232708468388u }, { 16508440839411459773u, 1848519040885585485u }, - { 12623618533845856310u, 1155324400553490928u }, { 15779523167307320387u, 1444155500691863660u }, - { 1277659885424598868u, 1805194375864829576u }, { 1597074856780748586u, 2256492969831036970u }, - { 5609857803915355770u, 1410308106144398106u }, { 16235694291748970521u, 1762885132680497632u }, - { 1847873790976661535u, 2203606415850622041u }, { 12684136165428883219u, 1377254009906638775u }, - { 11243484188358716120u, 1721567512383298469u }, { 219297180166231438u, 2151959390479123087u }, - { 7054589765244976505u, 1344974619049451929u }, { 13429923224983608535u, 1681218273811814911u }, - { 12175718012802122765u, 2101522842264768639u }, { 14527352785642408584u, 1313451776415480399u }, - { 13547504963625622826u, 1641814720519350499u }, { 12322695186104640628u, 2052268400649188124u }, - { 16925056528170176201u, 1282667750405742577u }, { 7321262604930556539u, 1603334688007178222u }, - { 18374950293017971482u, 2004168360008972777u }, { 4566814905495150320u, 1252605225005607986u }, - { 14931890668723713708u, 1565756531257009982u }, { 9441491299049866327u, 1957195664071262478u }, - { 1289246043478778550u, 1223247290044539049u }, { 6223243572775861092u, 1529059112555673811u }, - { 3167368447542438461u, 1911323890694592264u }, { 1979605279714024038u, 1194577431684120165u }, - { 7086192618069917952u, 1493221789605150206u }, { 18081112809442173248u, 1866527237006437757u }, - { 13606538515115052232u, 1166579523129023598u }, { 7784801107039039482u, 1458224403911279498u }, - { 507629346944023544u, 1822780504889099373u }, { 5246222702107417334u, 2278475631111374216u }, - { 3278889188817135834u, 1424047269444608885u }, { 8710297504448807696u, 1780059086805761106u } -}; - -// IEEE 754 double precision constants -#define DOUBLE_MANTISSA_BITS 52 -#define DOUBLE_EXPONENT_BITS 11 -#define DOUBLE_EXPONENT_BIAS 1023 - -// Helper: floor(log2(value)) using ryu_leading_zeros64 -static inline uint32_t floor_log2(const uint64_t value) { - return 63 - ryu_leading_zeros64(value); -} - -// Helper: log2(5^e) approximation -static inline int32_t log2pow5(const int32_t e) { - return (int32_t) ((((uint32_t) e) * 1217359) >> 19); -} - -// Helper: ceil(log2(5^e)) -static inline int32_t ceil_log2pow5(const int32_t e) { - return log2pow5(e) + 1; -} - -// Helper: max of two int32 -static inline int32_t max32(int32_t a, int32_t b) { - return a < b ? b : a; -} - -// Helper: convert uint64 bits to double -static inline double int64Bits2Double(uint64_t bits) { - double f; - memcpy(&f, &bits, sizeof(double)); - return f; -} - -// Check if value is multiple of 2^p -static inline bool multipleOfPowerOf2(const uint64_t value, const uint32_t p) { - return (value & ((1ull << p) - 1)) == 0; -} - -// Count how many times value is divisible by 5 -// Uses modular inverse to avoid expensive division -static inline uint32_t pow5Factor(uint64_t value) { - const uint64_t m_inv_5 = 14757395258967641293u; // 5 * m_inv_5 = 1 (mod 2^64) - const uint64_t n_div_5 = 3689348814741910323u; // 2^64 / 5 - uint32_t count = 0; - for (;;) { - value *= m_inv_5; - if (value > n_div_5) - break; - ++count; - } - return count; -} - -// Check if value is multiple of 5^p -// Optimized: uses modular inverse instead of division -static inline bool multipleOfPowerOf5(const uint64_t value, const uint32_t p) { - return pow5Factor(value) >= p; -} - -// 128-bit multiplication with shift -// This is the core operation for converting decimal to binary -#if defined(__SIZEOF_INT128__) -// Use native 128-bit integers if available (GCC/Clang) -static inline uint64_t mulShift64(const uint64_t m, const uint64_t* const mul, const int32_t j) { - const unsigned __int128 b0 = ((unsigned __int128) m) * mul[0]; - const unsigned __int128 b2 = ((unsigned __int128) m) * mul[1]; - return (uint64_t) (((b0 >> 64) + b2) >> (j - 64)); -} -#else -// Fallback for systems without 128-bit integers -static inline uint64_t umul128(const uint64_t a, const uint64_t b, uint64_t* const productHi) { - const uint32_t aLo = (uint32_t)a; - const uint32_t aHi = (uint32_t)(a >> 32); - const uint32_t bLo = (uint32_t)b; - const uint32_t bHi = (uint32_t)(b >> 32); - - const uint64_t b00 = (uint64_t)aLo * bLo; - const uint64_t b01 = (uint64_t)aLo * bHi; - const uint64_t b10 = (uint64_t)aHi * bLo; - const uint64_t b11 = (uint64_t)aHi * bHi; - - const uint32_t b00Lo = (uint32_t)b00; - const uint32_t b00Hi = (uint32_t)(b00 >> 32); - - const uint64_t mid1 = b10 + b00Hi; - const uint32_t mid1Lo = (uint32_t)(mid1); - const uint32_t mid1Hi = (uint32_t)(mid1 >> 32); - - const uint64_t mid2 = b01 + mid1Lo; - const uint32_t mid2Lo = (uint32_t)(mid2); - const uint32_t mid2Hi = (uint32_t)(mid2 >> 32); - - const uint64_t pHi = b11 + mid1Hi + mid2Hi; - const uint64_t pLo = ((uint64_t)mid2Lo << 32) | b00Lo; - - *productHi = pHi; - return pLo; -} - -static inline uint64_t shiftright128(const uint64_t lo, const uint64_t hi, const uint32_t dist) { - return (hi << (64 - dist)) | (lo >> dist); -} - -static inline uint64_t mulShift64(const uint64_t m, const uint64_t* const mul, const int32_t j) { - uint64_t high1; - const uint64_t low1 = umul128(m, mul[1], &high1); - uint64_t high0; - umul128(m, mul[0], &high0); - const uint64_t sum = high0 + low1; - if (sum < high0) { - ++high1; - } - return shiftright128(sum, high1, j - 64); -} -#endif - -// Main conversion function: decimal mantissa+exponent to IEEE 754 double -// Optimized for JSON parsing with fast paths for edge cases -static inline double ryu_s2d_from_parts(uint64_t m10, int m10digits, int32_t e10, bool signedM) { - // Fast path: handle zero explicitly (e.g., "0.0", "0e0") - if (m10 == 0) { - return int64Bits2Double(((uint64_t) signedM) << 63); - } - - // Fast path: handle overflow/underflow early - if (m10digits + e10 <= -324) { - // Underflow to zero - return int64Bits2Double(((uint64_t) signedM) << 63); - } - - if (m10digits + e10 >= 310) { - // Overflow to infinity - return int64Bits2Double((((uint64_t) signedM) << 63) | 0x7ff0000000000000ULL); - } - - // Convert decimal to binary: m10 * 10^e10 = m2 * 2^e2 - int32_t e2; - uint64_t m2; - bool trailingZeros; - - if (e10 >= 0) { - // Positive exponent: multiply by 5^e10 and adjust binary exponent - e2 = floor_log2(m10) + e10 + log2pow5(e10) - (DOUBLE_MANTISSA_BITS + 1); - int j = e2 - e10 - ceil_log2pow5(e10) + DOUBLE_POW5_BITCOUNT; - m2 = mulShift64(m10, DOUBLE_POW5_SPLIT[e10], j); - trailingZeros = e2 < e10 || (e2 - e10 < 64 && multipleOfPowerOf2(m10, e2 - e10)); - } else { - // Negative exponent: divide by 5^(-e10) - e2 = floor_log2(m10) + e10 - ceil_log2pow5(-e10) - (DOUBLE_MANTISSA_BITS + 1); - int j = e2 - e10 + ceil_log2pow5(-e10) - 1 + DOUBLE_POW5_INV_BITCOUNT; - m2 = mulShift64(m10, DOUBLE_POW5_INV_SPLIT[-e10], j); - trailingZeros = multipleOfPowerOf5(m10, -e10); - } - - // Compute IEEE 754 exponent - uint32_t ieee_e2 = (uint32_t) max32(0, e2 + DOUBLE_EXPONENT_BIAS + floor_log2(m2)); - - if (ieee_e2 > 0x7fe) { - // Overflow to infinity - return int64Bits2Double((((uint64_t) signedM) << 63) | 0x7ff0000000000000ULL); - } - - // Compute shift amount for rounding - int32_t shift = (ieee_e2 == 0 ? 1 : ieee_e2) - e2 - DOUBLE_EXPONENT_BIAS - DOUBLE_MANTISSA_BITS; - - // IEEE 754 round-to-even (banker's rounding) - trailingZeros &= (m2 & ((1ull << (shift - 1)) - 1)) == 0; - uint64_t lastRemovedBit = (m2 >> (shift - 1)) & 1; - bool roundUp = (lastRemovedBit != 0) && (!trailingZeros || (((m2 >> shift) & 1) != 0)); - - uint64_t ieee_m2 = (m2 >> shift) + roundUp; - ieee_m2 &= (1ull << DOUBLE_MANTISSA_BITS) - 1; - - if (ieee_m2 == 0 && roundUp) { - ieee_e2++; - } - - // Pack sign, exponent, and mantissa into IEEE 754 format - // Match original Ryu: group sign+exponent, then shift and add mantissa - uint64_t ieee = (((((uint64_t) signedM) << DOUBLE_EXPONENT_BITS) | (uint64_t)ieee_e2) << DOUBLE_MANTISSA_BITS) | ieee_m2; - return int64Bits2Double(ieee); -} - -#endif // RYU_H diff --git a/test/json/json_ryu_fallback_test.rb b/test/json/json_float_parser_fallback_test.rb similarity index 85% rename from test/json/json_ryu_fallback_test.rb rename to test/json/json_float_parser_fallback_test.rb index a61b3e668..39801d90d 100644 --- a/test/json/json_ryu_fallback_test.rb +++ b/test/json/json_float_parser_fallback_test.rb @@ -5,31 +5,25 @@ rescue LoadError end -class JSONRyuFallbackTest < Test::Unit::TestCase +class JSONFloatParserFallbackTest < Test::Unit::TestCase include JSON - # Test that numbers with more than 17 significant digits fall back to rb_cstr_to_dbl - def test_more_than_17_significant_digits - # These numbers have > 17 significant digits and should use fallback path - # They should still parse correctly, just not via the Ryu optimization + # Test that numbers with more than 18 significant digits fall back to rb_cstr_to_dbl + def test_more_than_18_significant_digits + # These numbers have > 18 significant digits and should use fallback path. + # They should still parse correctly, just not via the Fast Float optimization - test_cases = [ - # input, expected (rounded to double precision) - ["1.23456789012345678901234567890", 1.2345678901234567], - ["123456789012345678.901234567890", 1.2345678901234568e+17], - ["0.123456789012345678901234567890", 0.12345678901234568], - ["9999999999999999999999999999.9", 1.0e+28], - # Edge case: exactly 18 digits - ["123456789012345678", 123456789012345680.0], - # Many fractional digits - ["0.12345678901234567890123456789", 0.12345678901234568], - ] + # input, expected (rounded to double precision) + assert_float_parsing "1.23456789012345678901234567890" + assert_float_parsing "123456789012345678.901234567890" + assert_float_parsing "0.123456789012345678901234567890" + assert_float_parsing "9999999999999999999999999999.9" - test_cases.each do |input, expected| - result = JSON.parse(input) - assert_in_delta(expected, result, 1e-10, - "Failed to parse #{input} correctly (>17 digits, fallback path)") - end + # Edge case: exactly 18 digits + assert_float_parsing "123456789012345678.0" + + # Many fractional digits + assert_float_parsing "0.12345678901234567890123456789" end # Test decimal_class option forces fallback @@ -188,4 +182,10 @@ def test_large_exponent_numbers assert_equal 0.0, JSON.parse("1e-18446744073709551615") assert_equal 0.0, JSON.parse("1e-9223372036854775809") end + + private + + def assert_float_parsing(number) + assert_equal(Float(number), JSON.parse(number), "Failed to parse #{number} correctly") + end end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index a4292871a..c1aa2f615 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -60,7 +60,8 @@ def test_parse_values assert_equal(-23, parse('-23')) assert_equal(23, parse('23')) assert_in_delta(0.23, parse('0.23'), 1e-2) - assert_in_delta(0.0, parse('0e0'), 1e-2) + assert_equal(0.0, parse('0e0')) + assert_equal(-0.0, parse('-0e0')) assert_equal("", parse('""')) assert_equal("foobar", parse('"foobar"')) end From c171e349393f5691785dbe219e4aad63ee8090e0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 15 Jun 2026 16:16:38 +0200 Subject: [PATCH 356/506] fast_float_parser.h: add portable __builtin_clzll Required for Windows support. --- ext/json/ext/vendor/fast_float_parser.h | 34 +++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 6bbb77dae..00c1f1552 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -707,6 +707,36 @@ static inline double ffp_bits2double(uint64_t bits) { return d; } +static inline unsigned int +ffc_nlz_int64(uint64_t x) +{ +#if defined(_MSC_VER) && defined(__AVX2__) + return (unsigned int)__lzcnt64(x); + +#elif defined(__x86_64__) && defined(__LZCNT__) + return (unsigned int)_lzcnt_u64(x); + +#elif defined(_WIN64) && defined(_MSC_VER) /* &&! defined(__AVX2__) */ + unsigned long r; + return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; + +#elif __has_builtin(__builtin_clzll) + return (unsigned int)__builtin_clzll((unsigned long long)x); + +#else + uint64_t y; + unsigned int n = 64; + y = x >> 32; if (y) {n -= 32; x = y;} + y = x >> 16; if (y) {n -= 16; x = y;} + y = x >> 8; if (y) {n -= 8; x = y;} + y = x >> 4; if (y) {n -= 4; x = y;} + y = x >> 2; if (y) {n -= 2; x = y;} + y = x >> 1; if (y) {return n - 2;} + return (unsigned int)(n - x); + +#endif +} + /* q = power of ten, w = mantissa (exact, fits in uint64). neg = sign. */ static inline double ffp_s2d(int64_t q, uint64_t w, bool neg) { if (w == 0) { @@ -716,14 +746,14 @@ static inline double ffp_s2d(int64_t q, uint64_t w, bool neg) { const uint64_t sign = (uint64_t)(neg != 0) << 63; uint64_t mantissa, prod_hi, prod_lo, sp_hi, sp_lo; int32_t power2; - int lz, upperbit, shift, index; + int upperbit, shift, index; if (q < FFP_SMALLEST_POW10) return ffp_bits2double(sign); /* underflow -> 0 */ if (q > FFP_LARGEST_POW10) { return ffp_bits2double(sign | ((uint64_t)FFP_INFINITE_POWER << FFP_MANTISSA_BITS)); } - lz = __builtin_clzll(w); + unsigned int lz = ffc_nlz_int64(w); w <<= lz; /* compute_product_approximation: precision_mask = 0x1FF. */ From 34bc289d6916a81edda46fa3adb12a0d3f4f4ccd Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 12:09:51 +0200 Subject: [PATCH 357/506] fast_float_parser.h: handle `__has_builtin` not being defined --- ext/json/ext/vendor/fast_float_parser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 00c1f1552..66e1b1325 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -720,7 +720,7 @@ ffc_nlz_int64(uint64_t x) unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; -#elif __has_builtin(__builtin_clzll) +#elif defined(__has_builtin) && __has_builtin(__builtin_clzll) return (unsigned int)__builtin_clzll((unsigned long long)x); #else From 8d2a1ff57d9cd742481c235eb3fb79c8adf92bb5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 12:54:01 +0200 Subject: [PATCH 358/506] fast_float_parser.h: handle more environments Inspired by bigdecimal --- ext/json/ext/parser/extconf.rb | 25 +++++++++++++++++++++++++ ext/json/ext/vendor/fast_float_parser.h | 6 ++++++ 2 files changed, 31 insertions(+) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index a9d740c75..bd537c9c5 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -8,6 +8,31 @@ have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby have_func("ruby_xfree_sized", "ruby.h") # RUBY_VERSION >= 4.1 +def have_builtin_func(name, check_expr, opt = "", &b) + checking_for checking_message(name.funcall_style, nil, opt) do + if try_compile(<= 3.3 end diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 66e1b1325..67bc69009 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -707,6 +707,12 @@ static inline double ffp_bits2double(uint64_t bits) { return d; } +#if defined(__x86_64__) && defined(HAVE_X86INTRIN_H) +# include +#elif defined(_MSC_VER) && defined(HAVE_INTRIN_H) +# include +#endif + static inline unsigned int ffc_nlz_int64(uint64_t x) { From 2203b8dce29154d1a167904feb1f9225f57ac213 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 14:17:13 +0200 Subject: [PATCH 359/506] fast_float_parser.h: copy `nlz_int64` from the bigdecimal gem --- ext/json/ext/vendor/fast_float_parser.h | 26 ++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 67bc69009..9e4a66913 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -713,21 +713,37 @@ static inline double ffp_bits2double(uint64_t bits) { # include #endif +#if defined(_MSC_VER) && defined(__AVX2__) +# pragma intrinsic(__lzcnt64) +#endif + static inline unsigned int ffc_nlz_int64(uint64_t x) { -#if defined(_MSC_VER) && defined(__AVX2__) +#if defined(_MSC_VER) && defined(__AVX2__) && defined(HAVE___LZCNT64) return (unsigned int)__lzcnt64(x); -#elif defined(__x86_64__) && defined(__LZCNT__) +#elif defined(__x86_64__) && defined(__LZCNT__) && defined(HAVE__LZCNT_U64) return (unsigned int)_lzcnt_u64(x); -#elif defined(_WIN64) && defined(_MSC_VER) /* &&! defined(__AVX2__) */ +#elif defined(_WIN64) && defined(_MSC_VER) && defined(HAVE__BITSCANREVERSE64) unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; -#elif defined(__has_builtin) && __has_builtin(__builtin_clzll) - return (unsigned int)__builtin_clzll((unsigned long long)x); +#elif __has_builtin(__builtin_clzl) && __has_builtin(__builtin_clzll) && !(defined(__sun) && defined(__sparc)) + if (x == 0) { + return 64; + } + else if (sizeof(long) * CHAR_BIT == 64) { + return (unsigned int)__builtin_clzl((unsigned long)x); + } + else if (sizeof(long long) * CHAR_BIT == 64) { + return (unsigned int)__builtin_clzll((unsigned long long)x); + } + else { + /* :FIXME: Is there a way to make this branch a compile-time error? */ + __builtin_unreachable(); + } #else uint64_t y; From 97dda6509670d7ef5ca4d6f63ab41b8757c4580a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 14:59:42 +0200 Subject: [PATCH 360/506] fast_float_parser.h: handle `__has_builtin` not being defined --- ext/json/ext/vendor/fast_float_parser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 9e4a66913..833c1462e 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -730,7 +730,7 @@ ffc_nlz_int64(uint64_t x) unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; -#elif __has_builtin(__builtin_clzl) && __has_builtin(__builtin_clzll) && !(defined(__sun) && defined(__sparc)) +#elif defined(__has_builtin) && __has_builtin(__builtin_clzl) && __has_builtin(__builtin_clzll) && !(defined(__sun) && defined(__sparc)) if (x == 0) { return 64; } From cdad7dfb2679cda8aff992c270030049e3b10674 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 16 Jun 2026 15:42:48 +0200 Subject: [PATCH 361/506] fast_float_parser.h: fix GCC 8 support --- ext/json/ext/vendor/fast_float_parser.h | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/ext/json/ext/vendor/fast_float_parser.h b/ext/json/ext/vendor/fast_float_parser.h index 833c1462e..6e43b254c 100644 --- a/ext/json/ext/vendor/fast_float_parser.h +++ b/ext/json/ext/vendor/fast_float_parser.h @@ -720,7 +720,10 @@ static inline double ffp_bits2double(uint64_t bits) { static inline unsigned int ffc_nlz_int64(uint64_t x) { -#if defined(_MSC_VER) && defined(__AVX2__) && defined(HAVE___LZCNT64) +#if defined(HAVE_BUILTIN___BUILTIN_CLZLL) && !(defined(__sun) && defined(__sparc)) + return (unsigned int)__builtin_clzll((unsigned long long)x); + +#elif defined(_MSC_VER) && defined(__AVX2__) && defined(HAVE___LZCNT64) return (unsigned int)__lzcnt64(x); #elif defined(__x86_64__) && defined(__LZCNT__) && defined(HAVE__LZCNT_U64) @@ -730,21 +733,6 @@ ffc_nlz_int64(uint64_t x) unsigned long r; return _BitScanReverse64(&r, x) ? (63u - (unsigned int)r) : 64; -#elif defined(__has_builtin) && __has_builtin(__builtin_clzl) && __has_builtin(__builtin_clzll) && !(defined(__sun) && defined(__sparc)) - if (x == 0) { - return 64; - } - else if (sizeof(long) * CHAR_BIT == 64) { - return (unsigned int)__builtin_clzl((unsigned long)x); - } - else if (sizeof(long long) * CHAR_BIT == 64) { - return (unsigned int)__builtin_clzll((unsigned long long)x); - } - else { - /* :FIXME: Is there a way to make this branch a compile-time error? */ - __builtin_unreachable(); - } - #else uint64_t y; unsigned int n = 64; From 11ed69439ff9b418d505a3d167de45361cb1973a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 17 Jun 2026 08:26:47 +0200 Subject: [PATCH 362/506] ResumableParser#<<: call rb_str_modify before shrinking the buffer Fix: https://github.com/ruby/json/issues/1013 The string may be shared. --- ext/json/ext/parser/parser.c | 9 ++++++--- test/json/resumable_parser_test.rb | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c216faf4b..136aab6ae 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2300,16 +2300,19 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) const size_t size = parser->state.end - parser->state.start; const size_t consumed = size - remaining; - char *old_ptr = RSTRING_PTR(parser->buffer); if (RB_OBJ_FROZEN_RAW(parser->buffer)) { VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str))); - MEMCPY(RSTRING_PTR(new_buffer), old_ptr + consumed, char, remaining); + + char *old_ptr = RSTRING_PTR(parser->buffer); + memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining); rb_str_set_len(new_buffer, remaining); offset = 0; parser->buffer = new_buffer; } else if (consumed > (size / 2) && size >= 512) { - MEMMOVE(old_ptr, old_ptr + consumed, char, remaining); + rb_str_modify(parser->buffer); + char *old_ptr = RSTRING_PTR(parser->buffer); + memmove(old_ptr, old_ptr + consumed, remaining); rb_str_set_len(parser->buffer, remaining); offset = 0; } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 600cd33e5..52f1356a7 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -244,6 +244,21 @@ def test_spill_frames_stack assert_equal expected, @parser.value end + def test_buffer_shrink + doc1 = '{"a":"' + ("x" * 800) + '"} {' # >= 512 bytes + doc2 = '"b":1} ' + + parser = JSON::ResumableParser.new({}) + + parser << doc1 # internal buffer becomes a *shared* string here + parser.parse # consume doc1 -> >50% of a >=512B buffer is now consumed + parser.value + + parser << doc2 # buffer is shrinked + parser.parse + parser.value + end + private def assert_partial_value(expected, json) From 5973ecf23eef37070d8019f396e3670a09e273ee Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Jun 2026 10:11:10 +0200 Subject: [PATCH 363/506] Vendor tests from https://github.com/nst/JSONTestSuite It's only testing if we run into a parsing error or not, not testing the actual parsed value, but that's still useful to help ensure we're not changing behavior on dark corners of the spec. --- .../i_number_double_huge_neg_exp.json | 1 + .../fixtures/minefield/i_number_huge_exp.json | 1 + .../minefield/i_number_neg_int_huge_exp.json | 1 + .../i_number_pos_double_huge_exp.json | 1 + .../minefield/i_number_real_neg_overflow.json | 1 + .../minefield/i_number_real_pos_overflow.json | 1 + .../minefield/i_number_real_underflow.json | 1 + .../minefield/i_number_too_big_neg_int.json | 1 + .../minefield/i_number_too_big_pos_int.json | 1 + .../i_number_very_big_negative_int.json | 1 + .../i_object_key_lone_2nd_surrogate.json | 1 + ..._string_1st_surrogate_but_2nd_missing.json | 1 + ...tring_1st_valid_surrogate_2nd_invalid.json | 1 + .../minefield/i_string_UTF-16LE_with_BOM.json | Bin 0 -> 12 bytes .../i_string_UTF-8_invalid_sequence.json | 1 + .../i_string_UTF8_surrogate_U+D800.json | 1 + ...incomplete_surrogate_and_escape_valid.json | 1 + .../i_string_incomplete_surrogate_pair.json | 1 + ...ng_incomplete_surrogates_escape_valid.json | 1 + .../i_string_invalid_lonely_surrogate.json | 1 + .../minefield/i_string_invalid_surrogate.json | 1 + .../minefield/i_string_invalid_utf-8.json | 1 + .../i_string_inverted_surrogates_U+1D11E.json | 1 + .../minefield/i_string_iso_latin_1.json | 1 + .../i_string_lone_second_surrogate.json | 1 + .../i_string_lone_utf8_continuation_byte.json | 1 + .../i_string_not_in_unicode_range.json | 1 + .../i_string_overlong_sequence_2_bytes.json | 1 + .../i_string_overlong_sequence_6_bytes.json | 1 + ...string_overlong_sequence_6_bytes_null.json | 1 + .../minefield/i_string_truncated-utf-8.json | 1 + .../minefield/i_string_utf16BE_no_BOM.json | Bin 0 -> 10 bytes .../minefield/i_string_utf16LE_no_BOM.json | Bin 0 -> 10 bytes .../i_structure_500_nested_arrays.json | 1 + .../i_structure_UTF-8_BOM_empty_object.json | 1 + .../n_array_1_true_without_comma.json | 1 + .../minefield/n_array_a_invalid_utf8.json | 1 + .../n_array_colon_instead_of_comma.json | 1 + .../minefield/n_array_comma_after_close.json | 1 + .../minefield/n_array_comma_and_number.json | 1 + .../minefield/n_array_double_comma.json | 1 + .../minefield/n_array_double_extra_comma.json | 1 + .../minefield/n_array_extra_close.json | 1 + .../minefield/n_array_extra_comma.json | 1 + .../minefield/n_array_incomplete.json | 1 + .../n_array_incomplete_invalid_value.json | 1 + .../n_array_inner_array_no_comma.json | 1 + .../minefield/n_array_invalid_utf8.json | 1 + .../n_array_items_separated_by_semicolon.json | 1 + .../minefield/n_array_just_comma.json | 1 + .../minefield/n_array_just_minus.json | 1 + .../minefield/n_array_missing_value.json | 1 + .../minefield/n_array_newlines_unclosed.json | 3 + .../minefield/n_array_number_and_comma.json | 1 + .../n_array_number_and_several_commas.json | 1 + .../n_array_spaces_vertical_tab_formfeed.json | 1 + .../minefield/n_array_star_inside.json | 1 + .../fixtures/minefield/n_array_unclosed.json | 1 + .../n_array_unclosed_trailing_comma.json | 1 + .../n_array_unclosed_with_new_lines.json | 3 + .../n_array_unclosed_with_object_inside.json | 1 + .../minefield/n_incomplete_false.json | 1 + .../fixtures/minefield/n_incomplete_null.json | 1 + .../fixtures/minefield/n_incomplete_true.json | 1 + .../n_multidigit_number_then_00.json | Bin 0 -> 4 bytes test/json/fixtures/minefield/n_number_++.json | 1 + test/json/fixtures/minefield/n_number_+1.json | 1 + .../fixtures/minefield/n_number_+Inf.json | 1 + .../json/fixtures/minefield/n_number_-01.json | 1 + .../fixtures/minefield/n_number_-1.0..json | 1 + .../json/fixtures/minefield/n_number_-2..json | 1 + .../fixtures/minefield/n_number_-NaN.json | 1 + .../json/fixtures/minefield/n_number_.-1.json | 1 + .../fixtures/minefield/n_number_.2e-3.json | 1 + .../fixtures/minefield/n_number_0.1.2.json | 1 + .../fixtures/minefield/n_number_0.3e+.json | 1 + .../fixtures/minefield/n_number_0.3e.json | 1 + .../fixtures/minefield/n_number_0.e1.json | 1 + .../minefield/n_number_0_capital_E+.json | 1 + .../minefield/n_number_0_capital_E.json | 1 + .../json/fixtures/minefield/n_number_0e+.json | 1 + test/json/fixtures/minefield/n_number_0e.json | 1 + .../fixtures/minefield/n_number_1.0e+.json | 1 + .../fixtures/minefield/n_number_1.0e-.json | 1 + .../fixtures/minefield/n_number_1.0e.json | 1 + .../fixtures/minefield/n_number_1_000.json | 1 + .../fixtures/minefield/n_number_1eE2.json | 1 + .../fixtures/minefield/n_number_2.e+3.json | 1 + .../fixtures/minefield/n_number_2.e-3.json | 1 + .../fixtures/minefield/n_number_2.e3.json | 1 + .../fixtures/minefield/n_number_9.e+.json | 1 + .../json/fixtures/minefield/n_number_Inf.json | 1 + .../json/fixtures/minefield/n_number_NaN.json | 1 + .../n_number_U+FF11_fullwidth_digit_one.json | 1 + .../minefield/n_number_expression.json | 1 + .../minefield/n_number_hex_1_digit.json | 1 + .../minefield/n_number_hex_2_digits.json | 1 + .../fixtures/minefield/n_number_infinity.json | 1 + .../minefield/n_number_invalid+-.json | 1 + .../n_number_invalid-negative-real.json | 1 + .../n_number_invalid-utf-8-in-bigger-int.json | 1 + .../n_number_invalid-utf-8-in-exponent.json | 1 + .../n_number_invalid-utf-8-in-int.json | 1 + .../minefield/n_number_minus_infinity.json | 1 + ...mber_minus_sign_with_trailing_garbage.json | 1 + .../minefield/n_number_minus_space_1.json | 1 + .../n_number_neg_int_starting_with_zero.json | 1 + .../n_number_neg_real_without_int_part.json | 1 + .../n_number_neg_with_garbage_at_end.json | 1 + .../n_number_real_garbage_after_e.json | 1 + ...number_real_with_invalid_utf8_after_e.json | 1 + ...n_number_real_without_fractional_part.json | 1 + .../minefield/n_number_starting_with_dot.json | 1 + .../minefield/n_number_with_alpha.json | 1 + .../minefield/n_number_with_alpha_char.json | 1 + .../minefield/n_number_with_leading_zero.json | 1 + .../minefield/n_object_bad_value.json | 1 + .../minefield/n_object_bracket_key.json | 1 + .../n_object_comma_instead_of_colon.json | 1 + .../minefield/n_object_double_colon.json | 1 + .../fixtures/minefield/n_object_emoji.json | 1 + .../minefield/n_object_garbage_at_end.json | 1 + .../n_object_key_with_single_quotes.json | 1 + ...uation_byte_in_key_and_trailing_comma.json | 1 + .../minefield/n_object_missing_colon.json | 1 + .../minefield/n_object_missing_key.json | 1 + .../minefield/n_object_missing_semicolon.json | 1 + .../minefield/n_object_missing_value.json | 1 + .../fixtures/minefield/n_object_no-colon.json | 1 + .../minefield/n_object_non_string_key.json | 1 + ...on_string_key_but_huge_number_instead.json | 1 + .../n_object_repeated_null_null.json | 1 + .../n_object_several_trailing_commas.json | 1 + .../minefield/n_object_single_quote.json | 1 + .../minefield/n_object_trailing_comma.json | 1 + .../minefield/n_object_trailing_comment.json | 1 + .../n_object_trailing_comment_open.json | 1 + .../n_object_trailing_comment_slash_open.json | 1 + ...railing_comment_slash_open_incomplete.json | 1 + .../n_object_two_commas_in_a_row.json | 1 + .../minefield/n_object_unquoted_key.json | 1 + .../n_object_unterminated-value.json | 1 + .../n_object_with_single_string.json | 1 + .../n_object_with_trailing_garbage.json | 1 + .../fixtures/minefield/n_single_space.json | 1 + .../n_string_1_surrogate_then_escape.json | 1 + .../n_string_1_surrogate_then_escape_u.json | 1 + .../n_string_1_surrogate_then_escape_u1.json | 1 + .../n_string_1_surrogate_then_escape_u1x.json | 1 + .../n_string_accentuated_char_no_quotes.json | 1 + .../minefield/n_string_backslash_00.json | Bin 0 -> 6 bytes .../fixtures/minefield/n_string_escape_x.json | 1 + .../n_string_escaped_backslash_bad.json | 1 + .../n_string_escaped_ctrl_char_tab.json | 1 + .../minefield/n_string_escaped_emoji.json | 1 + .../minefield/n_string_incomplete_escape.json | 1 + ...n_string_incomplete_escaped_character.json | 1 + .../n_string_incomplete_surrogate.json | 1 + ...g_incomplete_surrogate_escape_invalid.json | 1 + .../n_string_invalid-utf-8-in-escape.json | 1 + .../n_string_invalid_backslash_esc.json | 1 + .../n_string_invalid_unicode_escape.json | 1 + .../n_string_invalid_utf8_after_escape.json | 1 + .../n_string_leading_uescaped_thinspace.json | 1 + .../n_string_no_quotes_with_bad_escape.json | 1 + .../n_string_single_doublequote.json | 1 + .../minefield/n_string_single_quote.json | 1 + ...string_single_string_no_double_quotes.json | 1 + .../n_string_start_escape_unclosed.json | 1 + .../n_string_unescaped_ctrl_char.json | Bin 0 -> 7 bytes .../minefield/n_string_unescaped_newline.json | 2 + .../minefield/n_string_unescaped_tab.json | 1 + .../minefield/n_string_unicode_CapitalU.json | 1 + .../n_string_with_trailing_garbage.json | 1 + .../n_structure_100000_opening_arrays.json | 1 + .../n_structure_U+2060_word_joined.json | 1 + .../n_structure_UTF8_BOM_no_data.json | 1 + .../n_structure_angle_bracket_..json | 1 + .../n_structure_angle_bracket_null.json | 1 + .../n_structure_array_trailing_garbage.json | 1 + ...tructure_array_with_extra_array_close.json | 1 + ..._structure_array_with_unclosed_string.json | 1 + .../n_structure_ascii-unicode-identifier.json | 1 + .../n_structure_capitalized_True.json | 1 + .../n_structure_close_unopened_array.json | 1 + ...ucture_comma_instead_of_closing_brace.json | 1 + .../minefield/n_structure_double_array.json | 1 + .../minefield/n_structure_end_array.json | 1 + .../n_structure_incomplete_UTF8_BOM.json | 1 + .../n_structure_lone-invalid-utf-8.json | 1 + .../n_structure_lone-open-bracket.json | 1 + .../minefield/n_structure_no_data.json | 0 .../n_structure_null-byte-outside-string.json | Bin 0 -> 3 bytes ...tructure_number_with_trailing_garbage.json | 1 + ...ure_object_followed_by_closing_object.json | 1 + .../n_structure_object_unclosed_no_value.json | 1 + .../n_structure_object_with_comment.json | 1 + ...tructure_object_with_trailing_garbage.json | 1 + .../n_structure_open_array_apostrophe.json | 1 + .../n_structure_open_array_comma.json | 1 + .../n_structure_open_array_object.json | 1 + .../n_structure_open_array_open_object.json | 1 + .../n_structure_open_array_open_string.json | 1 + .../n_structure_open_array_string.json | 1 + .../minefield/n_structure_open_object.json | 1 + .../n_structure_open_object_close_array.json | 1 + .../n_structure_open_object_comma.json | 1 + .../n_structure_open_object_open_array.json | 1 + .../n_structure_open_object_open_string.json | 1 + ...e_open_object_string_with_apostrophes.json | 1 + .../minefield/n_structure_open_open.json | 1 + .../minefield/n_structure_single_eacute.json | 1 + .../minefield/n_structure_single_star.json | 1 + .../minefield/n_structure_trailing_#.json | 1 + ...n_structure_uescaped_LF_before_string.json | 1 + .../minefield/n_structure_unclosed_array.json | 1 + ...structure_unclosed_array_partial_null.json | 1 + ...cture_unclosed_array_unfinished_false.json | 1 + ...ucture_unclosed_array_unfinished_true.json | 1 + .../n_structure_unclosed_object.json | 1 + .../n_structure_unicode-identifier.json | 1 + ...ructure_whitespace_U+2060_word_joiner.json | 1 + .../n_structure_whitespace_formfeed.json | 1 + .../minefield/y_array_arraysWithSpaces.json | 1 + .../minefield/y_array_empty-string.json | 1 + .../fixtures/minefield/y_array_empty.json | 1 + .../y_array_ending_with_newline.json | 1 + .../fixtures/minefield/y_array_false.json | 1 + .../minefield/y_array_heterogeneous.json | 1 + .../json/fixtures/minefield/y_array_null.json | 1 + .../minefield/y_array_with_1_and_newline.json | 2 + .../minefield/y_array_with_leading_space.json | 1 + .../minefield/y_array_with_several_null.json | 1 + .../y_array_with_trailing_space.json | 1 + test/json/fixtures/minefield/y_number.json | 1 + .../fixtures/minefield/y_number_0e+1.json | 1 + .../json/fixtures/minefield/y_number_0e1.json | 1 + .../minefield/y_number_after_space.json | 1 + .../y_number_double_close_to_zero.json | 1 + .../minefield/y_number_int_with_exp.json | 1 + .../minefield/y_number_minus_zero.json | 1 + .../minefield/y_number_negative_int.json | 1 + .../minefield/y_number_negative_one.json | 1 + .../minefield/y_number_negative_zero.json | 1 + .../minefield/y_number_real_capital_e.json | 1 + .../y_number_real_capital_e_neg_exp.json | 1 + .../y_number_real_capital_e_pos_exp.json | 1 + .../minefield/y_number_real_exponent.json | 1 + .../y_number_real_fraction_exponent.json | 1 + .../minefield/y_number_real_neg_exp.json | 1 + .../minefield/y_number_real_pos_exponent.json | 1 + .../minefield/y_number_simple_int.json | 1 + .../minefield/y_number_simple_real.json | 1 + test/json/fixtures/minefield/y_object.json | 1 + .../fixtures/minefield/y_object_basic.json | 1 + .../minefield/y_object_duplicated_key.json | 1 + .../y_object_duplicated_key_and_value.json | 1 + .../fixtures/minefield/y_object_empty.json | 1 + .../minefield/y_object_empty_key.json | 1 + .../y_object_escaped_null_in_key.json | 1 + .../minefield/y_object_extreme_numbers.json | 1 + .../minefield/y_object_long_strings.json | 1 + .../fixtures/minefield/y_object_simple.json | 1 + .../minefield/y_object_string_unicode.json | 1 + .../minefield/y_object_with_newlines.json | 3 + .../y_string_1_2_3_bytes_UTF-8_sequences.json | 1 + .../y_string_accepted_surrogate_pair.json | 1 + .../y_string_accepted_surrogate_pairs.json | 1 + .../minefield/y_string_allowed_escapes.json | 1 + ...y_string_backslash_and_u_escaped_zero.json | 1 + .../y_string_backslash_doublequotes.json | 1 + .../fixtures/minefield/y_string_comments.json | 1 + .../minefield/y_string_double_escape_a.json | 1 + .../minefield/y_string_double_escape_n.json | 1 + .../y_string_escaped_control_character.json | 1 + .../y_string_escaped_noncharacter.json | 1 + .../fixtures/minefield/y_string_in_array.json | 1 + .../y_string_in_array_with_leading_space.json | 1 + .../y_string_last_surrogates_1_and_2.json | 1 + .../minefield/y_string_nbsp_uescaped.json | 1 + ...y_string_nonCharacterInUTF-8_U+10FFFF.json | 1 + .../y_string_nonCharacterInUTF-8_U+FFFF.json | 1 + .../minefield/y_string_null_escape.json | 1 + .../minefield/y_string_one-byte-utf-8.json | 1 + test/json/fixtures/minefield/y_string_pi.json | 1 + ...ring_reservedCharacterInUTF-8_U+1BFFF.json | 1 + .../minefield/y_string_simple_ascii.json | 1 + .../fixtures/minefield/y_string_space.json | 1 + ...rogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json | 1 + .../minefield/y_string_three-byte-utf-8.json | 1 + .../minefield/y_string_two-byte-utf-8.json | 1 + .../minefield/y_string_u+2028_line_sep.json | 1 + .../minefield/y_string_u+2029_par_sep.json | 1 + .../fixtures/minefield/y_string_uEscape.json | 1 + .../minefield/y_string_uescaped_newline.json | 1 + .../y_string_unescaped_char_delete.json | 1 + .../fixtures/minefield/y_string_unicode.json | 1 + .../y_string_unicodeEscapedBackslash.json | 1 + .../minefield/y_string_unicode_2.json | 1 + .../y_string_unicode_U+10FFFE_nonchar.json | 1 + .../y_string_unicode_U+1FFFE_nonchar.json | 1 + ...tring_unicode_U+200B_ZERO_WIDTH_SPACE.json | 1 + ..._string_unicode_U+2064_invisible_plus.json | 1 + .../y_string_unicode_U+FDD0_nonchar.json | 1 + .../y_string_unicode_U+FFFE_nonchar.json | 1 + ...y_string_unicode_escaped_double_quote.json | 1 + .../fixtures/minefield/y_string_utf8.json | 1 + .../y_string_with_del_character.json | 1 + .../minefield/y_structure_lonely_false.json | 1 + .../minefield/y_structure_lonely_int.json | 1 + .../y_structure_lonely_negative_real.json | 1 + .../minefield/y_structure_lonely_null.json | 1 + .../minefield/y_structure_lonely_string.json | 1 + .../minefield/y_structure_lonely_true.json | 1 + .../minefield/y_structure_string_empty.json | 1 + .../y_structure_trailing_newline.json | 1 + .../minefield/y_structure_true_in_array.json | 1 + .../y_structure_whitespace_array.json | 1 + test/json/json_minefield_parser_test.rb | 114 ++++++++++++++++++ 319 files changed, 432 insertions(+) create mode 100644 test/json/fixtures/minefield/i_number_double_huge_neg_exp.json create mode 100644 test/json/fixtures/minefield/i_number_huge_exp.json create mode 100755 test/json/fixtures/minefield/i_number_neg_int_huge_exp.json create mode 100755 test/json/fixtures/minefield/i_number_pos_double_huge_exp.json create mode 100644 test/json/fixtures/minefield/i_number_real_neg_overflow.json create mode 100644 test/json/fixtures/minefield/i_number_real_pos_overflow.json create mode 100644 test/json/fixtures/minefield/i_number_real_underflow.json create mode 100644 test/json/fixtures/minefield/i_number_too_big_neg_int.json create mode 100644 test/json/fixtures/minefield/i_number_too_big_pos_int.json create mode 100755 test/json/fixtures/minefield/i_number_very_big_negative_int.json create mode 100644 test/json/fixtures/minefield/i_object_key_lone_2nd_surrogate.json create mode 100644 test/json/fixtures/minefield/i_string_1st_surrogate_but_2nd_missing.json create mode 100644 test/json/fixtures/minefield/i_string_1st_valid_surrogate_2nd_invalid.json create mode 100644 test/json/fixtures/minefield/i_string_UTF-16LE_with_BOM.json create mode 100755 test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json create mode 100644 test/json/fixtures/minefield/i_string_UTF8_surrogate_U+D800.json create mode 100755 test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json create mode 100755 test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json create mode 100755 test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json create mode 100755 test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json create mode 100755 test/json/fixtures/minefield/i_string_invalid_surrogate.json create mode 100644 test/json/fixtures/minefield/i_string_invalid_utf-8.json create mode 100755 test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json create mode 100644 test/json/fixtures/minefield/i_string_iso_latin_1.json create mode 100644 test/json/fixtures/minefield/i_string_lone_second_surrogate.json create mode 100644 test/json/fixtures/minefield/i_string_lone_utf8_continuation_byte.json create mode 100644 test/json/fixtures/minefield/i_string_not_in_unicode_range.json create mode 100644 test/json/fixtures/minefield/i_string_overlong_sequence_2_bytes.json create mode 100755 test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json create mode 100755 test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json create mode 100644 test/json/fixtures/minefield/i_string_truncated-utf-8.json create mode 100644 test/json/fixtures/minefield/i_string_utf16BE_no_BOM.json create mode 100644 test/json/fixtures/minefield/i_string_utf16LE_no_BOM.json create mode 100644 test/json/fixtures/minefield/i_structure_500_nested_arrays.json create mode 100755 test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json create mode 100644 test/json/fixtures/minefield/n_array_1_true_without_comma.json create mode 100644 test/json/fixtures/minefield/n_array_a_invalid_utf8.json create mode 100644 test/json/fixtures/minefield/n_array_colon_instead_of_comma.json create mode 100644 test/json/fixtures/minefield/n_array_comma_after_close.json create mode 100755 test/json/fixtures/minefield/n_array_comma_and_number.json create mode 100755 test/json/fixtures/minefield/n_array_double_comma.json create mode 100644 test/json/fixtures/minefield/n_array_double_extra_comma.json create mode 100644 test/json/fixtures/minefield/n_array_extra_close.json create mode 100644 test/json/fixtures/minefield/n_array_extra_comma.json create mode 100644 test/json/fixtures/minefield/n_array_incomplete.json create mode 100644 test/json/fixtures/minefield/n_array_incomplete_invalid_value.json create mode 100644 test/json/fixtures/minefield/n_array_inner_array_no_comma.json create mode 100644 test/json/fixtures/minefield/n_array_invalid_utf8.json create mode 100755 test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json create mode 100755 test/json/fixtures/minefield/n_array_just_comma.json create mode 100755 test/json/fixtures/minefield/n_array_just_minus.json create mode 100644 test/json/fixtures/minefield/n_array_missing_value.json create mode 100644 test/json/fixtures/minefield/n_array_newlines_unclosed.json create mode 100755 test/json/fixtures/minefield/n_array_number_and_comma.json create mode 100755 test/json/fixtures/minefield/n_array_number_and_several_commas.json create mode 100755 test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json create mode 100755 test/json/fixtures/minefield/n_array_star_inside.json create mode 100644 test/json/fixtures/minefield/n_array_unclosed.json create mode 100644 test/json/fixtures/minefield/n_array_unclosed_trailing_comma.json create mode 100644 test/json/fixtures/minefield/n_array_unclosed_with_new_lines.json create mode 100644 test/json/fixtures/minefield/n_array_unclosed_with_object_inside.json create mode 100644 test/json/fixtures/minefield/n_incomplete_false.json create mode 100644 test/json/fixtures/minefield/n_incomplete_null.json create mode 100644 test/json/fixtures/minefield/n_incomplete_true.json create mode 100644 test/json/fixtures/minefield/n_multidigit_number_then_00.json create mode 100644 test/json/fixtures/minefield/n_number_++.json create mode 100755 test/json/fixtures/minefield/n_number_+1.json create mode 100755 test/json/fixtures/minefield/n_number_+Inf.json create mode 100755 test/json/fixtures/minefield/n_number_-01.json create mode 100755 test/json/fixtures/minefield/n_number_-1.0..json create mode 100755 test/json/fixtures/minefield/n_number_-2..json create mode 100755 test/json/fixtures/minefield/n_number_-NaN.json create mode 100644 test/json/fixtures/minefield/n_number_.-1.json create mode 100755 test/json/fixtures/minefield/n_number_.2e-3.json create mode 100755 test/json/fixtures/minefield/n_number_0.1.2.json create mode 100644 test/json/fixtures/minefield/n_number_0.3e+.json create mode 100644 test/json/fixtures/minefield/n_number_0.3e.json create mode 100644 test/json/fixtures/minefield/n_number_0.e1.json create mode 100644 test/json/fixtures/minefield/n_number_0_capital_E+.json create mode 100755 test/json/fixtures/minefield/n_number_0_capital_E.json create mode 100644 test/json/fixtures/minefield/n_number_0e+.json create mode 100644 test/json/fixtures/minefield/n_number_0e.json create mode 100755 test/json/fixtures/minefield/n_number_1.0e+.json create mode 100755 test/json/fixtures/minefield/n_number_1.0e-.json create mode 100755 test/json/fixtures/minefield/n_number_1.0e.json create mode 100755 test/json/fixtures/minefield/n_number_1_000.json create mode 100755 test/json/fixtures/minefield/n_number_1eE2.json create mode 100755 test/json/fixtures/minefield/n_number_2.e+3.json create mode 100755 test/json/fixtures/minefield/n_number_2.e-3.json create mode 100755 test/json/fixtures/minefield/n_number_2.e3.json create mode 100644 test/json/fixtures/minefield/n_number_9.e+.json create mode 100755 test/json/fixtures/minefield/n_number_Inf.json create mode 100755 test/json/fixtures/minefield/n_number_NaN.json create mode 100644 test/json/fixtures/minefield/n_number_U+FF11_fullwidth_digit_one.json create mode 100644 test/json/fixtures/minefield/n_number_expression.json create mode 100644 test/json/fixtures/minefield/n_number_hex_1_digit.json create mode 100644 test/json/fixtures/minefield/n_number_hex_2_digits.json create mode 100755 test/json/fixtures/minefield/n_number_infinity.json create mode 100644 test/json/fixtures/minefield/n_number_invalid+-.json create mode 100644 test/json/fixtures/minefield/n_number_invalid-negative-real.json create mode 100644 test/json/fixtures/minefield/n_number_invalid-utf-8-in-bigger-int.json create mode 100644 test/json/fixtures/minefield/n_number_invalid-utf-8-in-exponent.json create mode 100644 test/json/fixtures/minefield/n_number_invalid-utf-8-in-int.json create mode 100755 test/json/fixtures/minefield/n_number_minus_infinity.json create mode 100644 test/json/fixtures/minefield/n_number_minus_sign_with_trailing_garbage.json create mode 100644 test/json/fixtures/minefield/n_number_minus_space_1.json create mode 100644 test/json/fixtures/minefield/n_number_neg_int_starting_with_zero.json create mode 100755 test/json/fixtures/minefield/n_number_neg_real_without_int_part.json create mode 100644 test/json/fixtures/minefield/n_number_neg_with_garbage_at_end.json create mode 100644 test/json/fixtures/minefield/n_number_real_garbage_after_e.json create mode 100644 test/json/fixtures/minefield/n_number_real_with_invalid_utf8_after_e.json create mode 100755 test/json/fixtures/minefield/n_number_real_without_fractional_part.json create mode 100755 test/json/fixtures/minefield/n_number_starting_with_dot.json create mode 100644 test/json/fixtures/minefield/n_number_with_alpha.json create mode 100644 test/json/fixtures/minefield/n_number_with_alpha_char.json create mode 100755 test/json/fixtures/minefield/n_number_with_leading_zero.json create mode 100644 test/json/fixtures/minefield/n_object_bad_value.json create mode 100644 test/json/fixtures/minefield/n_object_bracket_key.json create mode 100644 test/json/fixtures/minefield/n_object_comma_instead_of_colon.json create mode 100644 test/json/fixtures/minefield/n_object_double_colon.json create mode 100644 test/json/fixtures/minefield/n_object_emoji.json create mode 100644 test/json/fixtures/minefield/n_object_garbage_at_end.json create mode 100755 test/json/fixtures/minefield/n_object_key_with_single_quotes.json create mode 100644 test/json/fixtures/minefield/n_object_lone_continuation_byte_in_key_and_trailing_comma.json create mode 100644 test/json/fixtures/minefield/n_object_missing_colon.json create mode 100755 test/json/fixtures/minefield/n_object_missing_key.json create mode 100755 test/json/fixtures/minefield/n_object_missing_semicolon.json create mode 100644 test/json/fixtures/minefield/n_object_missing_value.json create mode 100644 test/json/fixtures/minefield/n_object_no-colon.json create mode 100755 test/json/fixtures/minefield/n_object_non_string_key.json create mode 100755 test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json create mode 100755 test/json/fixtures/minefield/n_object_repeated_null_null.json create mode 100755 test/json/fixtures/minefield/n_object_several_trailing_commas.json create mode 100644 test/json/fixtures/minefield/n_object_single_quote.json create mode 100755 test/json/fixtures/minefield/n_object_trailing_comma.json create mode 100644 test/json/fixtures/minefield/n_object_trailing_comment.json create mode 100644 test/json/fixtures/minefield/n_object_trailing_comment_open.json create mode 100644 test/json/fixtures/minefield/n_object_trailing_comment_slash_open.json create mode 100644 test/json/fixtures/minefield/n_object_trailing_comment_slash_open_incomplete.json create mode 100755 test/json/fixtures/minefield/n_object_two_commas_in_a_row.json create mode 100644 test/json/fixtures/minefield/n_object_unquoted_key.json create mode 100644 test/json/fixtures/minefield/n_object_unterminated-value.json create mode 100644 test/json/fixtures/minefield/n_object_with_single_string.json create mode 100644 test/json/fixtures/minefield/n_object_with_trailing_garbage.json create mode 100755 test/json/fixtures/minefield/n_single_space.json create mode 100644 test/json/fixtures/minefield/n_string_1_surrogate_then_escape.json create mode 100644 test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u.json create mode 100644 test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1.json create mode 100644 test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1x.json create mode 100644 test/json/fixtures/minefield/n_string_accentuated_char_no_quotes.json create mode 100644 test/json/fixtures/minefield/n_string_backslash_00.json create mode 100644 test/json/fixtures/minefield/n_string_escape_x.json create mode 100755 test/json/fixtures/minefield/n_string_escaped_backslash_bad.json create mode 100644 test/json/fixtures/minefield/n_string_escaped_ctrl_char_tab.json create mode 100644 test/json/fixtures/minefield/n_string_escaped_emoji.json create mode 100755 test/json/fixtures/minefield/n_string_incomplete_escape.json create mode 100755 test/json/fixtures/minefield/n_string_incomplete_escaped_character.json create mode 100755 test/json/fixtures/minefield/n_string_incomplete_surrogate.json create mode 100755 test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json create mode 100644 test/json/fixtures/minefield/n_string_invalid-utf-8-in-escape.json create mode 100755 test/json/fixtures/minefield/n_string_invalid_backslash_esc.json create mode 100644 test/json/fixtures/minefield/n_string_invalid_unicode_escape.json create mode 100644 test/json/fixtures/minefield/n_string_invalid_utf8_after_escape.json create mode 100755 test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json create mode 100644 test/json/fixtures/minefield/n_string_no_quotes_with_bad_escape.json create mode 100755 test/json/fixtures/minefield/n_string_single_doublequote.json create mode 100644 test/json/fixtures/minefield/n_string_single_quote.json create mode 100755 test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json create mode 100644 test/json/fixtures/minefield/n_string_start_escape_unclosed.json create mode 100755 test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json create mode 100644 test/json/fixtures/minefield/n_string_unescaped_newline.json create mode 100644 test/json/fixtures/minefield/n_string_unescaped_tab.json create mode 100644 test/json/fixtures/minefield/n_string_unicode_CapitalU.json create mode 100644 test/json/fixtures/minefield/n_string_with_trailing_garbage.json create mode 100644 test/json/fixtures/minefield/n_structure_100000_opening_arrays.json create mode 100644 test/json/fixtures/minefield/n_structure_U+2060_word_joined.json create mode 100755 test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json create mode 100755 test/json/fixtures/minefield/n_structure_angle_bracket_..json create mode 100755 test/json/fixtures/minefield/n_structure_angle_bracket_null.json create mode 100644 test/json/fixtures/minefield/n_structure_array_trailing_garbage.json create mode 100755 test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json create mode 100755 test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json create mode 100644 test/json/fixtures/minefield/n_structure_ascii-unicode-identifier.json create mode 100755 test/json/fixtures/minefield/n_structure_capitalized_True.json create mode 100755 test/json/fixtures/minefield/n_structure_close_unopened_array.json create mode 100644 test/json/fixtures/minefield/n_structure_comma_instead_of_closing_brace.json create mode 100755 test/json/fixtures/minefield/n_structure_double_array.json create mode 100644 test/json/fixtures/minefield/n_structure_end_array.json create mode 100755 test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json create mode 100644 test/json/fixtures/minefield/n_structure_lone-invalid-utf-8.json create mode 100644 test/json/fixtures/minefield/n_structure_lone-open-bracket.json create mode 100644 test/json/fixtures/minefield/n_structure_no_data.json create mode 100644 test/json/fixtures/minefield/n_structure_null-byte-outside-string.json create mode 100644 test/json/fixtures/minefield/n_structure_number_with_trailing_garbage.json create mode 100644 test/json/fixtures/minefield/n_structure_object_followed_by_closing_object.json create mode 100644 test/json/fixtures/minefield/n_structure_object_unclosed_no_value.json create mode 100644 test/json/fixtures/minefield/n_structure_object_with_comment.json create mode 100644 test/json/fixtures/minefield/n_structure_object_with_trailing_garbage.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_apostrophe.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_comma.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_object.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_open_object.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_open_string.json create mode 100644 test/json/fixtures/minefield/n_structure_open_array_string.json create mode 100644 test/json/fixtures/minefield/n_structure_open_object.json create mode 100755 test/json/fixtures/minefield/n_structure_open_object_close_array.json create mode 100644 test/json/fixtures/minefield/n_structure_open_object_comma.json create mode 100644 test/json/fixtures/minefield/n_structure_open_object_open_array.json create mode 100644 test/json/fixtures/minefield/n_structure_open_object_open_string.json create mode 100644 test/json/fixtures/minefield/n_structure_open_object_string_with_apostrophes.json create mode 100644 test/json/fixtures/minefield/n_structure_open_open.json create mode 100644 test/json/fixtures/minefield/n_structure_single_eacute.json create mode 100755 test/json/fixtures/minefield/n_structure_single_star.json create mode 100644 test/json/fixtures/minefield/n_structure_trailing_#.json create mode 100755 test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json create mode 100755 test/json/fixtures/minefield/n_structure_unclosed_array.json create mode 100644 test/json/fixtures/minefield/n_structure_unclosed_array_partial_null.json create mode 100644 test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_false.json create mode 100644 test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_true.json create mode 100755 test/json/fixtures/minefield/n_structure_unclosed_object.json create mode 100644 test/json/fixtures/minefield/n_structure_unicode-identifier.json create mode 100755 test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json create mode 100755 test/json/fixtures/minefield/n_structure_whitespace_formfeed.json create mode 100755 test/json/fixtures/minefield/y_array_arraysWithSpaces.json create mode 100644 test/json/fixtures/minefield/y_array_empty-string.json create mode 100755 test/json/fixtures/minefield/y_array_empty.json create mode 100755 test/json/fixtures/minefield/y_array_ending_with_newline.json create mode 100644 test/json/fixtures/minefield/y_array_false.json create mode 100755 test/json/fixtures/minefield/y_array_heterogeneous.json create mode 100644 test/json/fixtures/minefield/y_array_null.json create mode 100644 test/json/fixtures/minefield/y_array_with_1_and_newline.json create mode 100755 test/json/fixtures/minefield/y_array_with_leading_space.json create mode 100755 test/json/fixtures/minefield/y_array_with_several_null.json create mode 100755 test/json/fixtures/minefield/y_array_with_trailing_space.json create mode 100644 test/json/fixtures/minefield/y_number.json create mode 100755 test/json/fixtures/minefield/y_number_0e+1.json create mode 100755 test/json/fixtures/minefield/y_number_0e1.json create mode 100644 test/json/fixtures/minefield/y_number_after_space.json create mode 100755 test/json/fixtures/minefield/y_number_double_close_to_zero.json create mode 100755 test/json/fixtures/minefield/y_number_int_with_exp.json create mode 100755 test/json/fixtures/minefield/y_number_minus_zero.json create mode 100644 test/json/fixtures/minefield/y_number_negative_int.json create mode 100644 test/json/fixtures/minefield/y_number_negative_one.json create mode 100644 test/json/fixtures/minefield/y_number_negative_zero.json create mode 100644 test/json/fixtures/minefield/y_number_real_capital_e.json create mode 100644 test/json/fixtures/minefield/y_number_real_capital_e_neg_exp.json create mode 100644 test/json/fixtures/minefield/y_number_real_capital_e_pos_exp.json create mode 100644 test/json/fixtures/minefield/y_number_real_exponent.json create mode 100644 test/json/fixtures/minefield/y_number_real_fraction_exponent.json create mode 100644 test/json/fixtures/minefield/y_number_real_neg_exp.json create mode 100644 test/json/fixtures/minefield/y_number_real_pos_exponent.json create mode 100644 test/json/fixtures/minefield/y_number_simple_int.json create mode 100644 test/json/fixtures/minefield/y_number_simple_real.json create mode 100755 test/json/fixtures/minefield/y_object.json create mode 100755 test/json/fixtures/minefield/y_object_basic.json create mode 100755 test/json/fixtures/minefield/y_object_duplicated_key.json create mode 100755 test/json/fixtures/minefield/y_object_duplicated_key_and_value.json create mode 100644 test/json/fixtures/minefield/y_object_empty.json create mode 100755 test/json/fixtures/minefield/y_object_empty_key.json create mode 100644 test/json/fixtures/minefield/y_object_escaped_null_in_key.json create mode 100644 test/json/fixtures/minefield/y_object_extreme_numbers.json create mode 100644 test/json/fixtures/minefield/y_object_long_strings.json create mode 100644 test/json/fixtures/minefield/y_object_simple.json create mode 100644 test/json/fixtures/minefield/y_object_string_unicode.json create mode 100644 test/json/fixtures/minefield/y_object_with_newlines.json create mode 100755 test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json create mode 100755 test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json create mode 100755 test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json create mode 100644 test/json/fixtures/minefield/y_string_allowed_escapes.json create mode 100755 test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json create mode 100644 test/json/fixtures/minefield/y_string_backslash_doublequotes.json create mode 100644 test/json/fixtures/minefield/y_string_comments.json create mode 100644 test/json/fixtures/minefield/y_string_double_escape_a.json create mode 100644 test/json/fixtures/minefield/y_string_double_escape_n.json create mode 100644 test/json/fixtures/minefield/y_string_escaped_control_character.json create mode 100755 test/json/fixtures/minefield/y_string_escaped_noncharacter.json create mode 100755 test/json/fixtures/minefield/y_string_in_array.json create mode 100755 test/json/fixtures/minefield/y_string_in_array_with_leading_space.json create mode 100644 test/json/fixtures/minefield/y_string_last_surrogates_1_and_2.json create mode 100644 test/json/fixtures/minefield/y_string_nbsp_uescaped.json create mode 100755 test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json create mode 100755 test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json create mode 100644 test/json/fixtures/minefield/y_string_null_escape.json create mode 100644 test/json/fixtures/minefield/y_string_one-byte-utf-8.json create mode 100644 test/json/fixtures/minefield/y_string_pi.json create mode 100755 test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json create mode 100644 test/json/fixtures/minefield/y_string_simple_ascii.json create mode 100644 test/json/fixtures/minefield/y_string_space.json create mode 100755 test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json create mode 100644 test/json/fixtures/minefield/y_string_three-byte-utf-8.json create mode 100644 test/json/fixtures/minefield/y_string_two-byte-utf-8.json create mode 100755 test/json/fixtures/minefield/y_string_u+2028_line_sep.json create mode 100755 test/json/fixtures/minefield/y_string_u+2029_par_sep.json create mode 100755 test/json/fixtures/minefield/y_string_uEscape.json create mode 100644 test/json/fixtures/minefield/y_string_uescaped_newline.json create mode 100755 test/json/fixtures/minefield/y_string_unescaped_char_delete.json create mode 100644 test/json/fixtures/minefield/y_string_unicode.json create mode 100755 test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_2.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+10FFFE_nonchar.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+1FFFE_nonchar.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+2064_invisible_plus.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+FDD0_nonchar.json create mode 100644 test/json/fixtures/minefield/y_string_unicode_U+FFFE_nonchar.json create mode 100755 test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json create mode 100644 test/json/fixtures/minefield/y_string_utf8.json create mode 100755 test/json/fixtures/minefield/y_string_with_del_character.json create mode 100644 test/json/fixtures/minefield/y_structure_lonely_false.json create mode 100755 test/json/fixtures/minefield/y_structure_lonely_int.json create mode 100755 test/json/fixtures/minefield/y_structure_lonely_negative_real.json create mode 100644 test/json/fixtures/minefield/y_structure_lonely_null.json create mode 100755 test/json/fixtures/minefield/y_structure_lonely_string.json create mode 100755 test/json/fixtures/minefield/y_structure_lonely_true.json create mode 100644 test/json/fixtures/minefield/y_structure_string_empty.json create mode 100644 test/json/fixtures/minefield/y_structure_trailing_newline.json create mode 100644 test/json/fixtures/minefield/y_structure_true_in_array.json create mode 100644 test/json/fixtures/minefield/y_structure_whitespace_array.json create mode 100644 test/json/json_minefield_parser_test.rb diff --git a/test/json/fixtures/minefield/i_number_double_huge_neg_exp.json b/test/json/fixtures/minefield/i_number_double_huge_neg_exp.json new file mode 100644 index 000000000..ae4c7b71f --- /dev/null +++ b/test/json/fixtures/minefield/i_number_double_huge_neg_exp.json @@ -0,0 +1 @@ +[123.456e-789] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_huge_exp.json b/test/json/fixtures/minefield/i_number_huge_exp.json new file mode 100644 index 000000000..9b5efa236 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_huge_exp.json @@ -0,0 +1 @@ +[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_neg_int_huge_exp.json b/test/json/fixtures/minefield/i_number_neg_int_huge_exp.json new file mode 100755 index 000000000..3abd58a5c --- /dev/null +++ b/test/json/fixtures/minefield/i_number_neg_int_huge_exp.json @@ -0,0 +1 @@ +[-1e+9999] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_pos_double_huge_exp.json b/test/json/fixtures/minefield/i_number_pos_double_huge_exp.json new file mode 100755 index 000000000..e10a7eb62 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_pos_double_huge_exp.json @@ -0,0 +1 @@ +[1.5e+9999] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_real_neg_overflow.json b/test/json/fixtures/minefield/i_number_real_neg_overflow.json new file mode 100644 index 000000000..3d628a994 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_real_neg_overflow.json @@ -0,0 +1 @@ +[-123123e100000] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_real_pos_overflow.json b/test/json/fixtures/minefield/i_number_real_pos_overflow.json new file mode 100644 index 000000000..54d7d3dcd --- /dev/null +++ b/test/json/fixtures/minefield/i_number_real_pos_overflow.json @@ -0,0 +1 @@ +[123123e100000] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_real_underflow.json b/test/json/fixtures/minefield/i_number_real_underflow.json new file mode 100644 index 000000000..c5236eb26 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_real_underflow.json @@ -0,0 +1 @@ +[123e-10000000] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_too_big_neg_int.json b/test/json/fixtures/minefield/i_number_too_big_neg_int.json new file mode 100644 index 000000000..dfa384619 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_too_big_neg_int.json @@ -0,0 +1 @@ +[-123123123123123123123123123123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_too_big_pos_int.json b/test/json/fixtures/minefield/i_number_too_big_pos_int.json new file mode 100644 index 000000000..338a8c3c0 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_too_big_pos_int.json @@ -0,0 +1 @@ +[100000000000000000000] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_number_very_big_negative_int.json b/test/json/fixtures/minefield/i_number_very_big_negative_int.json new file mode 100755 index 000000000..e2d9738c2 --- /dev/null +++ b/test/json/fixtures/minefield/i_number_very_big_negative_int.json @@ -0,0 +1 @@ +[-237462374673276894279832749832423479823246327846] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_object_key_lone_2nd_surrogate.json b/test/json/fixtures/minefield/i_object_key_lone_2nd_surrogate.json new file mode 100644 index 000000000..5be7ebaf9 --- /dev/null +++ b/test/json/fixtures/minefield/i_object_key_lone_2nd_surrogate.json @@ -0,0 +1 @@ +{"\uDFAA":0} \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_1st_surrogate_but_2nd_missing.json b/test/json/fixtures/minefield/i_string_1st_surrogate_but_2nd_missing.json new file mode 100644 index 000000000..3b9e37c67 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_1st_surrogate_but_2nd_missing.json @@ -0,0 +1 @@ +["\uDADA"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_1st_valid_surrogate_2nd_invalid.json b/test/json/fixtures/minefield/i_string_1st_valid_surrogate_2nd_invalid.json new file mode 100644 index 000000000..487592832 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_1st_valid_surrogate_2nd_invalid.json @@ -0,0 +1 @@ +["\uD888\u1234"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_UTF-16LE_with_BOM.json b/test/json/fixtures/minefield/i_string_UTF-16LE_with_BOM.json new file mode 100644 index 0000000000000000000000000000000000000000..2a79c0629a49133d8f715bddbef19cfb3ef025bd GIT binary patch literal 12 ScmezWFPcG#;Uy5qG5`P~Km+3d literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json b/test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json new file mode 100755 index 000000000..e2a968a15 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json @@ -0,0 +1 @@ +["日ш"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_UTF8_surrogate_U+D800.json b/test/json/fixtures/minefield/i_string_UTF8_surrogate_U+D800.json new file mode 100644 index 000000000..916bff920 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_UTF8_surrogate_U+D800.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json b/test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json new file mode 100755 index 000000000..3cb11d229 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json @@ -0,0 +1 @@ +["\uD800\n"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json b/test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json new file mode 100755 index 000000000..38ec23bb0 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json @@ -0,0 +1 @@ +["\uDd1ea"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json b/test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json new file mode 100755 index 000000000..c9cd6f6c3 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json @@ -0,0 +1 @@ +["\uD800\uD800\n"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json b/test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json new file mode 100755 index 000000000..3abbd8d8d --- /dev/null +++ b/test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json @@ -0,0 +1 @@ +["\ud800"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_invalid_surrogate.json b/test/json/fixtures/minefield/i_string_invalid_surrogate.json new file mode 100755 index 000000000..ffddc04f5 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_invalid_surrogate.json @@ -0,0 +1 @@ +["\ud800abc"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_invalid_utf-8.json b/test/json/fixtures/minefield/i_string_invalid_utf-8.json new file mode 100644 index 000000000..8e45a7eca --- /dev/null +++ b/test/json/fixtures/minefield/i_string_invalid_utf-8.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json b/test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json new file mode 100755 index 000000000..0d5456cc3 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json @@ -0,0 +1 @@ +["\uDd1e\uD834"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_iso_latin_1.json b/test/json/fixtures/minefield/i_string_iso_latin_1.json new file mode 100644 index 000000000..9389c9823 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_iso_latin_1.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_lone_second_surrogate.json b/test/json/fixtures/minefield/i_string_lone_second_surrogate.json new file mode 100644 index 000000000..1dbd397f3 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_lone_second_surrogate.json @@ -0,0 +1 @@ +["\uDFAA"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_lone_utf8_continuation_byte.json b/test/json/fixtures/minefield/i_string_lone_utf8_continuation_byte.json new file mode 100644 index 000000000..729337c0a --- /dev/null +++ b/test/json/fixtures/minefield/i_string_lone_utf8_continuation_byte.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_not_in_unicode_range.json b/test/json/fixtures/minefield/i_string_not_in_unicode_range.json new file mode 100644 index 000000000..df90a2916 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_not_in_unicode_range.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_overlong_sequence_2_bytes.json b/test/json/fixtures/minefield/i_string_overlong_sequence_2_bytes.json new file mode 100644 index 000000000..c8cee5e0a --- /dev/null +++ b/test/json/fixtures/minefield/i_string_overlong_sequence_2_bytes.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json new file mode 100755 index 000000000..9a91da791 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json new file mode 100755 index 000000000..d24fffdd9 --- /dev/null +++ b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_truncated-utf-8.json b/test/json/fixtures/minefield/i_string_truncated-utf-8.json new file mode 100644 index 000000000..63c7777fb --- /dev/null +++ b/test/json/fixtures/minefield/i_string_truncated-utf-8.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_string_utf16BE_no_BOM.json b/test/json/fixtures/minefield/i_string_utf16BE_no_BOM.json new file mode 100644 index 0000000000000000000000000000000000000000..57e5392ff6309134268c7e3ec2a9289b99ff7148 GIT binary patch literal 10 PcmZRGW>8{y3B<7g33~zN literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/i_string_utf16LE_no_BOM.json b/test/json/fixtures/minefield/i_string_utf16LE_no_BOM.json new file mode 100644 index 0000000000000000000000000000000000000000..c49c1b25d8e5819591993d4d45e4649a46c1b3e0 GIT binary patch literal 10 Pcma!MP-1uq#IXzj3t$1} literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/i_structure_500_nested_arrays.json b/test/json/fixtures/minefield/i_structure_500_nested_arrays.json new file mode 100644 index 000000000..711840589 --- /dev/null +++ b/test/json/fixtures/minefield/i_structure_500_nested_arrays.json @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json b/test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json new file mode 100755 index 000000000..22fdca1b2 --- /dev/null +++ b/test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_1_true_without_comma.json b/test/json/fixtures/minefield/n_array_1_true_without_comma.json new file mode 100644 index 000000000..c14e3f6b1 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_1_true_without_comma.json @@ -0,0 +1 @@ +[1 true] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_a_invalid_utf8.json b/test/json/fixtures/minefield/n_array_a_invalid_utf8.json new file mode 100644 index 000000000..38a86e2e6 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_a_invalid_utf8.json @@ -0,0 +1 @@ +[a] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_colon_instead_of_comma.json b/test/json/fixtures/minefield/n_array_colon_instead_of_comma.json new file mode 100644 index 000000000..0d02ad448 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_colon_instead_of_comma.json @@ -0,0 +1 @@ +["": 1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_comma_after_close.json b/test/json/fixtures/minefield/n_array_comma_after_close.json new file mode 100644 index 000000000..2ccba8d95 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_comma_after_close.json @@ -0,0 +1 @@ +[""], \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_comma_and_number.json b/test/json/fixtures/minefield/n_array_comma_and_number.json new file mode 100755 index 000000000..d2c84e374 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_comma_and_number.json @@ -0,0 +1 @@ +[,1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_double_comma.json b/test/json/fixtures/minefield/n_array_double_comma.json new file mode 100755 index 000000000..0431712bc --- /dev/null +++ b/test/json/fixtures/minefield/n_array_double_comma.json @@ -0,0 +1 @@ +[1,,2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_double_extra_comma.json b/test/json/fixtures/minefield/n_array_double_extra_comma.json new file mode 100644 index 000000000..3f01d3129 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_double_extra_comma.json @@ -0,0 +1 @@ +["x",,] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_extra_close.json b/test/json/fixtures/minefield/n_array_extra_close.json new file mode 100644 index 000000000..c12f9fae1 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_extra_close.json @@ -0,0 +1 @@ +["x"]] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_extra_comma.json b/test/json/fixtures/minefield/n_array_extra_comma.json new file mode 100644 index 000000000..5f8ce18e4 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_extra_comma.json @@ -0,0 +1 @@ +["",] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_incomplete.json b/test/json/fixtures/minefield/n_array_incomplete.json new file mode 100644 index 000000000..cc65b0b51 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_incomplete.json @@ -0,0 +1 @@ +["x" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_incomplete_invalid_value.json b/test/json/fixtures/minefield/n_array_incomplete_invalid_value.json new file mode 100644 index 000000000..c21a8f6cf --- /dev/null +++ b/test/json/fixtures/minefield/n_array_incomplete_invalid_value.json @@ -0,0 +1 @@ +[x \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_inner_array_no_comma.json b/test/json/fixtures/minefield/n_array_inner_array_no_comma.json new file mode 100644 index 000000000..c70b71647 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_inner_array_no_comma.json @@ -0,0 +1 @@ +[3[4]] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_invalid_utf8.json b/test/json/fixtures/minefield/n_array_invalid_utf8.json new file mode 100644 index 000000000..6099d3441 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_invalid_utf8.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json b/test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json new file mode 100755 index 000000000..d4bd7314c --- /dev/null +++ b/test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json @@ -0,0 +1 @@ +[1:2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_just_comma.json b/test/json/fixtures/minefield/n_array_just_comma.json new file mode 100755 index 000000000..9d7077c68 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_just_comma.json @@ -0,0 +1 @@ +[,] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_just_minus.json b/test/json/fixtures/minefield/n_array_just_minus.json new file mode 100755 index 000000000..29501c6ca --- /dev/null +++ b/test/json/fixtures/minefield/n_array_just_minus.json @@ -0,0 +1 @@ +[-] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_missing_value.json b/test/json/fixtures/minefield/n_array_missing_value.json new file mode 100644 index 000000000..3a6ba86f3 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_missing_value.json @@ -0,0 +1 @@ +[ , ""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_newlines_unclosed.json b/test/json/fixtures/minefield/n_array_newlines_unclosed.json new file mode 100644 index 000000000..646680065 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_newlines_unclosed.json @@ -0,0 +1,3 @@ +["a", +4 +,1, \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_number_and_comma.json b/test/json/fixtures/minefield/n_array_number_and_comma.json new file mode 100755 index 000000000..13f6f1d18 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_number_and_comma.json @@ -0,0 +1 @@ +[1,] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_number_and_several_commas.json b/test/json/fixtures/minefield/n_array_number_and_several_commas.json new file mode 100755 index 000000000..0ac408cb8 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_number_and_several_commas.json @@ -0,0 +1 @@ +[1,,] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json b/test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json new file mode 100755 index 000000000..6cd7cf585 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json @@ -0,0 +1 @@ +[" a"\f] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_star_inside.json b/test/json/fixtures/minefield/n_array_star_inside.json new file mode 100755 index 000000000..5a5194647 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_star_inside.json @@ -0,0 +1 @@ +[*] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_unclosed.json b/test/json/fixtures/minefield/n_array_unclosed.json new file mode 100644 index 000000000..060733059 --- /dev/null +++ b/test/json/fixtures/minefield/n_array_unclosed.json @@ -0,0 +1 @@ +["" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_unclosed_trailing_comma.json b/test/json/fixtures/minefield/n_array_unclosed_trailing_comma.json new file mode 100644 index 000000000..6604698ff --- /dev/null +++ b/test/json/fixtures/minefield/n_array_unclosed_trailing_comma.json @@ -0,0 +1 @@ +[1, \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_unclosed_with_new_lines.json b/test/json/fixtures/minefield/n_array_unclosed_with_new_lines.json new file mode 100644 index 000000000..4f61de3fb --- /dev/null +++ b/test/json/fixtures/minefield/n_array_unclosed_with_new_lines.json @@ -0,0 +1,3 @@ +[1, +1 +,1 \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_array_unclosed_with_object_inside.json b/test/json/fixtures/minefield/n_array_unclosed_with_object_inside.json new file mode 100644 index 000000000..043a87e2d --- /dev/null +++ b/test/json/fixtures/minefield/n_array_unclosed_with_object_inside.json @@ -0,0 +1 @@ +[{} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_incomplete_false.json b/test/json/fixtures/minefield/n_incomplete_false.json new file mode 100644 index 000000000..eb18c6a14 --- /dev/null +++ b/test/json/fixtures/minefield/n_incomplete_false.json @@ -0,0 +1 @@ +[fals] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_incomplete_null.json b/test/json/fixtures/minefield/n_incomplete_null.json new file mode 100644 index 000000000..c18ef5385 --- /dev/null +++ b/test/json/fixtures/minefield/n_incomplete_null.json @@ -0,0 +1 @@ +[nul] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_incomplete_true.json b/test/json/fixtures/minefield/n_incomplete_true.json new file mode 100644 index 000000000..f451ac6d2 --- /dev/null +++ b/test/json/fixtures/minefield/n_incomplete_true.json @@ -0,0 +1 @@ +[tru] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_multidigit_number_then_00.json b/test/json/fixtures/minefield/n_multidigit_number_then_00.json new file mode 100644 index 0000000000000000000000000000000000000000..c22507b864f7f089c91c1eb85b1b15cc63b943a6 GIT binary patch literal 4 LcmXpsGG+h(0mJ~8 literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/n_number_++.json b/test/json/fixtures/minefield/n_number_++.json new file mode 100644 index 000000000..bdb62aaf4 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_++.json @@ -0,0 +1 @@ +[++1234] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_+1.json b/test/json/fixtures/minefield/n_number_+1.json new file mode 100755 index 000000000..3cbe58c92 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_+1.json @@ -0,0 +1 @@ +[+1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_+Inf.json b/test/json/fixtures/minefield/n_number_+Inf.json new file mode 100755 index 000000000..871ae14d5 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_+Inf.json @@ -0,0 +1 @@ +[+Inf] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_-01.json b/test/json/fixtures/minefield/n_number_-01.json new file mode 100755 index 000000000..0df32bac8 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_-01.json @@ -0,0 +1 @@ +[-01] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_-1.0..json b/test/json/fixtures/minefield/n_number_-1.0..json new file mode 100755 index 000000000..7cf55a85a --- /dev/null +++ b/test/json/fixtures/minefield/n_number_-1.0..json @@ -0,0 +1 @@ +[-1.0.] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_-2..json b/test/json/fixtures/minefield/n_number_-2..json new file mode 100755 index 000000000..9be84365d --- /dev/null +++ b/test/json/fixtures/minefield/n_number_-2..json @@ -0,0 +1 @@ +[-2.] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_-NaN.json b/test/json/fixtures/minefield/n_number_-NaN.json new file mode 100755 index 000000000..f61615d40 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_-NaN.json @@ -0,0 +1 @@ +[-NaN] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_.-1.json b/test/json/fixtures/minefield/n_number_.-1.json new file mode 100644 index 000000000..1c9f2dd1b --- /dev/null +++ b/test/json/fixtures/minefield/n_number_.-1.json @@ -0,0 +1 @@ +[.-1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_.2e-3.json b/test/json/fixtures/minefield/n_number_.2e-3.json new file mode 100755 index 000000000..c6c976f25 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_.2e-3.json @@ -0,0 +1 @@ +[.2e-3] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0.1.2.json b/test/json/fixtures/minefield/n_number_0.1.2.json new file mode 100755 index 000000000..c83a25621 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0.1.2.json @@ -0,0 +1 @@ +[0.1.2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0.3e+.json b/test/json/fixtures/minefield/n_number_0.3e+.json new file mode 100644 index 000000000..a55a1bfef --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0.3e+.json @@ -0,0 +1 @@ +[0.3e+] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0.3e.json b/test/json/fixtures/minefield/n_number_0.3e.json new file mode 100644 index 000000000..3dd5df4b3 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0.3e.json @@ -0,0 +1 @@ +[0.3e] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0.e1.json b/test/json/fixtures/minefield/n_number_0.e1.json new file mode 100644 index 000000000..c92c71ccb --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0.e1.json @@ -0,0 +1 @@ +[0.e1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0_capital_E+.json b/test/json/fixtures/minefield/n_number_0_capital_E+.json new file mode 100644 index 000000000..3ba2c7d6d --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0_capital_E+.json @@ -0,0 +1 @@ +[0E+] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0_capital_E.json b/test/json/fixtures/minefield/n_number_0_capital_E.json new file mode 100755 index 000000000..5301840d1 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0_capital_E.json @@ -0,0 +1 @@ +[0E] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0e+.json b/test/json/fixtures/minefield/n_number_0e+.json new file mode 100644 index 000000000..8ab0bc4b8 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0e+.json @@ -0,0 +1 @@ +[0e+] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_0e.json b/test/json/fixtures/minefield/n_number_0e.json new file mode 100644 index 000000000..47ec421bb --- /dev/null +++ b/test/json/fixtures/minefield/n_number_0e.json @@ -0,0 +1 @@ +[0e] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_1.0e+.json b/test/json/fixtures/minefield/n_number_1.0e+.json new file mode 100755 index 000000000..cd84b9f69 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_1.0e+.json @@ -0,0 +1 @@ +[1.0e+] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_1.0e-.json b/test/json/fixtures/minefield/n_number_1.0e-.json new file mode 100755 index 000000000..4eb7afa0f --- /dev/null +++ b/test/json/fixtures/minefield/n_number_1.0e-.json @@ -0,0 +1 @@ +[1.0e-] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_1.0e.json b/test/json/fixtures/minefield/n_number_1.0e.json new file mode 100755 index 000000000..21753f4c7 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_1.0e.json @@ -0,0 +1 @@ +[1.0e] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_1_000.json b/test/json/fixtures/minefield/n_number_1_000.json new file mode 100755 index 000000000..7b18b66b3 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_1_000.json @@ -0,0 +1 @@ +[1 000.0] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_1eE2.json b/test/json/fixtures/minefield/n_number_1eE2.json new file mode 100755 index 000000000..4318a341d --- /dev/null +++ b/test/json/fixtures/minefield/n_number_1eE2.json @@ -0,0 +1 @@ +[1eE2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_2.e+3.json b/test/json/fixtures/minefield/n_number_2.e+3.json new file mode 100755 index 000000000..4442f394d --- /dev/null +++ b/test/json/fixtures/minefield/n_number_2.e+3.json @@ -0,0 +1 @@ +[2.e+3] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_2.e-3.json b/test/json/fixtures/minefield/n_number_2.e-3.json new file mode 100755 index 000000000..a65060edf --- /dev/null +++ b/test/json/fixtures/minefield/n_number_2.e-3.json @@ -0,0 +1 @@ +[2.e-3] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_2.e3.json b/test/json/fixtures/minefield/n_number_2.e3.json new file mode 100755 index 000000000..66f7cf701 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_2.e3.json @@ -0,0 +1 @@ +[2.e3] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_9.e+.json b/test/json/fixtures/minefield/n_number_9.e+.json new file mode 100644 index 000000000..732a7b11c --- /dev/null +++ b/test/json/fixtures/minefield/n_number_9.e+.json @@ -0,0 +1 @@ +[9.e+] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_Inf.json b/test/json/fixtures/minefield/n_number_Inf.json new file mode 100755 index 000000000..c40c734c3 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_Inf.json @@ -0,0 +1 @@ +[Inf] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_NaN.json b/test/json/fixtures/minefield/n_number_NaN.json new file mode 100755 index 000000000..499231790 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_NaN.json @@ -0,0 +1 @@ +[NaN] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_U+FF11_fullwidth_digit_one.json b/test/json/fixtures/minefield/n_number_U+FF11_fullwidth_digit_one.json new file mode 100644 index 000000000..b14587e5e --- /dev/null +++ b/test/json/fixtures/minefield/n_number_U+FF11_fullwidth_digit_one.json @@ -0,0 +1 @@ +[１] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_expression.json b/test/json/fixtures/minefield/n_number_expression.json new file mode 100644 index 000000000..76fdbc8a4 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_expression.json @@ -0,0 +1 @@ +[1+2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_hex_1_digit.json b/test/json/fixtures/minefield/n_number_hex_1_digit.json new file mode 100644 index 000000000..3b214880c --- /dev/null +++ b/test/json/fixtures/minefield/n_number_hex_1_digit.json @@ -0,0 +1 @@ +[0x1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_hex_2_digits.json b/test/json/fixtures/minefield/n_number_hex_2_digits.json new file mode 100644 index 000000000..83e516ab0 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_hex_2_digits.json @@ -0,0 +1 @@ +[0x42] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_infinity.json b/test/json/fixtures/minefield/n_number_infinity.json new file mode 100755 index 000000000..8c2baf783 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_infinity.json @@ -0,0 +1 @@ +[Infinity] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_invalid+-.json b/test/json/fixtures/minefield/n_number_invalid+-.json new file mode 100644 index 000000000..1cce602b5 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_invalid+-.json @@ -0,0 +1 @@ +[0e+-1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_invalid-negative-real.json b/test/json/fixtures/minefield/n_number_invalid-negative-real.json new file mode 100644 index 000000000..5fc3c1efb --- /dev/null +++ b/test/json/fixtures/minefield/n_number_invalid-negative-real.json @@ -0,0 +1 @@ +[-123.123foo] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_invalid-utf-8-in-bigger-int.json b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-bigger-int.json new file mode 100644 index 000000000..3b97e580e --- /dev/null +++ b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-bigger-int.json @@ -0,0 +1 @@ +[123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_invalid-utf-8-in-exponent.json b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-exponent.json new file mode 100644 index 000000000..ea35d723c --- /dev/null +++ b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-exponent.json @@ -0,0 +1 @@ +[1e1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_invalid-utf-8-in-int.json b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-int.json new file mode 100644 index 000000000..371226e4c --- /dev/null +++ b/test/json/fixtures/minefield/n_number_invalid-utf-8-in-int.json @@ -0,0 +1 @@ +[0] diff --git a/test/json/fixtures/minefield/n_number_minus_infinity.json b/test/json/fixtures/minefield/n_number_minus_infinity.json new file mode 100755 index 000000000..cf4133d22 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_minus_infinity.json @@ -0,0 +1 @@ +[-Infinity] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_minus_sign_with_trailing_garbage.json b/test/json/fixtures/minefield/n_number_minus_sign_with_trailing_garbage.json new file mode 100644 index 000000000..a6d8e78e7 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_minus_sign_with_trailing_garbage.json @@ -0,0 +1 @@ +[-foo] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_minus_space_1.json b/test/json/fixtures/minefield/n_number_minus_space_1.json new file mode 100644 index 000000000..9a5ebedf6 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_minus_space_1.json @@ -0,0 +1 @@ +[- 1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_neg_int_starting_with_zero.json b/test/json/fixtures/minefield/n_number_neg_int_starting_with_zero.json new file mode 100644 index 000000000..67af0960a --- /dev/null +++ b/test/json/fixtures/minefield/n_number_neg_int_starting_with_zero.json @@ -0,0 +1 @@ +[-012] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_neg_real_without_int_part.json b/test/json/fixtures/minefield/n_number_neg_real_without_int_part.json new file mode 100755 index 000000000..1f2a43496 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_neg_real_without_int_part.json @@ -0,0 +1 @@ +[-.123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_neg_with_garbage_at_end.json b/test/json/fixtures/minefield/n_number_neg_with_garbage_at_end.json new file mode 100644 index 000000000..2aa73119f --- /dev/null +++ b/test/json/fixtures/minefield/n_number_neg_with_garbage_at_end.json @@ -0,0 +1 @@ +[-1x] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_real_garbage_after_e.json b/test/json/fixtures/minefield/n_number_real_garbage_after_e.json new file mode 100644 index 000000000..9213dfca8 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_real_garbage_after_e.json @@ -0,0 +1 @@ +[1ea] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_real_with_invalid_utf8_after_e.json b/test/json/fixtures/minefield/n_number_real_with_invalid_utf8_after_e.json new file mode 100644 index 000000000..1e52ef964 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_real_with_invalid_utf8_after_e.json @@ -0,0 +1 @@ +[1e] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_real_without_fractional_part.json b/test/json/fixtures/minefield/n_number_real_without_fractional_part.json new file mode 100755 index 000000000..1de287cf8 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_real_without_fractional_part.json @@ -0,0 +1 @@ +[1.] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_starting_with_dot.json b/test/json/fixtures/minefield/n_number_starting_with_dot.json new file mode 100755 index 000000000..f682dbdce --- /dev/null +++ b/test/json/fixtures/minefield/n_number_starting_with_dot.json @@ -0,0 +1 @@ +[.123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_with_alpha.json b/test/json/fixtures/minefield/n_number_with_alpha.json new file mode 100644 index 000000000..1e42d8182 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_with_alpha.json @@ -0,0 +1 @@ +[1.2a-3] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_with_alpha_char.json b/test/json/fixtures/minefield/n_number_with_alpha_char.json new file mode 100644 index 000000000..b79daccb8 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_with_alpha_char.json @@ -0,0 +1 @@ +[1.8011670033376514H-308] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_number_with_leading_zero.json b/test/json/fixtures/minefield/n_number_with_leading_zero.json new file mode 100755 index 000000000..7106da1f3 --- /dev/null +++ b/test/json/fixtures/minefield/n_number_with_leading_zero.json @@ -0,0 +1 @@ +[012] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_bad_value.json b/test/json/fixtures/minefield/n_object_bad_value.json new file mode 100644 index 000000000..a03a8c03b --- /dev/null +++ b/test/json/fixtures/minefield/n_object_bad_value.json @@ -0,0 +1 @@ +["x", truth] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_bracket_key.json b/test/json/fixtures/minefield/n_object_bracket_key.json new file mode 100644 index 000000000..cc443b483 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_bracket_key.json @@ -0,0 +1 @@ +{[: "x"} diff --git a/test/json/fixtures/minefield/n_object_comma_instead_of_colon.json b/test/json/fixtures/minefield/n_object_comma_instead_of_colon.json new file mode 100644 index 000000000..8d5637708 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_comma_instead_of_colon.json @@ -0,0 +1 @@ +{"x", null} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_double_colon.json b/test/json/fixtures/minefield/n_object_double_colon.json new file mode 100644 index 000000000..80e8c7b89 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_double_colon.json @@ -0,0 +1 @@ +{"x"::"b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_emoji.json b/test/json/fixtures/minefield/n_object_emoji.json new file mode 100644 index 000000000..cb4078eaa --- /dev/null +++ b/test/json/fixtures/minefield/n_object_emoji.json @@ -0,0 +1 @@ +{🇨🇭} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_garbage_at_end.json b/test/json/fixtures/minefield/n_object_garbage_at_end.json new file mode 100644 index 000000000..80c42cbad --- /dev/null +++ b/test/json/fixtures/minefield/n_object_garbage_at_end.json @@ -0,0 +1 @@ +{"a":"a" 123} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_key_with_single_quotes.json b/test/json/fixtures/minefield/n_object_key_with_single_quotes.json new file mode 100755 index 000000000..77c327599 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_key_with_single_quotes.json @@ -0,0 +1 @@ +{key: 'value'} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_lone_continuation_byte_in_key_and_trailing_comma.json b/test/json/fixtures/minefield/n_object_lone_continuation_byte_in_key_and_trailing_comma.json new file mode 100644 index 000000000..aa2cb637c --- /dev/null +++ b/test/json/fixtures/minefield/n_object_lone_continuation_byte_in_key_and_trailing_comma.json @@ -0,0 +1 @@ +{"":"0",} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_missing_colon.json b/test/json/fixtures/minefield/n_object_missing_colon.json new file mode 100644 index 000000000..b98eff62d --- /dev/null +++ b/test/json/fixtures/minefield/n_object_missing_colon.json @@ -0,0 +1 @@ +{"a" b} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_missing_key.json b/test/json/fixtures/minefield/n_object_missing_key.json new file mode 100755 index 000000000..b4fb0f528 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_missing_key.json @@ -0,0 +1 @@ +{:"b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_missing_semicolon.json b/test/json/fixtures/minefield/n_object_missing_semicolon.json new file mode 100755 index 000000000..e3451384f --- /dev/null +++ b/test/json/fixtures/minefield/n_object_missing_semicolon.json @@ -0,0 +1 @@ +{"a" "b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_missing_value.json b/test/json/fixtures/minefield/n_object_missing_value.json new file mode 100644 index 000000000..3ef538a60 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_missing_value.json @@ -0,0 +1 @@ +{"a": \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_no-colon.json b/test/json/fixtures/minefield/n_object_no-colon.json new file mode 100644 index 000000000..f3797b357 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_no-colon.json @@ -0,0 +1 @@ +{"a" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_non_string_key.json b/test/json/fixtures/minefield/n_object_non_string_key.json new file mode 100755 index 000000000..b9945b34b --- /dev/null +++ b/test/json/fixtures/minefield/n_object_non_string_key.json @@ -0,0 +1 @@ +{1:1} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json b/test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json new file mode 100755 index 000000000..b37fa86c0 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json @@ -0,0 +1 @@ +{9999E9999:1} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_repeated_null_null.json b/test/json/fixtures/minefield/n_object_repeated_null_null.json new file mode 100755 index 000000000..f7d2959d0 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_repeated_null_null.json @@ -0,0 +1 @@ +{null:null,null:null} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_several_trailing_commas.json b/test/json/fixtures/minefield/n_object_several_trailing_commas.json new file mode 100755 index 000000000..3c9afe8dc --- /dev/null +++ b/test/json/fixtures/minefield/n_object_several_trailing_commas.json @@ -0,0 +1 @@ +{"id":0,,,,,} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_single_quote.json b/test/json/fixtures/minefield/n_object_single_quote.json new file mode 100644 index 000000000..e5cdf976a --- /dev/null +++ b/test/json/fixtures/minefield/n_object_single_quote.json @@ -0,0 +1 @@ +{'a':0} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_trailing_comma.json b/test/json/fixtures/minefield/n_object_trailing_comma.json new file mode 100755 index 000000000..a4b025094 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_trailing_comma.json @@ -0,0 +1 @@ +{"id":0,} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_trailing_comment.json b/test/json/fixtures/minefield/n_object_trailing_comment.json new file mode 100644 index 000000000..a372c6553 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_trailing_comment.json @@ -0,0 +1 @@ +{"a":"b"}/**/ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_trailing_comment_open.json b/test/json/fixtures/minefield/n_object_trailing_comment_open.json new file mode 100644 index 000000000..d557f41ca --- /dev/null +++ b/test/json/fixtures/minefield/n_object_trailing_comment_open.json @@ -0,0 +1 @@ +{"a":"b"}/**// \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_trailing_comment_slash_open.json b/test/json/fixtures/minefield/n_object_trailing_comment_slash_open.json new file mode 100644 index 000000000..e335136c0 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_trailing_comment_slash_open.json @@ -0,0 +1 @@ +{"a":"b"}// \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_trailing_comment_slash_open_incomplete.json b/test/json/fixtures/minefield/n_object_trailing_comment_slash_open_incomplete.json new file mode 100644 index 000000000..d892e49f1 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_trailing_comment_slash_open_incomplete.json @@ -0,0 +1 @@ +{"a":"b"}/ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_two_commas_in_a_row.json b/test/json/fixtures/minefield/n_object_two_commas_in_a_row.json new file mode 100755 index 000000000..7c639ae64 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_two_commas_in_a_row.json @@ -0,0 +1 @@ +{"a":"b",,"c":"d"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_unquoted_key.json b/test/json/fixtures/minefield/n_object_unquoted_key.json new file mode 100644 index 000000000..8ba137293 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_unquoted_key.json @@ -0,0 +1 @@ +{a: "b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_unterminated-value.json b/test/json/fixtures/minefield/n_object_unterminated-value.json new file mode 100644 index 000000000..7fe699a6a --- /dev/null +++ b/test/json/fixtures/minefield/n_object_unterminated-value.json @@ -0,0 +1 @@ +{"a":"a \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_with_single_string.json b/test/json/fixtures/minefield/n_object_with_single_string.json new file mode 100644 index 000000000..d63f7fbb7 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_with_single_string.json @@ -0,0 +1 @@ +{ "foo" : "bar", "a" } \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_object_with_trailing_garbage.json b/test/json/fixtures/minefield/n_object_with_trailing_garbage.json new file mode 100644 index 000000000..787c8f0a8 --- /dev/null +++ b/test/json/fixtures/minefield/n_object_with_trailing_garbage.json @@ -0,0 +1 @@ +{"a":"b"}# \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_single_space.json b/test/json/fixtures/minefield/n_single_space.json new file mode 100755 index 000000000..0519ecba6 --- /dev/null +++ b/test/json/fixtures/minefield/n_single_space.json @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_1_surrogate_then_escape.json b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape.json new file mode 100644 index 000000000..acec66d8f --- /dev/null +++ b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape.json @@ -0,0 +1 @@ +["\uD800\"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u.json b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u.json new file mode 100644 index 000000000..e834b05e9 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u.json @@ -0,0 +1 @@ +["\uD800\u"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1.json b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1.json new file mode 100644 index 000000000..a04cd3489 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1.json @@ -0,0 +1 @@ +["\uD800\u1"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1x.json b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1x.json new file mode 100644 index 000000000..bfbd23409 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_1_surrogate_then_escape_u1x.json @@ -0,0 +1 @@ +["\uD800\u1x"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_accentuated_char_no_quotes.json b/test/json/fixtures/minefield/n_string_accentuated_char_no_quotes.json new file mode 100644 index 000000000..fd6895693 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_accentuated_char_no_quotes.json @@ -0,0 +1 @@ +[é] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_backslash_00.json b/test/json/fixtures/minefield/n_string_backslash_00.json new file mode 100644 index 0000000000000000000000000000000000000000..b5bf267b5d4ee922d20cec1543b66d1530ff76cf GIT binary patch literal 6 Ncma!6ieXTS1pox&0a*Y5 literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/n_string_escape_x.json b/test/json/fixtures/minefield/n_string_escape_x.json new file mode 100644 index 000000000..fae291938 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_escape_x.json @@ -0,0 +1 @@ +["\x00"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_escaped_backslash_bad.json b/test/json/fixtures/minefield/n_string_escaped_backslash_bad.json new file mode 100755 index 000000000..016fcb47e --- /dev/null +++ b/test/json/fixtures/minefield/n_string_escaped_backslash_bad.json @@ -0,0 +1 @@ +["\\\"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_escaped_ctrl_char_tab.json b/test/json/fixtures/minefield/n_string_escaped_ctrl_char_tab.json new file mode 100644 index 000000000..f35ea382b --- /dev/null +++ b/test/json/fixtures/minefield/n_string_escaped_ctrl_char_tab.json @@ -0,0 +1 @@ +["\ "] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_escaped_emoji.json b/test/json/fixtures/minefield/n_string_escaped_emoji.json new file mode 100644 index 000000000..a27775421 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_escaped_emoji.json @@ -0,0 +1 @@ +["\🌀"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_incomplete_escape.json b/test/json/fixtures/minefield/n_string_incomplete_escape.json new file mode 100755 index 000000000..3415c33ca --- /dev/null +++ b/test/json/fixtures/minefield/n_string_incomplete_escape.json @@ -0,0 +1 @@ +["\"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_incomplete_escaped_character.json b/test/json/fixtures/minefield/n_string_incomplete_escaped_character.json new file mode 100755 index 000000000..0f2197ea2 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_incomplete_escaped_character.json @@ -0,0 +1 @@ +["\u00A"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_incomplete_surrogate.json b/test/json/fixtures/minefield/n_string_incomplete_surrogate.json new file mode 100755 index 000000000..75504a656 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_incomplete_surrogate.json @@ -0,0 +1 @@ +["\uD834\uDd"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json b/test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json new file mode 100755 index 000000000..bd9656060 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json @@ -0,0 +1 @@ +["\uD800\uD800\x"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_invalid-utf-8-in-escape.json b/test/json/fixtures/minefield/n_string_invalid-utf-8-in-escape.json new file mode 100644 index 000000000..0c4300643 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_invalid-utf-8-in-escape.json @@ -0,0 +1 @@ +["\u"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_invalid_backslash_esc.json b/test/json/fixtures/minefield/n_string_invalid_backslash_esc.json new file mode 100755 index 000000000..d1eb60921 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_invalid_backslash_esc.json @@ -0,0 +1 @@ +["\a"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_invalid_unicode_escape.json b/test/json/fixtures/minefield/n_string_invalid_unicode_escape.json new file mode 100644 index 000000000..7608cb6ba --- /dev/null +++ b/test/json/fixtures/minefield/n_string_invalid_unicode_escape.json @@ -0,0 +1 @@ +["\uqqqq"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_invalid_utf8_after_escape.json b/test/json/fixtures/minefield/n_string_invalid_utf8_after_escape.json new file mode 100644 index 000000000..2f757a25b --- /dev/null +++ b/test/json/fixtures/minefield/n_string_invalid_utf8_after_escape.json @@ -0,0 +1 @@ +["\"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json b/test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json new file mode 100755 index 000000000..7b297c636 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json @@ -0,0 +1 @@ +[\u0020"asd"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_no_quotes_with_bad_escape.json b/test/json/fixtures/minefield/n_string_no_quotes_with_bad_escape.json new file mode 100644 index 000000000..01bc70aba --- /dev/null +++ b/test/json/fixtures/minefield/n_string_no_quotes_with_bad_escape.json @@ -0,0 +1 @@ +[\n] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_single_doublequote.json b/test/json/fixtures/minefield/n_string_single_doublequote.json new file mode 100755 index 000000000..9d68933c4 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_single_doublequote.json @@ -0,0 +1 @@ +" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_single_quote.json b/test/json/fixtures/minefield/n_string_single_quote.json new file mode 100644 index 000000000..caff239bf --- /dev/null +++ b/test/json/fixtures/minefield/n_string_single_quote.json @@ -0,0 +1 @@ +['single quote'] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json b/test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json new file mode 100755 index 000000000..f2ba8f84a --- /dev/null +++ b/test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_start_escape_unclosed.json b/test/json/fixtures/minefield/n_string_start_escape_unclosed.json new file mode 100644 index 000000000..db62a46fc --- /dev/null +++ b/test/json/fixtures/minefield/n_string_start_escape_unclosed.json @@ -0,0 +1 @@ +["\ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json b/test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json new file mode 100755 index 0000000000000000000000000000000000000000..9f21348071d3d736bdd469f159cea674c09237af GIT binary patch literal 7 Ocma!6N@Pe>iUj}$`2oKG literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/n_string_unescaped_newline.json b/test/json/fixtures/minefield/n_string_unescaped_newline.json new file mode 100644 index 000000000..700d36086 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_unescaped_newline.json @@ -0,0 +1,2 @@ +["new +line"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_unescaped_tab.json b/test/json/fixtures/minefield/n_string_unescaped_tab.json new file mode 100644 index 000000000..160264a2d --- /dev/null +++ b/test/json/fixtures/minefield/n_string_unescaped_tab.json @@ -0,0 +1 @@ +[" "] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_unicode_CapitalU.json b/test/json/fixtures/minefield/n_string_unicode_CapitalU.json new file mode 100644 index 000000000..17332bb17 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_unicode_CapitalU.json @@ -0,0 +1 @@ +"\UA66D" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_string_with_trailing_garbage.json b/test/json/fixtures/minefield/n_string_with_trailing_garbage.json new file mode 100644 index 000000000..efe3bd272 --- /dev/null +++ b/test/json/fixtures/minefield/n_string_with_trailing_garbage.json @@ -0,0 +1 @@ +""x \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_100000_opening_arrays.json b/test/json/fixtures/minefield/n_structure_100000_opening_arrays.json new file mode 100644 index 000000000..a4823eecc --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_100000_opening_arrays.json @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_U+2060_word_joined.json b/test/json/fixtures/minefield/n_structure_U+2060_word_joined.json new file mode 100644 index 000000000..81156a699 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_U+2060_word_joined.json @@ -0,0 +1 @@ +[⁠] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json b/test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json new file mode 100755 index 000000000..5f282702b --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_angle_bracket_..json b/test/json/fixtures/minefield/n_structure_angle_bracket_..json new file mode 100755 index 000000000..a56fef0b0 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_angle_bracket_..json @@ -0,0 +1 @@ +<.> \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_angle_bracket_null.json b/test/json/fixtures/minefield/n_structure_angle_bracket_null.json new file mode 100755 index 000000000..617f26254 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_angle_bracket_null.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_array_trailing_garbage.json b/test/json/fixtures/minefield/n_structure_array_trailing_garbage.json new file mode 100644 index 000000000..5a745e6f3 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_array_trailing_garbage.json @@ -0,0 +1 @@ +[1]x \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json b/test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json new file mode 100755 index 000000000..6cfb1398d --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json @@ -0,0 +1 @@ +[1]] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json b/test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json new file mode 100755 index 000000000..ba6b1788b --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json @@ -0,0 +1 @@ +["asd] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_ascii-unicode-identifier.json b/test/json/fixtures/minefield/n_structure_ascii-unicode-identifier.json new file mode 100644 index 000000000..ef2ab62fe --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_ascii-unicode-identifier.json @@ -0,0 +1 @@ +aå \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_capitalized_True.json b/test/json/fixtures/minefield/n_structure_capitalized_True.json new file mode 100755 index 000000000..7cd88469a --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_capitalized_True.json @@ -0,0 +1 @@ +[True] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_close_unopened_array.json b/test/json/fixtures/minefield/n_structure_close_unopened_array.json new file mode 100755 index 000000000..d2af0c646 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_close_unopened_array.json @@ -0,0 +1 @@ +1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_comma_instead_of_closing_brace.json b/test/json/fixtures/minefield/n_structure_comma_instead_of_closing_brace.json new file mode 100644 index 000000000..ac61b8200 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_comma_instead_of_closing_brace.json @@ -0,0 +1 @@ +{"x": true, \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_double_array.json b/test/json/fixtures/minefield/n_structure_double_array.json new file mode 100755 index 000000000..058d1626e --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_double_array.json @@ -0,0 +1 @@ +[][] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_end_array.json b/test/json/fixtures/minefield/n_structure_end_array.json new file mode 100644 index 000000000..54caf60b1 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_end_array.json @@ -0,0 +1 @@ +] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json b/test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json new file mode 100755 index 000000000..bfcdd514f --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_lone-invalid-utf-8.json b/test/json/fixtures/minefield/n_structure_lone-invalid-utf-8.json new file mode 100644 index 000000000..8b1296cad --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_lone-invalid-utf-8.json @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_lone-open-bracket.json b/test/json/fixtures/minefield/n_structure_lone-open-bracket.json new file mode 100644 index 000000000..8e2f0bef1 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_lone-open-bracket.json @@ -0,0 +1 @@ +[ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_no_data.json b/test/json/fixtures/minefield/n_structure_no_data.json new file mode 100644 index 000000000..e69de29bb diff --git a/test/json/fixtures/minefield/n_structure_null-byte-outside-string.json b/test/json/fixtures/minefield/n_structure_null-byte-outside-string.json new file mode 100644 index 0000000000000000000000000000000000000000..326db14422a756e9bd39221edf37b844cb8471c7 GIT binary patch literal 3 Kcma!Mhy?%vaR9jh literal 0 HcmV?d00001 diff --git a/test/json/fixtures/minefield/n_structure_number_with_trailing_garbage.json b/test/json/fixtures/minefield/n_structure_number_with_trailing_garbage.json new file mode 100644 index 000000000..0746539d2 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_number_with_trailing_garbage.json @@ -0,0 +1 @@ +2@ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_object_followed_by_closing_object.json b/test/json/fixtures/minefield/n_structure_object_followed_by_closing_object.json new file mode 100644 index 000000000..aa9ebaec5 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_object_followed_by_closing_object.json @@ -0,0 +1 @@ +{}} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_object_unclosed_no_value.json b/test/json/fixtures/minefield/n_structure_object_unclosed_no_value.json new file mode 100644 index 000000000..17d045147 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_object_unclosed_no_value.json @@ -0,0 +1 @@ +{"": \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_object_with_comment.json b/test/json/fixtures/minefield/n_structure_object_with_comment.json new file mode 100644 index 000000000..ed1b569b7 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_object_with_comment.json @@ -0,0 +1 @@ +{"a":/*comment*/"b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_object_with_trailing_garbage.json b/test/json/fixtures/minefield/n_structure_object_with_trailing_garbage.json new file mode 100644 index 000000000..9ca2336d7 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_object_with_trailing_garbage.json @@ -0,0 +1 @@ +{"a": true} "x" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_array_apostrophe.json b/test/json/fixtures/minefield/n_structure_open_array_apostrophe.json new file mode 100644 index 000000000..8bebe3af0 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_apostrophe.json @@ -0,0 +1 @@ +[' \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_array_comma.json b/test/json/fixtures/minefield/n_structure_open_array_comma.json new file mode 100644 index 000000000..6295fdc36 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_comma.json @@ -0,0 +1 @@ +[, \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_array_object.json b/test/json/fixtures/minefield/n_structure_open_array_object.json new file mode 100644 index 000000000..e870445b2 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_object.json @@ -0,0 +1 @@ +[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"": diff --git a/test/json/fixtures/minefield/n_structure_open_array_open_object.json b/test/json/fixtures/minefield/n_structure_open_array_open_object.json new file mode 100644 index 000000000..7a63c8c57 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_open_object.json @@ -0,0 +1 @@ +[{ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_array_open_string.json b/test/json/fixtures/minefield/n_structure_open_array_open_string.json new file mode 100644 index 000000000..9822a6baf --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_open_string.json @@ -0,0 +1 @@ +["a \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_array_string.json b/test/json/fixtures/minefield/n_structure_open_array_string.json new file mode 100644 index 000000000..42a619362 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_array_string.json @@ -0,0 +1 @@ +["a" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object.json b/test/json/fixtures/minefield/n_structure_open_object.json new file mode 100644 index 000000000..81750b96f --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object.json @@ -0,0 +1 @@ +{ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object_close_array.json b/test/json/fixtures/minefield/n_structure_open_object_close_array.json new file mode 100755 index 000000000..eebc700a1 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object_close_array.json @@ -0,0 +1 @@ +{] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object_comma.json b/test/json/fixtures/minefield/n_structure_open_object_comma.json new file mode 100644 index 000000000..47bc9106f --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object_comma.json @@ -0,0 +1 @@ +{, \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object_open_array.json b/test/json/fixtures/minefield/n_structure_open_object_open_array.json new file mode 100644 index 000000000..381ede5de --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object_open_array.json @@ -0,0 +1 @@ +{[ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object_open_string.json b/test/json/fixtures/minefield/n_structure_open_object_open_string.json new file mode 100644 index 000000000..328c30cd7 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object_open_string.json @@ -0,0 +1 @@ +{"a \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_object_string_with_apostrophes.json b/test/json/fixtures/minefield/n_structure_open_object_string_with_apostrophes.json new file mode 100644 index 000000000..9dba17090 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_object_string_with_apostrophes.json @@ -0,0 +1 @@ +{'a' \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_open_open.json b/test/json/fixtures/minefield/n_structure_open_open.json new file mode 100644 index 000000000..841fd5f86 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_open_open.json @@ -0,0 +1 @@ +["\{["\{["\{["\{ \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_single_eacute.json b/test/json/fixtures/minefield/n_structure_single_eacute.json new file mode 100644 index 000000000..92a39f398 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_single_eacute.json @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_single_star.json b/test/json/fixtures/minefield/n_structure_single_star.json new file mode 100755 index 000000000..f59ec20aa --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_single_star.json @@ -0,0 +1 @@ +* \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_trailing_#.json b/test/json/fixtures/minefield/n_structure_trailing_#.json new file mode 100644 index 000000000..898611087 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_trailing_#.json @@ -0,0 +1 @@ +{"a":"b"}#{} \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json b/test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json new file mode 100755 index 000000000..df2f0f242 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json @@ -0,0 +1 @@ +[\u000A""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unclosed_array.json b/test/json/fixtures/minefield/n_structure_unclosed_array.json new file mode 100755 index 000000000..11209515c --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unclosed_array.json @@ -0,0 +1 @@ +[1 \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unclosed_array_partial_null.json b/test/json/fixtures/minefield/n_structure_unclosed_array_partial_null.json new file mode 100644 index 000000000..0d591762c --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unclosed_array_partial_null.json @@ -0,0 +1 @@ +[ false, nul \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_false.json b/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_false.json new file mode 100644 index 000000000..a2ff8504a --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_false.json @@ -0,0 +1 @@ +[ true, fals \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_true.json b/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_true.json new file mode 100644 index 000000000..3149e8f5a --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unclosed_array_unfinished_true.json @@ -0,0 +1 @@ +[ false, tru \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unclosed_object.json b/test/json/fixtures/minefield/n_structure_unclosed_object.json new file mode 100755 index 000000000..694d69dbd --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unclosed_object.json @@ -0,0 +1 @@ +{"asd":"asd" \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_unicode-identifier.json b/test/json/fixtures/minefield/n_structure_unicode-identifier.json new file mode 100644 index 000000000..7284aea33 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_unicode-identifier.json @@ -0,0 +1 @@ +å \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json b/test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json new file mode 100755 index 000000000..81156a699 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json @@ -0,0 +1 @@ +[⁠] \ No newline at end of file diff --git a/test/json/fixtures/minefield/n_structure_whitespace_formfeed.json b/test/json/fixtures/minefield/n_structure_whitespace_formfeed.json new file mode 100755 index 000000000..a9ea535d1 --- /dev/null +++ b/test/json/fixtures/minefield/n_structure_whitespace_formfeed.json @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_arraysWithSpaces.json b/test/json/fixtures/minefield/y_array_arraysWithSpaces.json new file mode 100755 index 000000000..582290798 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_arraysWithSpaces.json @@ -0,0 +1 @@ +[[] ] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_empty-string.json b/test/json/fixtures/minefield/y_array_empty-string.json new file mode 100644 index 000000000..93b6be2bc --- /dev/null +++ b/test/json/fixtures/minefield/y_array_empty-string.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_empty.json b/test/json/fixtures/minefield/y_array_empty.json new file mode 100755 index 000000000..0637a088a --- /dev/null +++ b/test/json/fixtures/minefield/y_array_empty.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_ending_with_newline.json b/test/json/fixtures/minefield/y_array_ending_with_newline.json new file mode 100755 index 000000000..eac5f7b46 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_ending_with_newline.json @@ -0,0 +1 @@ +["a"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_false.json b/test/json/fixtures/minefield/y_array_false.json new file mode 100644 index 000000000..67b2f0760 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_false.json @@ -0,0 +1 @@ +[false] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_heterogeneous.json b/test/json/fixtures/minefield/y_array_heterogeneous.json new file mode 100755 index 000000000..d3c1e2648 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_heterogeneous.json @@ -0,0 +1 @@ +[null, 1, "1", {}] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_null.json b/test/json/fixtures/minefield/y_array_null.json new file mode 100644 index 000000000..500db4a86 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_null.json @@ -0,0 +1 @@ +[null] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_with_1_and_newline.json b/test/json/fixtures/minefield/y_array_with_1_and_newline.json new file mode 100644 index 000000000..994825500 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_with_1_and_newline.json @@ -0,0 +1,2 @@ +[1 +] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_with_leading_space.json b/test/json/fixtures/minefield/y_array_with_leading_space.json new file mode 100755 index 000000000..18bfe6422 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_with_leading_space.json @@ -0,0 +1 @@ + [1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_with_several_null.json b/test/json/fixtures/minefield/y_array_with_several_null.json new file mode 100755 index 000000000..99f6c5d1d --- /dev/null +++ b/test/json/fixtures/minefield/y_array_with_several_null.json @@ -0,0 +1 @@ +[1,null,null,null,2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_array_with_trailing_space.json b/test/json/fixtures/minefield/y_array_with_trailing_space.json new file mode 100755 index 000000000..de9e7a944 --- /dev/null +++ b/test/json/fixtures/minefield/y_array_with_trailing_space.json @@ -0,0 +1 @@ +[2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number.json b/test/json/fixtures/minefield/y_number.json new file mode 100644 index 000000000..e5f5cc334 --- /dev/null +++ b/test/json/fixtures/minefield/y_number.json @@ -0,0 +1 @@ +[123e65] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_0e+1.json b/test/json/fixtures/minefield/y_number_0e+1.json new file mode 100755 index 000000000..d1d396706 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_0e+1.json @@ -0,0 +1 @@ +[0e+1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_0e1.json b/test/json/fixtures/minefield/y_number_0e1.json new file mode 100755 index 000000000..3283a7936 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_0e1.json @@ -0,0 +1 @@ +[0e1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_after_space.json b/test/json/fixtures/minefield/y_number_after_space.json new file mode 100644 index 000000000..623570d96 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_after_space.json @@ -0,0 +1 @@ +[ 4] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_double_close_to_zero.json b/test/json/fixtures/minefield/y_number_double_close_to_zero.json new file mode 100755 index 000000000..96555ff78 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_double_close_to_zero.json @@ -0,0 +1 @@ +[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001] diff --git a/test/json/fixtures/minefield/y_number_int_with_exp.json b/test/json/fixtures/minefield/y_number_int_with_exp.json new file mode 100755 index 000000000..a4ca9e754 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_int_with_exp.json @@ -0,0 +1 @@ +[20e1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_minus_zero.json b/test/json/fixtures/minefield/y_number_minus_zero.json new file mode 100755 index 000000000..37af1312a --- /dev/null +++ b/test/json/fixtures/minefield/y_number_minus_zero.json @@ -0,0 +1 @@ +[-0] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_negative_int.json b/test/json/fixtures/minefield/y_number_negative_int.json new file mode 100644 index 000000000..8e30f8bd9 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_negative_int.json @@ -0,0 +1 @@ +[-123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_negative_one.json b/test/json/fixtures/minefield/y_number_negative_one.json new file mode 100644 index 000000000..99d21a2a0 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_negative_one.json @@ -0,0 +1 @@ +[-1] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_negative_zero.json b/test/json/fixtures/minefield/y_number_negative_zero.json new file mode 100644 index 000000000..37af1312a --- /dev/null +++ b/test/json/fixtures/minefield/y_number_negative_zero.json @@ -0,0 +1 @@ +[-0] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_capital_e.json b/test/json/fixtures/minefield/y_number_real_capital_e.json new file mode 100644 index 000000000..6edbdfcb1 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_capital_e.json @@ -0,0 +1 @@ +[1E22] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_capital_e_neg_exp.json b/test/json/fixtures/minefield/y_number_real_capital_e_neg_exp.json new file mode 100644 index 000000000..0a01bd3ef --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_capital_e_neg_exp.json @@ -0,0 +1 @@ +[1E-2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_capital_e_pos_exp.json b/test/json/fixtures/minefield/y_number_real_capital_e_pos_exp.json new file mode 100644 index 000000000..5a8fc0972 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_capital_e_pos_exp.json @@ -0,0 +1 @@ +[1E+2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_exponent.json b/test/json/fixtures/minefield/y_number_real_exponent.json new file mode 100644 index 000000000..da2522d61 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_exponent.json @@ -0,0 +1 @@ +[123e45] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_fraction_exponent.json b/test/json/fixtures/minefield/y_number_real_fraction_exponent.json new file mode 100644 index 000000000..3944a7a45 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_fraction_exponent.json @@ -0,0 +1 @@ +[123.456e78] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_neg_exp.json b/test/json/fixtures/minefield/y_number_real_neg_exp.json new file mode 100644 index 000000000..ca40d3c25 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_neg_exp.json @@ -0,0 +1 @@ +[1e-2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_real_pos_exponent.json b/test/json/fixtures/minefield/y_number_real_pos_exponent.json new file mode 100644 index 000000000..343601d51 --- /dev/null +++ b/test/json/fixtures/minefield/y_number_real_pos_exponent.json @@ -0,0 +1 @@ +[1e+2] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_simple_int.json b/test/json/fixtures/minefield/y_number_simple_int.json new file mode 100644 index 000000000..e47f69afc --- /dev/null +++ b/test/json/fixtures/minefield/y_number_simple_int.json @@ -0,0 +1 @@ +[123] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_number_simple_real.json b/test/json/fixtures/minefield/y_number_simple_real.json new file mode 100644 index 000000000..b02878e5f --- /dev/null +++ b/test/json/fixtures/minefield/y_number_simple_real.json @@ -0,0 +1 @@ +[123.456789] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object.json b/test/json/fixtures/minefield/y_object.json new file mode 100755 index 000000000..78262eda3 --- /dev/null +++ b/test/json/fixtures/minefield/y_object.json @@ -0,0 +1 @@ +{"asd":"sdf", "dfg":"fgh"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_basic.json b/test/json/fixtures/minefield/y_object_basic.json new file mode 100755 index 000000000..646bbe7bb --- /dev/null +++ b/test/json/fixtures/minefield/y_object_basic.json @@ -0,0 +1 @@ +{"asd":"sdf"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_duplicated_key.json b/test/json/fixtures/minefield/y_object_duplicated_key.json new file mode 100755 index 000000000..bbc2e1ce4 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_duplicated_key.json @@ -0,0 +1 @@ +{"a":"b","a":"c"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_duplicated_key_and_value.json b/test/json/fixtures/minefield/y_object_duplicated_key_and_value.json new file mode 100755 index 000000000..211581c20 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_duplicated_key_and_value.json @@ -0,0 +1 @@ +{"a":"b","a":"b"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_empty.json b/test/json/fixtures/minefield/y_object_empty.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/test/json/fixtures/minefield/y_object_empty.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_empty_key.json b/test/json/fixtures/minefield/y_object_empty_key.json new file mode 100755 index 000000000..c0013d3b8 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_empty_key.json @@ -0,0 +1 @@ +{"":0} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_escaped_null_in_key.json b/test/json/fixtures/minefield/y_object_escaped_null_in_key.json new file mode 100644 index 000000000..593f0f67f --- /dev/null +++ b/test/json/fixtures/minefield/y_object_escaped_null_in_key.json @@ -0,0 +1 @@ +{"foo\u0000bar": 42} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_extreme_numbers.json b/test/json/fixtures/minefield/y_object_extreme_numbers.json new file mode 100644 index 000000000..a0d3531c3 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_extreme_numbers.json @@ -0,0 +1 @@ +{ "min": -1.0e+28, "max": 1.0e+28 } \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_long_strings.json b/test/json/fixtures/minefield/y_object_long_strings.json new file mode 100644 index 000000000..bdc4a0871 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_long_strings.json @@ -0,0 +1 @@ +{"x":[{"id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}], "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_simple.json b/test/json/fixtures/minefield/y_object_simple.json new file mode 100644 index 000000000..dacac917f --- /dev/null +++ b/test/json/fixtures/minefield/y_object_simple.json @@ -0,0 +1 @@ +{"a":[]} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_string_unicode.json b/test/json/fixtures/minefield/y_object_string_unicode.json new file mode 100644 index 000000000..8effdb297 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_string_unicode.json @@ -0,0 +1 @@ +{"title":"\u041f\u043e\u043b\u0442\u043e\u0440\u0430 \u0417\u0435\u043c\u043b\u0435\u043a\u043e\u043f\u0430" } \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_object_with_newlines.json b/test/json/fixtures/minefield/y_object_with_newlines.json new file mode 100644 index 000000000..246ec6b34 --- /dev/null +++ b/test/json/fixtures/minefield/y_object_with_newlines.json @@ -0,0 +1,3 @@ +{ +"a": "b" +} \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json b/test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json new file mode 100755 index 000000000..9967ddeb8 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json @@ -0,0 +1 @@ +["\u0060\u012a\u12AB"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json b/test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json new file mode 100755 index 000000000..996875cc8 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json @@ -0,0 +1 @@ +["\uD801\udc37"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json b/test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json new file mode 100755 index 000000000..3401021ec --- /dev/null +++ b/test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json @@ -0,0 +1 @@ +["\ud83d\ude39\ud83d\udc8d"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_allowed_escapes.json b/test/json/fixtures/minefield/y_string_allowed_escapes.json new file mode 100644 index 000000000..7f495532f --- /dev/null +++ b/test/json/fixtures/minefield/y_string_allowed_escapes.json @@ -0,0 +1 @@ +["\"\\\/\b\f\n\r\t"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json b/test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json new file mode 100755 index 000000000..d4439eda7 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json @@ -0,0 +1 @@ +["\\u0000"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_backslash_doublequotes.json b/test/json/fixtures/minefield/y_string_backslash_doublequotes.json new file mode 100644 index 000000000..ae03243b6 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_backslash_doublequotes.json @@ -0,0 +1 @@ +["\""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_comments.json b/test/json/fixtures/minefield/y_string_comments.json new file mode 100644 index 000000000..2260c20c2 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_comments.json @@ -0,0 +1 @@ +["a/*b*/c/*d//e"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_double_escape_a.json b/test/json/fixtures/minefield/y_string_double_escape_a.json new file mode 100644 index 000000000..6715d6f40 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_double_escape_a.json @@ -0,0 +1 @@ +["\\a"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_double_escape_n.json b/test/json/fixtures/minefield/y_string_double_escape_n.json new file mode 100644 index 000000000..44ca56c4d --- /dev/null +++ b/test/json/fixtures/minefield/y_string_double_escape_n.json @@ -0,0 +1 @@ +["\\n"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_escaped_control_character.json b/test/json/fixtures/minefield/y_string_escaped_control_character.json new file mode 100644 index 000000000..5b014a9c2 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_escaped_control_character.json @@ -0,0 +1 @@ +["\u0012"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_escaped_noncharacter.json b/test/json/fixtures/minefield/y_string_escaped_noncharacter.json new file mode 100755 index 000000000..2ff52e2c5 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_escaped_noncharacter.json @@ -0,0 +1 @@ +["\uFFFF"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_in_array.json b/test/json/fixtures/minefield/y_string_in_array.json new file mode 100755 index 000000000..21d7ae4cd --- /dev/null +++ b/test/json/fixtures/minefield/y_string_in_array.json @@ -0,0 +1 @@ +["asd"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_in_array_with_leading_space.json b/test/json/fixtures/minefield/y_string_in_array_with_leading_space.json new file mode 100755 index 000000000..9e1887c1e --- /dev/null +++ b/test/json/fixtures/minefield/y_string_in_array_with_leading_space.json @@ -0,0 +1 @@ +[ "asd"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_last_surrogates_1_and_2.json b/test/json/fixtures/minefield/y_string_last_surrogates_1_and_2.json new file mode 100644 index 000000000..3919cef76 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_last_surrogates_1_and_2.json @@ -0,0 +1 @@ +["\uDBFF\uDFFF"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_nbsp_uescaped.json b/test/json/fixtures/minefield/y_string_nbsp_uescaped.json new file mode 100644 index 000000000..2085ab1a1 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_nbsp_uescaped.json @@ -0,0 +1 @@ +["new\u00A0line"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json new file mode 100755 index 000000000..059e4d9dd --- /dev/null +++ b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json @@ -0,0 +1 @@ +["􏿿"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json new file mode 100755 index 000000000..4c913bd41 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json @@ -0,0 +1 @@ +["￿"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_null_escape.json b/test/json/fixtures/minefield/y_string_null_escape.json new file mode 100644 index 000000000..c1ad84404 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_null_escape.json @@ -0,0 +1 @@ +["\u0000"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_one-byte-utf-8.json b/test/json/fixtures/minefield/y_string_one-byte-utf-8.json new file mode 100644 index 000000000..157185923 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_one-byte-utf-8.json @@ -0,0 +1 @@ +["\u002c"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_pi.json b/test/json/fixtures/minefield/y_string_pi.json new file mode 100644 index 000000000..9df11ae88 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_pi.json @@ -0,0 +1 @@ +["π"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json b/test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json new file mode 100755 index 000000000..10a33a171 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json @@ -0,0 +1 @@ +["𛿿"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_simple_ascii.json b/test/json/fixtures/minefield/y_string_simple_ascii.json new file mode 100644 index 000000000..8cadf7d05 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_simple_ascii.json @@ -0,0 +1 @@ +["asd "] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_space.json b/test/json/fixtures/minefield/y_string_space.json new file mode 100644 index 000000000..efd782cc3 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_space.json @@ -0,0 +1 @@ +" " \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json b/test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json new file mode 100755 index 000000000..7620b6655 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json @@ -0,0 +1 @@ +["\uD834\uDd1e"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_three-byte-utf-8.json b/test/json/fixtures/minefield/y_string_three-byte-utf-8.json new file mode 100644 index 000000000..108f1d67d --- /dev/null +++ b/test/json/fixtures/minefield/y_string_three-byte-utf-8.json @@ -0,0 +1 @@ +["\u0821"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_two-byte-utf-8.json b/test/json/fixtures/minefield/y_string_two-byte-utf-8.json new file mode 100644 index 000000000..461503c31 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_two-byte-utf-8.json @@ -0,0 +1 @@ +["\u0123"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_u+2028_line_sep.json b/test/json/fixtures/minefield/y_string_u+2028_line_sep.json new file mode 100755 index 000000000..897b6021a --- /dev/null +++ b/test/json/fixtures/minefield/y_string_u+2028_line_sep.json @@ -0,0 +1 @@ +["
"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_u+2029_par_sep.json b/test/json/fixtures/minefield/y_string_u+2029_par_sep.json new file mode 100755 index 000000000..8cd998c89 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_u+2029_par_sep.json @@ -0,0 +1 @@ +["
"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_uEscape.json b/test/json/fixtures/minefield/y_string_uEscape.json new file mode 100755 index 000000000..f7b41a02f --- /dev/null +++ b/test/json/fixtures/minefield/y_string_uEscape.json @@ -0,0 +1 @@ +["\u0061\u30af\u30EA\u30b9"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_uescaped_newline.json b/test/json/fixtures/minefield/y_string_uescaped_newline.json new file mode 100644 index 000000000..3a5a220b6 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_uescaped_newline.json @@ -0,0 +1 @@ +["new\u000Aline"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unescaped_char_delete.json b/test/json/fixtures/minefield/y_string_unescaped_char_delete.json new file mode 100755 index 000000000..7d064f498 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unescaped_char_delete.json @@ -0,0 +1 @@ +[""] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode.json b/test/json/fixtures/minefield/y_string_unicode.json new file mode 100644 index 000000000..3598095b7 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode.json @@ -0,0 +1 @@ +["\uA66D"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json b/test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json new file mode 100755 index 000000000..0bb3b51e7 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json @@ -0,0 +1 @@ +["\u005C"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_2.json b/test/json/fixtures/minefield/y_string_unicode_2.json new file mode 100644 index 000000000..a7dcb9768 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_2.json @@ -0,0 +1 @@ +["⍂㈴⍂"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+10FFFE_nonchar.json b/test/json/fixtures/minefield/y_string_unicode_U+10FFFE_nonchar.json new file mode 100644 index 000000000..9a8370b96 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+10FFFE_nonchar.json @@ -0,0 +1 @@ +["\uDBFF\uDFFE"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+1FFFE_nonchar.json b/test/json/fixtures/minefield/y_string_unicode_U+1FFFE_nonchar.json new file mode 100644 index 000000000..c51f8ae45 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+1FFFE_nonchar.json @@ -0,0 +1 @@ +["\uD83F\uDFFE"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json b/test/json/fixtures/minefield/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json new file mode 100644 index 000000000..626d5f815 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json @@ -0,0 +1 @@ +["\u200B"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+2064_invisible_plus.json b/test/json/fixtures/minefield/y_string_unicode_U+2064_invisible_plus.json new file mode 100644 index 000000000..1e23972c6 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+2064_invisible_plus.json @@ -0,0 +1 @@ +["\u2064"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+FDD0_nonchar.json b/test/json/fixtures/minefield/y_string_unicode_U+FDD0_nonchar.json new file mode 100644 index 000000000..18ef151b4 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+FDD0_nonchar.json @@ -0,0 +1 @@ +["\uFDD0"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_U+FFFE_nonchar.json b/test/json/fixtures/minefield/y_string_unicode_U+FFFE_nonchar.json new file mode 100644 index 000000000..13d261fda --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_U+FFFE_nonchar.json @@ -0,0 +1 @@ +["\uFFFE"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json b/test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json new file mode 100755 index 000000000..4e6257856 --- /dev/null +++ b/test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json @@ -0,0 +1 @@ +["\u0022"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_utf8.json b/test/json/fixtures/minefield/y_string_utf8.json new file mode 100644 index 000000000..40878435f --- /dev/null +++ b/test/json/fixtures/minefield/y_string_utf8.json @@ -0,0 +1 @@ +["€𝄞"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_string_with_del_character.json b/test/json/fixtures/minefield/y_string_with_del_character.json new file mode 100755 index 000000000..8bd24907d --- /dev/null +++ b/test/json/fixtures/minefield/y_string_with_del_character.json @@ -0,0 +1 @@ +["aa"] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_false.json b/test/json/fixtures/minefield/y_structure_lonely_false.json new file mode 100644 index 000000000..02e4a84d6 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_false.json @@ -0,0 +1 @@ +false \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_int.json b/test/json/fixtures/minefield/y_structure_lonely_int.json new file mode 100755 index 000000000..f70d7bba4 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_int.json @@ -0,0 +1 @@ +42 \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_negative_real.json b/test/json/fixtures/minefield/y_structure_lonely_negative_real.json new file mode 100755 index 000000000..b5135a207 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_negative_real.json @@ -0,0 +1 @@ +-0.1 \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_null.json b/test/json/fixtures/minefield/y_structure_lonely_null.json new file mode 100644 index 000000000..ec747fa47 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_null.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_string.json b/test/json/fixtures/minefield/y_structure_lonely_string.json new file mode 100755 index 000000000..b6e982ca9 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_string.json @@ -0,0 +1 @@ +"asd" \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_lonely_true.json b/test/json/fixtures/minefield/y_structure_lonely_true.json new file mode 100755 index 000000000..f32a5804e --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_lonely_true.json @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_string_empty.json b/test/json/fixtures/minefield/y_structure_string_empty.json new file mode 100644 index 000000000..3cc762b55 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_string_empty.json @@ -0,0 +1 @@ +"" \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_trailing_newline.json b/test/json/fixtures/minefield/y_structure_trailing_newline.json new file mode 100644 index 000000000..0c3426d4c --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_trailing_newline.json @@ -0,0 +1 @@ +["a"] diff --git a/test/json/fixtures/minefield/y_structure_true_in_array.json b/test/json/fixtures/minefield/y_structure_true_in_array.json new file mode 100644 index 000000000..de601e305 --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_true_in_array.json @@ -0,0 +1 @@ +[true] \ No newline at end of file diff --git a/test/json/fixtures/minefield/y_structure_whitespace_array.json b/test/json/fixtures/minefield/y_structure_whitespace_array.json new file mode 100644 index 000000000..2bedf7f2d --- /dev/null +++ b/test/json/fixtures/minefield/y_structure_whitespace_array.json @@ -0,0 +1 @@ + [] \ No newline at end of file diff --git a/test/json/json_minefield_parser_test.rb b/test/json/json_minefield_parser_test.rb new file mode 100644 index 000000000..17619b2db --- /dev/null +++ b/test/json/json_minefield_parser_test.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true +require_relative 'test_helper' + +class JSONMinefieldParserTest < Test::Unit::TestCase + # Test fixtures from https://github.com/nst/JSONTestSuite + # https://seriot.ch/security/parsing_json.html + + fixtures = Dir[File.join(File.expand_path("../fixtures/minefield", __FILE__), "*.json")] + + class << self + private + + def define_test(name, &block) + if RUBY_ENGINE == 'jruby' && JRUBY_PENDING.include?(name) + define_method("test_#{name}") do + pend("#{name} doesn't pass on JRuby", &block) + end + else + define_method("test_#{name}", &block) + end + end + end + + JRUBY_PENDING = %w( + n_structure_open_array_object + n_structure_100000_opening_arrays + n_object_trailing_comment_slash_open + ).freeze + + INVALID_ENCODING_TESTS = %w( + i_string_truncated-utf-8 + i_string_overlong_sequence_6_bytes_null + i_string_overlong_sequence_6_bytes + i_string_overlong_sequence_2_bytes + i_string_not_in_unicode_range + i_string_lone_utf8_continuation_byte + i_string_lone_second_surrogate + i_string_iso_latin_1 + i_string_invalid_utf-8 + i_string_incomplete_surrogate_pair + i_string_UTF-8_invalid_sequence + i_object_key_lone_2nd_surrogate + ) + + COMMENT_TESTS = %w( + n_object_trailing_comment + n_object_trailing_comment_slash_open + n_structure_object_with_comment + ).freeze + + DUPLICATE_KEY_TESTS = %w( + y_object_duplicated_key + y_object_duplicated_key_and_value + ).freeze + + # Tests starting with `i_` aren't defined by the spec, we can chose to + # change our behavior, but ideally we'd be consistent across the Java and C + # parsers. + UNDEFINED_FAILING = %w( + i_string_1st_surrogate_but_2nd_missing + i_string_1st_valid_surrogate_2nd_invalid + i_string_UTF-16LE_with_BOM + i_string_incomplete_surrogate_and_escape_valid + i_string_incomplete_surrogates_escape_valid + i_string_invalid_lonely_surrogate + i_string_invalid_surrogate + i_string_inverted_surrogates_U+1D11E + i_string_utf16BE_no_BOM + i_string_utf16LE_no_BOM + i_structure_UTF-8_BOM_empty_object + ) + + if RUBY_ENGINE == 'jruby' + # The Java parser validate the document encoding + # but the C parser doesn't. + # See: https://github.com/ruby/json/issues/138 + UNDEFINED_FAILING.concat(INVALID_ENCODING_TESTS) + end + + fixtures.each do |path| + payload = File.read(path) + name = File.basename(path, '.json') + + if COMMENT_TESTS.include?(name) + define_test(name) do + JSON.parse(payload, allow_comments: true) + + assert_raise(JSON::ParserError) do + JSON.parse(payload, allow_comments: false) + end + end + elsif DUPLICATE_KEY_TESTS.include?(name) + define_test(name) do + JSON.parse(payload, allow_duplicate_key: true) + assert_raise(JSON::ParserError) do + JSON.parse(payload, allow_duplicate_key: false) + end + end + elsif name.start_with?("y_") || (name.start_with?("i_") && !UNDEFINED_FAILING.include?(name)) + define_test(name) do + JSON.parse(payload, max_nesting: false, allow_duplicate_key: false, allow_comments: false) + assert true + end + elsif name.start_with?("n_") || (name.start_with?("i_") && UNDEFINED_FAILING.include?(name)) + define_test(name) do + assert_raise(JSON::ParserError) do + JSON.parse(payload, max_nesting: false, allow_comments: true) + end + end + else + raise "Unexpected minefield test: #{name}" + end + end +end \ No newline at end of file From e5fa06aa426f8c39519924f0d94393dd82a51acf Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 00:10:12 +0900 Subject: [PATCH 364/506] Fix heap-use-after-free in JSON::ResumableParser#partial_value Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 35 ++++++++++++++++++++---------- test/json/resumable_parser_test.rb | 20 +++++++++++++++++ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 136aab6ae..fe87d5c44 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2481,18 +2481,9 @@ static VALUE cResumableParser_clear(VALUE self) return self; } -/* - * call-seq: partial_value -> object - * - * Returns the Ruby objects parsed up to this point: - * parser << '[1, [2, 3,' - * parser.parse # => false - * parser.value # ArgumentError no ready value - * parser.partial_value # => [1, [2, 3]] - */ -static VALUE cResumableParser_partial_value(VALUE self) +static VALUE cResumableParser_partial_value_body(VALUE self) { - JSON_ResumableParser *original_parser = ResumableParser_acquire(self, false); + JSON_ResumableParser *original_parser = cResumableParser_get(self); JSON_ResumableParser parser = *original_parser; parser.state.frames = &parser.frames; @@ -2559,6 +2550,28 @@ static VALUE cResumableParser_partial_value(VALUE self) return partial_result; } +/* + * call-seq: partial_value -> object + * + * Returns the Ruby objects parsed up to this point: + * parser << '[1, [2, 3,' + * parser.parse # => false + * parser.value # ArgumentError no ready value + * parser.partial_value # => [1, [2, 3]] + */ +static VALUE cResumableParser_partial_value(VALUE self) +{ + JSON_ResumableParser *parser = ResumableParser_acquire(self, true); + + int status; + VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status); + parser->in_use = false; + if (status) { + rb_jump_tag(status); + } + return result; +} + /* * call-seq: rest -> string * diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 52f1356a7..6eeabf645 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -208,6 +208,26 @@ def test_reentrency_prevented assert_equal "ResumableParser can't be used recursively", error.message end + def test_reentrency_prevented_in_partial_value + parser = nil + callback = ->(o) do + # Arrays are only built while partial_value runs (the scalars were pushed by the + # earlier parse); re-entering here used to corrupt/free the shared frame stack. + parser.parse if o.is_a?(Array) + o + end + parser = new_parser(on_load: callback) + parser << '[1, [2, 3,' + parser.parse + error = assert_raise ArgumentError do + parser.partial_value + end + assert_equal "ResumableParser can't be used recursively", error.message + + # The in_use lock must be released even though partial_value raised. + refute_predicate parser, :value? + end + def test_exception_unlock_parser called = false parser = nil From 3927c5ca4cb2465e34e9544de0a3cf62a817441a Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 00:11:05 +0900 Subject: [PATCH 365/506] Fix JSON::ResumableParser#parse leaking the in_use lock on an empty buffer Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 1 + test/json/resumable_parser_test.rb | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index fe87d5c44..b67feacbc 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2377,6 +2377,7 @@ static VALUE cResumableParser_parse(VALUE self) { JSON_ResumableParser *parser = ResumableParser_acquire(self, true); if (!parser->buffer) { + parser->in_use = false; return Qfalse; } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 6eeabf645..329bbc34e 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -48,6 +48,21 @@ def test_clear refute_predicate @parser, :value? end + def test_parse_with_empty_buffer_keeps_parser_usable + # parse before any feed must not leak the in_use lock + refute @parser.parse + @parser << '[1, 2, 3]' + assert @parser.parse + assert_equal [1, 2, 3], @parser.value + + # same after a clear with no following feed + @parser.clear + refute @parser.parse + @parser << '[4]' + assert @parser.parse + assert_equal [4], @parser.value + end + def test_parse_document_direct @parser << '[true]' assert_equal true, @parser.parse From 489b8c13b99149d64350cbc87a8643dfb738e18d Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 00:12:06 +0900 Subject: [PATCH 366/506] Forbid JSON::ResumableParser#<< while the parser is in use Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 7 +++++-- test/json/resumable_parser_test.rb | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b67feacbc..e4f140d7c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2274,6 +2274,8 @@ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) return self; } +static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock); + /* * call-seq: self << string -> self * @@ -2282,13 +2284,14 @@ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) static VALUE cResumableParser_feed(VALUE self, VALUE str) { rb_check_frozen(self); + + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + str = convert_encoding(str); if (!RSTRING_LEN(str)) { return self; } - JSON_ResumableParser *parser = cResumableParser_get(self); - size_t offset = parser->state.cursor - parser->state.start; const size_t remaining = parser->state.end - parser->state.cursor; diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 329bbc34e..40dcb4a46 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -243,6 +243,26 @@ def test_reentrency_prevented_in_partial_value refute_predicate parser, :value? end + def test_feed_during_callback_prevented + parser = nil + callback = ->(o) do + parser << '99' if o == 1 # feeding while a parse is running must be rejected + o + end + parser = new_parser(on_load: callback) + parser << '[1, 2, 3]' + error = assert_raise ArgumentError do + parser.parse + end + assert_equal "ResumableParser can't be used recursively", error.message + + # the lock is released, so the parser stays usable + parser = new_parser + parser << '[1, 2, 3]' + assert parser.parse + assert_equal [1, 2, 3], parser.value + end + def test_exception_unlock_parser called = false parser = nil From d91b370b1e84bf6940c00bac8480824ab7c7f8a9 Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 00:14:30 +0900 Subject: [PATCH 367/506] Fix JSON::ResumableParser stalling on a NUL byte Co-Authored-By: Jean Boussier --- ext/json/ext/parser/parser.c | 16 +++++++++---- test/json/resumable_parser_test.rb | 38 ++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e4f140d7c..9762fb815 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1642,7 +1642,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo state->cursor++; value = json_decode_array(state, config, 0); break; - } else if (resumable && next == 0) { + } else if (resumable && eos(state)) { state->cursor = value_start; return false; } @@ -1691,8 +1691,14 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case 0: - return false; - + // peek() returns 0 both at end-of-stream and for a literal NUL byte in the + // buffer. Only a genuine EOS means "feed me more"; a NUL byte that is not at + // EOS is just an invalid character. + if (eos(state)) { + return false; + } else { + raise_syntax_error("unexpected NULL byte: %s", state); + } default: raise_syntax_error("unexpected character: %s", state); } @@ -1807,7 +1813,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - } else if (resumable && next_char == 0) { + } else if (resumable && eos(state)) { return false; } else { raise_syntax_error("expected ',' or ']' after array value", state); @@ -1858,7 +1864,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case JSON_PHASE_OBJECT_KEY: JSON_UNREACHABLE_RETURN(false); case JSON_PHASE_OBJECT_COLON: goto JSON_PHASE_OBJECT_COLON; } - } else if (resumable && next_char == 0) { + } else if (resumable && eos(state)) { return false; } else { raise_syntax_error("expected ',' or '}' after object value, got: %s", state); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 40dcb4a46..b48b9e056 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -133,6 +133,30 @@ def test_parse_byte_by_byte_numbers assert_resumed_parsing('123 ') end + def test_nul_byte_is_a_syntax_error + # A NUL byte in a structural position must raise, not stall forever waiting for more input + # (peek() returns 0 both at EOS and for a literal NUL byte). + assert_parse_error "\x00" # document value + assert_parse_error "[\x00]" # first array element + assert_parse_error "[1\x00]" # after an array element (',' or ']' expected) + assert_parse_error "[1,\x00]" # array element after ',' + assert_parse_error "{\x00}" # object key + assert_parse_error "{\"a\":1\x00}" # after an object value (',' or '}' expected) + assert_parse_error "{\"a\":1,\x00}" # object key after ',' + end + + def test_incomplete_input_at_structural_positions_resumes + # Counterpart of test_nul_byte_is_a_syntax_error: a genuine EOS at the same positions must + # stay incomplete (return false), not raise -- this is what distinguishes EOS from a NUL. + assert_incomplete "[" + assert_incomplete "[1" + assert_incomplete "[1," + assert_incomplete "{" + assert_incomplete "{\"a\"" + assert_incomplete "{\"a\":1" + assert_incomplete "{\"a\":1," + end + def test_rest @parser << '[1, 2, 3, "unterminated string' refute @parser.parse @@ -316,6 +340,20 @@ def test_buffer_shrink private + def assert_parse_error(json) + parser = new_parser + parser << json + assert_raise(JSON::ParserError, "expected a parse error for #{json.inspect}") do + parser.parse + end + end + + def assert_incomplete(json) + parser = new_parser + parser << json + refute(parser.parse, "expected #{json.inspect} not to produce a value") + end + def assert_partial_value(expected, json) parser = new_parser parser << json From 56d16deb46f81345ed7215ef91ce1a18b0a31fe1 Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 00:15:19 +0900 Subject: [PATCH 368/506] Reset all per-parse state in JSON::ResumableParser#clear Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 3 +++ test/json/resumable_parser_test.rb | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 9762fb815..48700911e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2487,6 +2487,9 @@ static VALUE cResumableParser_clear(VALUE self) parser->frames.head = 0; parser->value_stack.head = 0; parser->state.name_cache.length = 0; + parser->state.current_nesting = 0; + parser->state.in_array = 1; + parser->state.emitted_deprecations = 0; parser->state.start = parser->state.cursor = parser->state.end = NULL; return self; } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index b48b9e056..2801544c7 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -63,6 +63,20 @@ def test_parse_with_empty_buffer_keeps_parser_usable assert_equal [4], @parser.value end + def test_clear_resets_nesting_depth + # An unfinished document leaks a nesting level; #clear must reset it so a later shallow + # document is not rejected with a spurious NestingError. + parser = new_parser(max_nesting: 10) + 10.times do + parser << '[1' # opens an array that is never closed before clear + parser.parse + parser.clear + end + parser << '[1]' + assert parser.parse + assert_equal [1], parser.value + end + def test_parse_document_direct @parser << '[true]' assert_equal true, @parser.parse From 82b6b21354e9c35351b7699bd9050614dadd9223 Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 02:44:26 +0900 Subject: [PATCH 369/506] Fix off-by-one line number in parse error messages cursor_position consumed the newline ending the previous line (post-decrement) before counting lines, so any error past the first line was reported one line too low (e.g. "[1,\n@" reported '@' at line 1 instead of line 2). Count that newline when the column loop breaks on it. Co-Authored-By: Jean Boussier --- ext/json/ext/parser/parser.c | 1 + test/json/json_parser_test.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 48700911e..665d31527 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -593,6 +593,7 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum while (cursor >= state->start) { if (*cursor-- == '\n') { + line++; break; } column++; diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index c1aa2f615..c891dd7c2 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -862,7 +862,7 @@ def test_parse_error_incomplete_hash end def test_parse_error_snippet - omit "C ext only test" unless RUBY_ENGINE == "ruby" + omit "JRuby errors don't contain positions" unless RUBY_ENGINE == "ruby" error = assert_raise(JSON::ParserError) { JSON.parse("あああああああああああああああああああああああ") } assert_equal "unexpected character: 'ああああああああああ' at line 1 column 1", error.message @@ -875,6 +875,15 @@ def test_parse_error_snippet error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") } assert_equal "unexpected character: 'abcあああああああああ' at line 1 column 1", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse("[1,\n@") } + assert_equal "unexpected character: '@' at line 2 column 1", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse("[\n 1,\n @\n]") } + assert_equal "unexpected character: '@' at line 3 column 3", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse("@") } + assert_equal "unexpected character: '@' at line 1 column 1", error.message end def test_parse_leading_slash From 65549acb9cc49e64efc8cdc6028fafceacffa5c8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 18 Jun 2026 21:24:12 +0200 Subject: [PATCH 370/506] ResumableParser#parse: don't swallow other parsers errors Fix: https://github.com/ruby/json/issues/1024#issuecomment-4745232736 Co-Authored-By: Yusuke Endoh --- ext/json/ext/parser/parser.c | 37 +++++++++++++++--------------- test/json/resumable_parser_test.rb | 12 ++++++++++ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 665d31527..38e24374f 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -660,13 +660,13 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta return message; } -static VALUE parse_error_new(VALUE message, long line, long column, bool eos) +static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos) { VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); rb_ivar_set(exc, i_at_column, LONG2NUM(column)); - if (eos) { - rb_ivar_set(exc, i_at_eos, Qtrue); + if (eos && state->value_stack_handle) { + rb_ivar_set(exc, i_at_eos, *state->value_stack_handle); } return exc; } @@ -676,7 +676,7 @@ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *st long line, column; cursor_position(state, &line, &column); VALUE message = build_parse_error_message(format, state, line, column); - rb_exc_raise(parse_error_new(message, line, column, eos)); + rb_exc_raise(parse_error_new(state, message, line, column, eos)); } NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state) @@ -1169,7 +1169,7 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d long line, column; cursor_position(state, &line, &column); rb_str_concat(message, build_parse_error_message("", state, line, column)) ; - rb_exc_raise(parse_error_new(message, line, column, false)); + rb_exc_raise(parse_error_new(state, message, line, column, false)); } NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) @@ -2281,7 +2281,7 @@ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) return self; } -static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock); +static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock); /* * call-seq: self << string -> self @@ -2292,7 +2292,7 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) { rb_check_frozen(self); - JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); str = convert_encoding(str); if (!RSTRING_LEN(str)) { @@ -2350,9 +2350,9 @@ static VALUE json_parse_any_resumable_safe(VALUE _args) return (VALUE)json_parse_any(args->state, args->config, true); } -static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) +static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock) { - JSON_ResumableParser *parser = cResumableParser_get(self); + JSON_ResumableParser *parser = cResumableParser_get(*self); if (parser->in_use) { rb_raise(rb_eArgError, "ResumableParser can't be used recursively"); @@ -2365,8 +2365,8 @@ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) // self may have moved, so we need to update all pointers // Investigate: We might be better off keeping JSON_ParserState on the stack // and only persist what we need. - parser->state.value_stack_handle = &self; - parser->state.frame_stack_handle = &self; + parser->state.value_stack_handle = self; + parser->state.frame_stack_handle = self; parser->state.value_stack = &parser->value_stack; parser->state.frames = &parser->frames; @@ -2385,7 +2385,7 @@ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) */ static VALUE cResumableParser_parse(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(self, true); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, true); if (!parser->buffer) { parser->in_use = false; return Qfalse; @@ -2419,8 +2419,9 @@ static VALUE cResumableParser_parse(VALUE self) parser->in_use = false; if (status) { complete = false; - if (RTEST(rb_ivar_get(rb_errinfo(), rb_intern("@eos")))) { - complete = false; // is an EOS error + VALUE error_source = rb_ivar_get(rb_errinfo(), i_at_eos); + if (error_source == self) { + complete = false; // is an EOS error raised by ourself rb_set_errinfo(Qnil); } else { rb_jump_tag(status); // reraise @@ -2437,7 +2438,7 @@ static VALUE cResumableParser_parse(VALUE self) */ static VALUE cResumableParser_value_p(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); if (parser->value_stack.head > 0) { json_frame *frame = json_frame_stack_peek(&parser->frames); @@ -2461,7 +2462,7 @@ static VALUE cResumableParser_value_p(VALUE self) */ static VALUE cResumableParser_value(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); if (parser->frames.head > 0) { json_frame *frame = json_frame_stack_peek(&parser->frames); @@ -2483,7 +2484,7 @@ static VALUE cResumableParser_value(VALUE self) */ static VALUE cResumableParser_clear(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); parser->buffer = 0; parser->frames.head = 0; parser->value_stack.head = 0; @@ -2575,7 +2576,7 @@ static VALUE cResumableParser_partial_value_body(VALUE self) */ static VALUE cResumableParser_partial_value(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(self, true); + JSON_ResumableParser *parser = ResumableParser_acquire(&self, true); int status; VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 2801544c7..a720679c2 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -77,6 +77,18 @@ def test_clear_resets_nesting_depth assert_equal [1], parser.value end + def test_nested_parse_error + parser = new_parser(on_load: ->(o) do + JSON.parse("") #=> raises JSON::ParserError + o + end) + parser << "[1]" + + assert_raise(JSON::ParserError) do + parser.parse + end + end + def test_parse_document_direct @parser << '[true]' assert_equal true, @parser.parse From 3650e2bf4139f982323a15e972e4116fa5e430c4 Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Fri, 19 Jun 2026 03:39:35 +0900 Subject: [PATCH 371/506] Document that JSON::ResumableParser does not bound its buffer size An incomplete document is buffered in full with no size limit, so reading from an untrusted source can grow memory without bound. Note in the rdoc that bounding the input is the caller's responsibility. Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 38e24374f..289bd4e37 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2268,6 +2268,11 @@ static inline JSON_ResumableParser *cResumableParser_get(VALUE self) * parser << ' ' * parser.parse # => true * parser.value # => 123 + * + * === Security + * + * An incomplete document is buffered in full and there is no size limit, so when reading + * from an untrusted source the caller is responsible for bounding how much data is fed. */ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) { From 004a5c1faa113a97ef2aeef695c62fe068dcb927 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 00:03:28 +0200 Subject: [PATCH 372/506] Refactor ResumableParser Stop updating stack handles, we don't actually need them as we're starting heap allocated. Add an assertion to prove it. Use a dedicated field for storing the parser reference to be used for marking EOS errors. Also stop marking EOS errors when not in resumable mode. --- ext/json/ext/parser/parser.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 289bd4e37..7b028635c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -233,6 +233,8 @@ static rvalue_stack *rvalue_stack_grow(rvalue_stack *stack, VALUE *handle, rvalu static VALUE rvalue_stack_push(rvalue_stack *stack, VALUE value, VALUE *handle, rvalue_stack **stack_ref) { + JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || handle); + if (RB_UNLIKELY(stack->head >= stack->capa)) { stack = rvalue_stack_grow(stack, handle, stack_ref); } @@ -431,6 +433,7 @@ typedef struct JSON_ParserStateStruct { int in_array; int current_nesting; unsigned int emitted_deprecations; + VALUE parser; } JSON_ParserState; static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref); @@ -451,6 +454,9 @@ static json_frame_stack *json_frame_stack_grow(json_frame_stack *stack, VALUE *h static json_frame *json_frame_stack_push(JSON_ParserState *state, json_frame frame) { json_frame_stack *stack = state->frames; + + JSON_ASSERT(stack->type != RVALUE_STACK_STACK_ALLOCATED || state->frame_stack_handle); + if (RB_UNLIKELY(stack->head >= stack->capa)) { stack = json_frame_stack_grow(stack, state->frame_stack_handle, &state->frames); } @@ -665,8 +671,8 @@ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); rb_ivar_set(exc, i_at_column, LONG2NUM(column)); - if (eos && state->value_stack_handle) { - rb_ivar_set(exc, i_at_eos, *state->value_stack_handle); + if (eos && state->parser) { + rb_ivar_set(exc, i_at_eos, state->parser); } return exc; } @@ -2166,6 +2172,7 @@ static void JSON_ResumableParser_mark(void *ptr) rvalue_stack_mark(&parser->value_stack); rvalue_cache_mark(&parser->state.name_cache); rb_gc_mark(parser->buffer); // pin the buffer + rb_gc_mark_movable(parser->state.parser); } static void JSON_ResumableParser_free(void *ptr) @@ -2200,6 +2207,7 @@ static void JSON_ResumableParser_compact(void *ptr) rvalue_stack_compact(&parser->value_stack); rvalue_cache_compact(&parser->state.name_cache); parser->buffer = rb_gc_location(parser->buffer); + parser->state.parser = rb_gc_location(parser->state.parser); } static const rb_data_type_t JSON_ResumableParser_type = { @@ -2221,6 +2229,7 @@ static VALUE cResumableParser_allocate(VALUE klass) JSON_ResumableParser *parser; VALUE obj = TypedData_Make_Struct(klass, JSON_ResumableParser, &JSON_ResumableParser_type, parser); parser->state.in_array++; + parser->state.parser = obj; return obj; } @@ -2286,7 +2295,7 @@ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) return self; } -static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock); +static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock); /* * call-seq: self << string -> self @@ -2297,7 +2306,7 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) { rb_check_frozen(self); - JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); str = convert_encoding(str); if (!RSTRING_LEN(str)) { @@ -2355,9 +2364,9 @@ static VALUE json_parse_any_resumable_safe(VALUE _args) return (VALUE)json_parse_any(args->state, args->config, true); } -static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock) +static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) { - JSON_ResumableParser *parser = cResumableParser_get(*self); + JSON_ResumableParser *parser = cResumableParser_get(self); if (parser->in_use) { rb_raise(rb_eArgError, "ResumableParser can't be used recursively"); @@ -2370,8 +2379,6 @@ static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock) // self may have moved, so we need to update all pointers // Investigate: We might be better off keeping JSON_ParserState on the stack // and only persist what we need. - parser->state.value_stack_handle = self; - parser->state.frame_stack_handle = self; parser->state.value_stack = &parser->value_stack; parser->state.frames = &parser->frames; @@ -2390,7 +2397,7 @@ static JSON_ResumableParser *ResumableParser_acquire(VALUE *self, bool lock) */ static VALUE cResumableParser_parse(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(&self, true); + JSON_ResumableParser *parser = ResumableParser_acquire(self, true); if (!parser->buffer) { parser->in_use = false; return Qfalse; @@ -2443,7 +2450,7 @@ static VALUE cResumableParser_parse(VALUE self) */ static VALUE cResumableParser_value_p(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); if (parser->value_stack.head > 0) { json_frame *frame = json_frame_stack_peek(&parser->frames); @@ -2467,7 +2474,7 @@ static VALUE cResumableParser_value_p(VALUE self) */ static VALUE cResumableParser_value(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); if (parser->frames.head > 0) { json_frame *frame = json_frame_stack_peek(&parser->frames); @@ -2489,7 +2496,7 @@ static VALUE cResumableParser_value(VALUE self) */ static VALUE cResumableParser_clear(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(&self, false); + JSON_ResumableParser *parser = ResumableParser_acquire(self, false); parser->buffer = 0; parser->frames.head = 0; parser->value_stack.head = 0; @@ -2581,7 +2588,7 @@ static VALUE cResumableParser_partial_value_body(VALUE self) */ static VALUE cResumableParser_partial_value(VALUE self) { - JSON_ResumableParser *parser = ResumableParser_acquire(&self, true); + JSON_ResumableParser *parser = ResumableParser_acquire(self, true); int status; VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status); From 46aa46d3dad627899af1ab6485ee2baf487fbec9 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 19 Jun 2026 10:35:09 +0900 Subject: [PATCH 373/506] Read ASCII-incompatible strings in binary mode --- test/json/json_minefield_parser_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/json/json_minefield_parser_test.rb b/test/json/json_minefield_parser_test.rb index 17619b2db..715903255 100644 --- a/test/json/json_minefield_parser_test.rb +++ b/test/json/json_minefield_parser_test.rb @@ -78,7 +78,7 @@ def define_test(name, &block) end fixtures.each do |path| - payload = File.read(path) + payload = File.binread(path) name = File.basename(path, '.json') if COMMENT_TESTS.include?(name) @@ -111,4 +111,4 @@ def define_test(name, &block) raise "Unexpected minefield test: #{name}" end end -end \ No newline at end of file +end From 1676fc015f77b248b47a1635db9e5e7255a67871 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Jun 2026 03:03:20 +0000 Subject: [PATCH 374/506] Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- .github/workflows/sync-ruby.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4bc00fa8..a82e21dd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: - { os: windows-latest, ruby: jruby } steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 @@ -70,7 +70,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Ruby uses: ruby/setup-ruby-pkgs@v1 diff --git a/.github/workflows/sync-ruby.yml b/.github/workflows/sync-ruby.yml index f5df56c5b..d6d95946a 100644 --- a/.github/workflows/sync-ruby.yml +++ b/.github/workflows/sync-ruby.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest if: ${{ github.repository_owner == 'ruby' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Create GitHub App token id: app-token From 9c76e3a45912fe1a3b5d01fa7a79944e2171701e Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Sat, 13 Jun 2026 21:24:52 -0500 Subject: [PATCH 375/506] Ported the parser.c to the java extension and remove the dependency on ragel. --- Rakefile | 34 +- java/src/json/ext/Parser.java | 955 +++++++ java/src/json/ext/ParserConfig.java | 2537 ----------------- java/src/json/ext/ParserConfig.rl | 861 ------ java/src/json/ext/ParserService.java | 4 +- java/src/json/ext/StringDecoder.java | 13 +- java/src/json/ext/StringScanner.java | 140 + .../src/json/ext/VectorizedStringScanner.java | 72 + 8 files changed, 1182 insertions(+), 3434 deletions(-) create mode 100644 java/src/json/ext/Parser.java delete mode 100644 java/src/json/ext/ParserConfig.java delete mode 100644 java/src/json/ext/ParserConfig.rl create mode 100644 java/src/json/ext/StringScanner.java create mode 100644 java/src/json/ext/VectorizedStringScanner.java diff --git a/Rakefile b/Rakefile index d05b45b79..20f1b3d8c 100644 --- a/Rakefile +++ b/Rakefile @@ -84,8 +84,6 @@ namespace :ci do end JAVA_DIR = "java/src/json/ext" -JAVA_RAGEL_PATH = "#{JAVA_DIR}/ParserConfig.rl" -JAVA_PARSER_SRC = "#{JAVA_DIR}/ParserConfig.java" JAVA_SOURCES = FileList["#{JAVA_DIR}/*.java"].exclude("#{JAVA_DIR}/Vectorized*.java") JAVA_VEC_SOURCES = FileList["#{JAVA_DIR}/Vectorized*.java"] JAVA_CLASSES = [] @@ -96,35 +94,6 @@ CLEAN.concat FileList["java/src/**/*.class"] CLEAN << JRUBY_PARSER_JAR CLEAN << JRUBY_GENERATOR_JAR -CLOBBER << JAVA_PARSER_SRC - -which = lambda { |c| - w = `which #{c}` - break w.chomp unless w.empty? -} - -if RUBY_PLATFORM =~ /mingw|mswin/ - # cleans up Windows CI output - RAGEL_CODEGEN = %w[ragel].find(&which) - RAGEL_DOTGEN = %w[ragel].find(&which) -else - RAGEL_CODEGEN = %w[rlcodegen rlgen-cd ragel].find(&which) - RAGEL_DOTGEN = %w[rlgen-dot rlgen-cd ragel].find(&which) -end - -file JAVA_PARSER_SRC => JAVA_RAGEL_PATH do - cd JAVA_DIR do - if RAGEL_CODEGEN == 'ragel' - sh "ragel ParserConfig.rl -J -o ParserConfig.java" - else - sh "ragel -x ParserConfig.rl | #{RAGEL_CODEGEN} -J" - end - end -end - -desc "Generate parser with ragel" -task :ragel => [JAVA_PARSER_SRC] - if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' path_separator = File::PATH_SEPARATOR ENV['JAVA_HOME'] ||= [ @@ -174,7 +143,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' end desc "Compiling jruby extension" - task :compile => [:ragel] + JAVA_CLASSES + task :compile => JAVA_CLASSES desc "Package the jruby gem" task :jruby_gem => :create_jar do @@ -199,6 +168,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' "json/ext/Parser*.class", "json/ext/RuntimeInfo*.class", "json/ext/StringDecoder*.class", + "json/ext/*StringScanner*.class", "json/ext/Utils*.class" ] sh 'jar', 'cf', File.basename(JRUBY_PARSER_JAR), *parser_classes diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java new file mode 100644 index 000000000..092ff4d41 --- /dev/null +++ b/java/src/json/ext/Parser.java @@ -0,0 +1,955 @@ +package json.ext; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.jcodings.specific.UTF8Encoding; +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubyFloat; +import org.jruby.RubyHash; +import org.jruby.RubyObject; +import org.jruby.RubyProc; +import org.jruby.RubyString; +import org.jruby.RubySymbol; +import org.jruby.anno.JRubyMethod; +import org.jruby.exceptions.RaiseException; +import org.jruby.runtime.Block; +import org.jruby.runtime.ObjectAllocator; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.Visibility; +import org.jruby.runtime.builtin.IRubyObject; +import org.jruby.util.ByteList; +import org.jruby.util.ConvertBytes; +import org.jruby.util.ConvertDouble.DoubleConverter; +import org.jruby.util.ConvertBytes; +import org.jruby.util.ConvertDouble.DoubleConverter; +import org.jruby.util.ConvertBytes; +import org.jruby.util.ConvertDouble.DoubleConverter; +import org.jruby.util.ConvertBytes; +import org.jruby.util.ConvertDouble.DoubleConverter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.function.BiFunction; + +import static org.jruby.util.ConvertDouble.DoubleConverter; + +/** + * This is a port of the parser from the C extension. + */ +public class Parser extends RubyObject { + private final RuntimeInfo info; + private int maxNesting; + private boolean allowNaN; + private boolean allowTrailingComma; + private boolean allowComments; + private boolean deprecateComments; + private boolean allowControlCharacters; + private boolean allowInvalidEscape; + private boolean allowDuplicateKey; + private boolean deprecateDuplicateKey; + private boolean symbolizeNames; + private boolean freeze; + private RubyProc onLoadProc; + private RubyClass decimalClass; + BiFunction decimalFactory; + + private static final int DEFAULT_MAX_NESTING = 100; + + // Maximum number of deprecation warnings emitted per parse, to avoid + // flooding output on pathological inputs. + private static final int MAX_DEPRECATIONS = 5; + + // constant names in the JSON module containing those values + private static final String CONST_NAN = "NaN"; + private static final String CONST_INFINITY = "Infinity"; + private static final String CONST_MINUS_INFINITY = "MinusInfinity"; + + static final ObjectAllocator ALLOCATOR = Parser::new; + + public Parser(Ruby runtime, RubyClass metaClass) { + super(runtime, metaClass); + info = RuntimeInfo.forRuntime(runtime); + } + + @JRubyMethod(name = "new", meta = true) + public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, Block block) { + Parser config = (Parser)((RubyClass)clazz).allocate(); + config.callInit(arg0, block); + return config; + } + + @JRubyMethod(name = "new", meta = true) + public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, IRubyObject arg1, Block block) { + Parser config = (Parser)((RubyClass)clazz).allocate(); + config.callInit(arg0, arg1, block); + return config; + } + + @JRubyMethod(visibility = Visibility.PRIVATE) + public IRubyObject initialize(ThreadContext context, IRubyObject options) { + checkFrozen(); + Ruby runtime = context.runtime; + + OptionsReader opts = new OptionsReader(context, options); + this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); + this.allowNaN = opts.getBool("allow_nan", false); + if (opts.hasKey("allow_comments")) { + this.allowComments = opts.getBool("allow_comments", false); + this.deprecateComments = false; + } else { + this.allowComments = true; + this.deprecateComments = true; + } + + this.allowControlCharacters = opts.getBool("allow_control_characters", false); + this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); + this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); + this.symbolizeNames = opts.getBool("symbolize_names", false); + if (opts.hasKey("allow_duplicate_key")) { + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.deprecateDuplicateKey = false; + } else { + this.allowDuplicateKey = false; + this.deprecateDuplicateKey = true; + } + + this.freeze = opts.getBool("freeze", false); + this.onLoadProc = opts.getProc("on_load"); + + this.decimalClass = opts.getClass("decimal_class", null); + + if (decimalClass == null) { + this.decimalFactory = this::createFloat; + } else if (decimalClass == runtime.getClass("BigDecimal")) { + this.decimalFactory = this::createBigDecimal; + } else { + this.decimalFactory = this::createCustomDecimal; + } + + return this; + } + + public IRubyObject onLoad(ThreadContext context, IRubyObject object) { + if (onLoadProc == null) { + return object; + } else { + return onLoadProc.call(context, object); + } + } + + /** + * Checks the given string's encoding. If a non-UTF-8 encoding is detected, + * a converted copy is returned. + * Returns the source string if no conversion is needed. + */ + private RubyString convertEncoding(ThreadContext context, RubyString source) { + Encoding encoding = source.getEncoding(); + if (encoding == ASCIIEncoding.INSTANCE) { + source = (RubyString) source.dup(); + source.setEncoding(UTF8Encoding.INSTANCE); + source.clearCodeRange(); + } else if (encoding != UTF8Encoding.INSTANCE) { + source = (RubyString) source.encode(context, context.runtime.getEncodingService().convertEncodingToRubyEncoding(UTF8Encoding.INSTANCE)); + } + return source; + } + + /** + * Parser#parse() + * + *

Parses the current JSON text source and returns the + * complete data structure as a result. + */ + @JRubyMethod + public IRubyObject parse(ThreadContext context, IRubyObject source) { + return new ParserSession(this, convertEncoding(context, source.convertToString()), info).parse(context); + } + + private RubyFloat createFloat(final ThreadContext context, final ByteList num) { + return RubyFloat.newFloat(context.runtime, new DoubleConverter().parse(num, true, true)); + } + + private IRubyObject createBigDecimal(final ThreadContext context, final ByteList num) { + final Ruby runtime = context.runtime; + return runtime.getKernel().callMethod(context, "BigDecimal", runtime.newString(num)); + } + + private IRubyObject createCustomDecimal(final ThreadContext context, final ByteList num) { + return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); + } + + /** + * A single parsing session over one source string. + * + *

Once a ParserSession is instantiated, the source string should not + * change until the parsing is complete. The ParserSession object assumes + * the source {@link RubyString} is still associated to its original + * {@link ByteList}, which in turn must still be bound to the same + * byte[] value (and on the same offset). + */ + private static final class ParserSession { + private static final int INITIAL_FRAME_CAPACITY = 32; + private static final int INITIAL_VALUE_CAPACITY = 64; + + // Integers with fewer than this many digits are built directly from a + // long accumulated during the digit scan; longer ones fall back to + // ConvertBytes (Bignum). 17 digits (< 1e17) always fit a signed long, + // including after negation. Mirrors parser.c's MAX_FAST_INTEGER_SIZE. + private static final int MAX_FAST_INTEGER_SIZE = 18; + + // Same idea as the rvalue_cache in the C extension. + // + // This is bigger than the C implementation as this is + // heap allocated. + private static final int KEY_CACHE_CAPA = 128; + private static final int KEY_CACHE_MAX_ENTRY_LENGTH = 55; + + private static final long SPACES = 0x2020202020202020L; + + private enum FrameType { + ROOT(FramePhase.DONE), + ARRAY(FramePhase.ARRAY_COMMA), + OBJECT(FramePhase.OBJECT_COMMA); + + private final FramePhase nextPhase; + + FrameType(FramePhase nextPhase) { + this.nextPhase = nextPhase; + } + + FramePhase nextPhase() { + return nextPhase; + } + } + + private enum FramePhase { + DONE, + ARRAY_COMMA, + OBJECT_COMMA, + VALUE, + OBJECT_KEY, + OBJECT_COLON + } + + private static final class Frame { + FrameType type; + FramePhase phase; + // The position within the value stack when this frame was created. + int valueStackHead; + // The cursor position when we encountered a '{'. + // Used for error message reporting when decoding an JSON object. + int startCursor; + } + + private final Parser config; + private final RuntimeInfo info; + private final ByteList byteList; + private final ByteList view; + private final byte[] data; + + // Little-endian view over {@link #data}, used by the SWAR string scan + // so that {@link Long#numberOfTrailingZeros} locates the first + // interesting" byte within an 8-byte chunk. + private final ByteBuffer chunks; + private final StringScanner scanner; + private final StringDecoder decoder; + private final int begin; + private final int end; + private int cursor; + + private ThreadContext context; + + // Integer value accumulated by the most recent scanDigits() call. + private long digitsValue; + private int currentNesting = 0; + + private int inArray = 0; + + private final IRubyObject[] keyCache = new IRubyObject[KEY_CACHE_CAPA]; + private int keyCacheLength = 0; + private int emittedDeprecations = 0; + + + private Frame[] frameStack = new Frame[INITIAL_FRAME_CAPACITY]; + private int frameDepth = 0; + + private IRubyObject[] valueStack = new IRubyObject[INITIAL_VALUE_CAPACITY]; + private int valueTop = 0; + + private ParserSession(Parser config, RubyString source, RuntimeInfo info) { + this.config = config; + this.info = info; + this.byteList = source.getByteList(); + this.data = byteList.unsafeBytes(); + this.chunks = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + this.scanner = StringScanner.getInstance(); + this.view = new ByteList(data, false); + this.begin = byteList.begin(); + this.end = begin + byteList.length(); + this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); + } + + public IRubyObject parse(ThreadContext context) { + this.context = context; + this.cursor = begin; + pushFrame(FrameType.ROOT, FramePhase.VALUE, 0, -1); + + IRubyObject result = run(); + + // Only trailing whitespace (and comments) may follow the document. + eatWhitespace(); + if (cursor < end) { + throw unexpectedToken(cursor, end); + } + return result; + } + + private IRubyObject run() { + while (true) { + Frame frame = topFrame(); + + switch (frame.phase) { + case DONE: + return valueStack[valueTop - 1]; + + case VALUE: { + eatWhitespace(); + IRubyObject value; + switch (peek()) { + case 'n': + if (matchKeyword("null")) { value = context.nil; break; } + throw unexpectedToken(cursor, end); + case 't': + if (matchKeyword("true")) { value = context.tru; break; } + throw unexpectedToken(cursor, end); + case 'f': + if (matchKeyword("false")) { value = context.fals; break; } + throw unexpectedToken(cursor, end); + case 'N': + if (config.allowNaN && matchKeyword("NaN")) { + value = getConstant(CONST_NAN); + break; + } + throw unexpectedToken(cursor, end); + case 'I': + if (config.allowNaN && matchKeyword("Infinity")) { + value = getConstant(CONST_INFINITY); + break; + } + throw unexpectedToken(cursor, end); + case '-': + cursor++; + if (config.allowNaN && matchKeyword("Infinity")) { + value = getConstant(CONST_MINUS_INFINITY); + } else { + value = parseNumber(cursor - 1); + } + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + value = parseNumber(cursor); + break; + case '"': + value = parseString(false); + break; + case '[': { + cursor++; + eatWhitespace(); + if (peek() == ']') { + cursor++; + value = decodeArray(0); + break; + } + currentNesting++; + checkNesting(); + inArray++; + // Phase stays VALUE: the next iteration reads + // the first element. + pushFrame(FrameType.ARRAY, FramePhase.VALUE, valueTop, -1); + continue; + } + case '{': { + int objectStart = cursor; + cursor++; + eatWhitespace(); + if (peek() == '}') { + cursor++; + value = decodeObject(0); + break; + } + currentNesting++; + checkNesting(); + // Phase KEY: the next iteration reads the first key. + pushFrame(FrameType.OBJECT, FramePhase.OBJECT_KEY, valueTop, objectStart); + continue; + } + case 0: + throw newException(Utils.M_PARSER_ERROR, "unexpected end of input"); + default: + throw unexpectedToken(cursor, end); + } + + pushValue(value); + valueCompleted(frame); + continue; + } + + case OBJECT_KEY: { + eatWhitespace(); + if (peek() == '"') { + pushValue(parseString(true)); + frame.phase = FramePhase.OBJECT_COLON; + continue; + } + throw unexpectedToken(cursor, end); + } + + case OBJECT_COLON: { + eatWhitespace(); + if (peek() == ':') { + cursor++; + frame.phase = FramePhase.VALUE; + continue; + } + throw unexpectedToken(cursor, end); + } + + case ARRAY_COMMA: { + eatWhitespace(); + byte b = peek(); + if (b == ',') { + cursor++; + if (config.allowTrailingComma) { + eatWhitespace(); + if (peek() == ']') { + // Trailing comma: re-enter COMMA to close. + continue; + } + } + frame.phase = FramePhase.VALUE; + continue; + } else if (b == ']') { + cursor++; + int count = entryCount(frame); + currentNesting--; + inArray--; + popFrame(); + pushValue(decodeArray(count)); + valueCompleted(topFrame()); + continue; + } + throw unexpectedToken(cursor, end); + } + + case OBJECT_COMMA: { + eatWhitespace(); + byte b = peek(); + if (b == ',') { + cursor++; + if (config.allowTrailingComma) { + eatWhitespace(); + if (peek() == '}') { + // Trailing comma: re-enter COMMA to close. + continue; + } + } + frame.phase = FramePhase.OBJECT_KEY; + continue; + } else if (b == '}') { + cursor++; + currentNesting--; + int count = entryCount(frame); + // Temporarily rewind the cursor so a duplicate-key + // error points at the object's opening brace. + int finalCursor = cursor; + cursor = frame.startCursor; + IRubyObject object = decodeObject(count); + cursor = finalCursor; + popFrame(); + pushValue(object); + valueCompleted(topFrame()); + continue; + } + throw unexpectedToken(cursor, end); + } + + default: + throw context.runtime.newRuntimeError("unreachable parser state"); + } + } + } + + private void pushValue(IRubyObject value) { + if (valueTop == valueStack.length) { + valueStack = Arrays.copyOf(valueStack, valueStack.length * 2); + } + valueStack[valueTop++] = config.onLoad(context, value); + } + + private void valueCompleted(Frame frame) { + frame.phase = frame.type.nextPhase(); + } + + private int entryCount(Frame frame) { + return valueTop - frame.valueStackHead; + } + + private Frame topFrame() { + return frameStack[frameDepth - 1]; + } + + private void pushFrame(FrameType type, FramePhase phase, + int valueStackHead, int startCursor) { + if (frameDepth == frameStack.length) { + frameStack = Arrays.copyOf(frameStack, frameStack.length * 2); + } + Frame frame = frameStack[frameDepth]; + if (frame == null) { + frame = new Frame(); + frameStack[frameDepth] = frame; + } + frame.type = type; + frame.phase = phase; + frame.valueStackHead = valueStackHead; + frame.startCursor = startCursor; + frameDepth++; + } + + private void popFrame() { + frameDepth--; + } + + private void checkNesting() { + if (config.maxNesting > 0 && currentNesting > config.maxNesting) { + throw newException(Utils.M_NESTING_ERROR, + "nesting of " + currentNesting + " is too deep"); + } + } + + private IRubyObject decodeArray(int count) { + int base = valueTop - count; + IRubyObject[] elements = new IRubyObject[count]; + System.arraycopy(valueStack, base, elements, 0, count); + valueTop = base; + RubyArray array = RubyArray.newArrayNoCopy(context.runtime, elements); + if (config.freeze) { + array.setFrozen(true); + } + return array; + } + + private IRubyObject decodeObject(int count) { + final Ruby runtime = context.runtime; + int base = valueTop - count; + int limit = valueTop; + RubyHash hash = RubyHash.newHash(runtime); + for (int i = base; i < limit; i += 2) { + // We use RubyHash#fastASet because all object keys have already been + // frozen and deduplicated. + // + // This was significantly faster than RubyHash#op_aset. + hash.fastASet(valueStack[i], valueStack[i + 1]); + } + valueTop = base; + + if (!config.allowDuplicateKey && hash.size() < count / 2) { + onDuplicateKey(findDuplicateKey(base, limit)); + } + + if (config.freeze) { + hash.setFrozen(true); + } + return hash; + } + + private IRubyObject findDuplicateKey(int base, int limit) { + RubyHash seen = RubyHash.newHash(context.runtime); + for (int i = base; i < limit; i += 2) { + int before = seen.size(); + IRubyObject key = valueStack[i]; + seen.fastASetCheckString(context.runtime, key, context.tru); + if (seen.size() == before) { + return key; + } + } + return context.nil; + } + + private void onDuplicateKey(IRubyObject key) { + // Symbol keys are reported by their string form (":a" -> "a") to + // match the C parser's message. + String keyInspect = key.callMethod(context, "to_s") + .callMethod(context, "inspect").asJavaString(); + if (config.deprecateDuplicateKey) { + if (emittedDeprecations < MAX_DEPRECATIONS) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "detected duplicate key " + keyInspect + " in JSON object. " + + "This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`"); + } + } else { + throw newException(Utils.M_PARSER_ERROR, "duplicate key " + keyInspect); + } + } + + private IRubyObject parseNumber(int numberStart) { + byte first = cursor < end ? data[cursor] : 0; + boolean negative = data[numberStart] == '-'; + + int intStart = cursor; + int p = scanDigits(intStart); + int intDigits = p - intStart; + long mantissa = digitsValue; + + if ((first == '0' && intDigits > 1) || intDigits == 0) { + cursor = p; + throw unexpectedToken(numberStart, end); + } + + boolean integer = true; + + if (p < end && data[p] == '.') { + integer = false; + p++; + int fracStart = p; + p = scanDigits(fracStart); + if (p == fracStart) { + cursor = p; + throw unexpectedToken(numberStart, end); + } + } + + if (p < end && (data[p] == 'e' || data[p] == 'E')) { + integer = false; + p++; + if (p < end && (data[p] == '+' || data[p] == '-')) p++; + int expStart = p; + p = scanDigits(expStart); + if (p == expStart) { + cursor = p; + throw unexpectedToken(numberStart, end); + } + } + + cursor = p; + + if (integer) { + if (intDigits < MAX_FAST_INTEGER_SIZE) { + return context.runtime.newFixnum(negative ? -mantissa : mantissa); + } + return ConvertBytes.byteListToInum(context.runtime, absSubSequence(numberStart, p), 10, true); + } + return config.decimalFactory.apply(context, absSubSequence(numberStart, p)); + } + + private int scanDigits(int p) { + long acc = 0; + while (p + 8 <= end) { + long next8 = chunks.getLong(p); + // Branchless all-eight-are-digits test (simdjson / 0x80.pl): + // each nibble pair resolves to 0x3 iff the byte is '0'..'9'. + long match = (next8 & 0xF0F0F0F0F0F0F0F0L) + | (((next8 + 0x0606060606060606L) & 0xF0F0F0F0F0F0F0F0L) >>> 4); + if (match == 0x3333333333333333L) { + acc = acc * 100000000L + decode8digits(next8); + p += 8; + continue; + } + int consecutive = Long.numberOfTrailingZeros(match ^ 0x3333333333333333L) >>> 3; + if (consecutive >= 4) { + acc = acc * 10000L + decode4digits((int) next8); + p += 4; + consecutive -= 4; + } + while (consecutive > 0) { + acc = acc * 10 + (data[p] - '0'); + p++; + consecutive--; + } + digitsValue = acc; + return p; + } + while (p < end && data[p] >= '0' && data[p] <= '9') { + acc = acc * 10 + (data[p] - '0'); + p++; + } + digitsValue = acc; + return p; + } + + // Decode eight packed little-endian ASCII digits to their integer value. + // From https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/ + private static long decode8digits(long val) { + final long mask = 0x000000FF000000FFL; + final long mul1 = 0x000F424000000064L; // 100 + (1000000 << 32) + final long mul2 = 0x0000271000000001L; // 1 + (10000 << 32) + val -= 0x3030303030303030L; + val = (val * 10) + (val >>> 8); + val = (((val & mask) * mul1) + (((val >>> 16) & mask) * mul2)) >>> 32; + return val; + } + + // Decode four packed little-endian ASCII digits to their integer value. + private static long decode4digits(int val) { + val -= 0x30303030; + val = (val * 10) + (val >>> 8); + return ((val & 0xFF) * 100) + ((val >>> 16) & 0xFF); + } + + private IRubyObject parseString(boolean isName) { + final byte[] data = this.data; + final int contentStart = cursor + 1; // skip opening quote + + // The scanner finds the closing quote and reports whether the body + // is plain printable ASCII (no escape, no ASCII control character, + // no non-ASCII byte). Anything non-plain is handed to StringDecoder, + // which performs the UTF-8/control validation, escape expansion, and + // error reporting. + long scanned = scanner.scan(data, chunks, contentStart, end); + final int q = (int) scanned; + if (q < 0) { + throw newException(Utils.M_PARSER_ERROR, + "unexpected end of input, expected closing \""); + } + + boolean plain = (scanned & StringScanner.PLAIN_BIT) != 0; + cursor = q + 1; // past closing quote + + // Note: When running multiple read-world benchmarks in the same JVM, + // this seems consistently faster than "if (isName && plain)" + // and only handling the ASCII-only path in the cache. + // + // Note 2: It's important that all object keys are frozen and deduplicated, + // decodeObject relies this. + if (isName) { + // Resolve the key's decoded bytes without building a RubyString + // yet, so a cache hit skips allocation entirely. + byte[] buf; + int off; + int len; + if (plain) { + buf = data; + off = contentStart; + len = q - contentStart; + } else { + ByteList decoded = decoder.decode(context, byteList, + contentStart - begin, q - begin); + buf = decoded.getUnsafeBytes(); + off = decoded.begin(); + len = decoded.realSize(); + } + // Same as the C extension. + if (inArray > 0 && len > 0 && len <= KEY_CACHE_MAX_ENTRY_LENGTH && isLetter(buf[off])) { + return cachedKey(buf, off, len); + } + return internedKey(buf, off, len); + } + + ByteList content; + if (plain) { + // Plain ASCII string, skip the decoder. + content = new ByteList(data, contentStart, q - contentStart, true); + } else { + content = decoder.decode(context, byteList, + contentStart - begin, q - begin); + } + + RubyString string = context.runtime.newString(content); + string.setEncoding(UTF8Encoding.INSTANCE); + string.clearCodeRange(); + + if (config.freeze) { + string.setFrozen(true); + return context.runtime.freezeAndDedupString(string); + } + + return string; + } + + private static boolean isLetter(byte b) { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'); + } + + private IRubyObject internedKey(byte[] buf, int off, int len) { + RubyString string = context.runtime.newString( + new ByteList(buf, off, len, UTF8Encoding.INSTANCE, true)); + if (config.symbolizeNames) { + return string.intern(); + } + string.setFrozen(true); + return context.runtime.freezeAndDedupString(string); + } + + private IRubyObject cachedKey(byte[] buf, int off, int len) { + int low = 0; + int high = keyCacheLength - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + int cmp = compareKey(buf, off, len, keyCache[mid]); + if (cmp == 0) { + return keyCache[mid]; + } else if (cmp > 0) { + low = mid + 1; + } else { + high = mid - 1; + } + } + + IRubyObject key = internedKey(buf, off, len); + if (keyCacheLength < KEY_CACHE_CAPA) { + System.arraycopy(keyCache, low, keyCache, low + 1, keyCacheLength - low); + keyCache[low] = key; + keyCacheLength++; + } + return key; + } + + // Orders by length first, then unsigned byte value + private static int compareKey(byte[] buf, int off, int len, IRubyObject entry) { + ByteList eb = entry instanceof RubySymbol + ? ((RubySymbol) entry).getBytes() + : ((RubyString) entry).getByteList(); + int elen = eb.realSize(); + if (len != elen) { + return len - elen; + } + byte[] ebuf = eb.getUnsafeBytes(); + int ebeg = eb.begin(); + for (int i = 0; i < len; i++) { + int cmp = Byte.toUnsignedInt(buf[off + i]) - Byte.toUnsignedInt(ebuf[ebeg + i]); + if (cmp != 0) { + return cmp; + } + } + return 0; + } + + private byte peek() { + return cursor < end ? data[cursor] : 0; + } + + private boolean matchKeyword(String keyword) { + int len = keyword.length(); + if (end - cursor < len) { + return false; + } + for (int i = 0; i < len; i++) { + if (data[cursor + i] != (byte) keyword.charAt(i)) { + return false; + } + } + cursor += len; + return true; + } + + private void eatWhitespace() { + while (cursor < end) { + switch (data[cursor]) { + case ' ': + case '\t': + case '\r': + cursor++; + break; + case '\n': + cursor++; + // Same heuristic from the C parser: a newline in + // pretty-printed JSON is almost always followed by a run + // of indentation spaces, so skip them eight at a time. + while (cursor + 8 <= end) { + long x = chunks.getLong(cursor); + if (x == SPACES) { + cursor += 8; + } else { + cursor += Long.numberOfTrailingZeros(x ^ SPACES) >>> 3; + break; + } + } + break; + case '/': + eatComments(); + break; + default: + return; + } + } + } + + private void eatComments() { + if (!config.allowComments) { + if (config.deprecateComments) { + if (emittedDeprecations < MAX_DEPRECATIONS) { + emittedDeprecations++; + context.runtime.getWarnings().warning( + "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`"); + } + } else { + throw unexpectedToken(cursor, end); + } + } + + int start = cursor; + cursor++; // skip '/' + switch (peek()) { + case '/': + cursor++; + while (cursor < end && data[cursor] != '\n') cursor++; + if (cursor < end) cursor++; // consume newline + break; + case '*': + cursor++; + while (true) { + while (cursor < end && data[cursor] != '*') cursor++; + if (cursor >= end) { + throw newException(Utils.M_PARSER_ERROR, + "unterminated comment, expected closing '*/'"); + } + cursor++; // past '*' + if (peek() == '/') { + cursor++; + break; + } + } + break; + default: + throw unexpectedToken(start, end); + } + } + + /** + * Updates the "view" ByteList with the new offsets and returns it. The + * returned ByteList must be consumed before the next call, since the + * same instance is reused. + */ + private ByteList absSubSequence(int absStart, int absEnd) { + view.setBegin(absStart); + view.setRealSize(absEnd - absStart); + return view; + } + + private IRubyObject getConstant(String name) { + return info.jsonModule.get().getConstant(name); + } + + private RaiseException parsingError(int absStart, int absEnd) { + RubyString msg = context.runtime.newString("unexpected token at '") + .cat(data, absStart, Math.min(absEnd - absStart, 32)) + .cat((byte)'\''); + return newException(Utils.M_PARSER_ERROR, msg); + } + + private RaiseException unexpectedToken(int absStart, int absEnd) { + return parsingError(absStart, absEnd); + } + + private RaiseException newException(String className, String message) { + return Utils.newException(context, className, message); + } + + private RaiseException newException(String className, RubyString message) { + return Utils.newException(context, className, message); + } + } +} diff --git a/java/src/json/ext/ParserConfig.java b/java/src/json/ext/ParserConfig.java deleted file mode 100644 index 562c0ab61..000000000 --- a/java/src/json/ext/ParserConfig.java +++ /dev/null @@ -1,2537 +0,0 @@ - -// line 1 "ParserConfig.rl" -/* - * This code is copyrighted work by Daniel Luz . - * - * Distributed under the Ruby license: https://www.ruby-lang.org/en/about/license.txt - */ -package json.ext; - -import org.jcodings.Encoding; -import org.jcodings.specific.ASCIIEncoding; -import org.jcodings.specific.UTF8Encoding; -import org.jruby.Ruby; -import org.jruby.RubyArray; -import org.jruby.RubyClass; -import org.jruby.RubyFloat; -import org.jruby.RubyHash; -import org.jruby.RubyInteger; -import org.jruby.RubyObject; -import org.jruby.RubyProc; -import org.jruby.RubyString; -import org.jruby.anno.JRubyMethod; -import org.jruby.exceptions.JumpException; -import org.jruby.exceptions.RaiseException; -import org.jruby.runtime.Block; -import org.jruby.runtime.Helpers; -import org.jruby.runtime.ObjectAllocator; -import org.jruby.runtime.ThreadContext; -import org.jruby.runtime.Visibility; -import org.jruby.runtime.builtin.IRubyObject; -import org.jruby.util.ByteList; -import org.jruby.util.ConvertBytes; - -import java.util.function.BiFunction; - -import static org.jruby.util.ConvertDouble.DoubleConverter; - -/** - * The JSON::Ext::Parser class. - * - *

This is the JSON parser implemented as a Java class. To use it as the - * standard parser, set - * 

JSON.parser = JSON::Ext::Parser
- * This is performed for you when you include "json/ext". - * - *

This class does not perform the actual parsing, just acts as an interface - * to Ruby code. When the {@link #parse(ThreadContext)} method is invoked, a - * ParserConfig.ParserSession object is instantiated, which handles the process. - * - * @author mernen - */ -public class ParserConfig extends RubyObject { - private final RuntimeInfo info; - private int maxNesting; - private boolean allowNaN; - private boolean allowTrailingComma; - private boolean allowComments; - private boolean deprecateComments; - private boolean allowControlCharacters; - private boolean allowInvalidEscape; - private boolean allowDuplicateKey; - private boolean deprecateDuplicateKey; - private boolean symbolizeNames; - private boolean freeze; - private RubyProc onLoadProc; - private RubyClass decimalClass; - BiFunction decimalFactory; - private RubyHash match_string; - - private static final int DEFAULT_MAX_NESTING = 100; - - private static final ByteList JSON_MINUS_INFINITY = new ByteList(ByteList.plain("-Infinity")); - // constant names in the JSON module containing those values - private static final String CONST_NAN = "NaN"; - private static final String CONST_INFINITY = "Infinity"; - private static final String CONST_MINUS_INFINITY = "MinusInfinity"; - - static final ObjectAllocator ALLOCATOR = ParserConfig::new; - - /** - * Multiple-value return for internal parser methods. - * - *

All the parseStuff methods return instances of - * ParserResult when successful, or null when - * there's a problem with the input data. - */ - static final class ParserResult { - /** - * The result of the successful parsing. Should never be - * null. - */ - IRubyObject result; - /** - * The point where the parser returned. - */ - int p; - - void update(IRubyObject result, int p) { - this.result = result; - this.p = p; - } - } - - public ParserConfig(Ruby runtime, RubyClass metaClass) { - super(runtime, metaClass); - info = RuntimeInfo.forRuntime(runtime); - } - - /** - * ParserConfig.new(source, opts = {}) - * - *

Creates a new JSON::Ext::Parser instance for the string - * source. - * It will be configured by the opts Hash. - * opts can have the following keys: - *

- *

- *
:max_nesting - *
The maximum depth of nesting allowed in the parsed data - * structures. Disable depth checking with :max_nesting => false|nil|0, - * it defaults to 100. - *

- *

:allow_nan - *
If set to true, allow NaN, - * Infinity and -Infinity in defiance of RFC 4627 - * to be parsed by the Parser. This option defaults to false. - *

- *

:allow_trailing_comma - *
If set to true, allow arrays and objects with a trailing - * comma in defiance of RFC 4627 to be parsed by the Parser. - * This option defaults to false. - *

- *

:symbolize_names - *
If set to true, returns symbols for the names (keys) in - * a JSON object. Otherwise strings are returned, which is also the default. - *

- *

:create_additions - *
If set to false, the Parser doesn't create additions - * even if a matching class and create_id was found. This option - * defaults to true. - *

- *

:object_class - *
Defaults to Hash. If another type is provided, it will be used - * instead of Hash to represent JSON objects. The type must respond to - * new without arguments, and return an object that respond to []=. - *

- *

:array_class - *
Defaults to Array. If another type is provided, it will be used - * instead of Hash to represent JSON arrays. The type must respond to - * new without arguments, and return an object that respond to <<. - *

- *

:decimal_class - *
Specifies which class to use instead of the default (Float) when - * parsing decimal numbers. This class must accept a single string argument - * in its constructor. - *
- */ - - @JRubyMethod(name = "new", meta = true) - public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, Block block) { - ParserConfig config = (ParserConfig)((RubyClass)clazz).allocate(); - - config.callInit(arg0, block); - - return config; - } - - @JRubyMethod(name = "new", meta = true) - public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, IRubyObject arg1, Block block) { - ParserConfig config = (ParserConfig)((RubyClass)clazz).allocate(); - - config.callInit(arg0, arg1, block); - - return config; - } - - @JRubyMethod(visibility = Visibility.PRIVATE) - public IRubyObject initialize(ThreadContext context, IRubyObject options) { - checkFrozen(); - Ruby runtime = context.runtime; - - OptionsReader opts = new OptionsReader(context, options); - this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); - this.allowNaN = opts.getBool("allow_nan", false); - if (opts.hasKey("allow_comments")) { - this.allowComments = opts.getBool("allow_comments", false); - this.deprecateComments = false; - } else { - this.allowComments = true; - this.deprecateComments = true; - } - - this.allowControlCharacters = opts.getBool("allow_control_characters", false); - this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); - this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); - this.symbolizeNames = opts.getBool("symbolize_names", false); - if (opts.hasKey("allow_duplicate_key")) { - this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); - this.deprecateDuplicateKey = false; - } else { - this.allowDuplicateKey = false; - this.deprecateDuplicateKey = true; - } - - this.freeze = opts.getBool("freeze", false); - this.onLoadProc = opts.getProc("on_load"); - - this.decimalClass = opts.getClass("decimal_class", null); - - if (decimalClass == null) { - this.decimalFactory = this::createFloat; - } else if (decimalClass == runtime.getClass("BigDecimal")) { - this.decimalFactory = this::createBigDecimal; - } else { - this.decimalFactory = this::createCustomDecimal; - } - - return this; - } - - public IRubyObject onLoad(ThreadContext context, IRubyObject object) { - if (onLoadProc == null) { - return object; - } else { - return onLoadProc.call(context, object); - } - } - - /** - * Checks the given string's encoding. If a non-UTF-8 encoding is detected, - * a converted copy is returned. - * Returns the source string if no conversion is needed. - */ - private RubyString convertEncoding(ThreadContext context, RubyString source) { - Encoding encoding = source.getEncoding(); - if (encoding == ASCIIEncoding.INSTANCE) { - source = (RubyString) source.dup(); - source.setEncoding(UTF8Encoding.INSTANCE); - source.clearCodeRange(); - } else if (encoding != UTF8Encoding.INSTANCE) { - source = (RubyString) source.encode(context, context.runtime.getEncodingService().convertEncodingToRubyEncoding(UTF8Encoding.INSTANCE)); - } - return source; - } - - /** - * Parser#parse() - * - *

Parses the current JSON text source and returns the - * complete data structure as a result. - */ - @JRubyMethod - public IRubyObject parse(ThreadContext context, IRubyObject source) { - return new ParserSession(this, convertEncoding(context, source.convertToString()), context, info).parse(context); - } - - /** - * Queries JSON.create_id. Returns null if it is - * set to nil or false, and a String if not. - */ - private RubyString getCreateId(ThreadContext context) { - IRubyObject v = info.jsonModule.get().callMethod(context, "create_id"); - return v.isTrue() ? v.convertToString() : null; - } - - private RubyFloat createFloat(final ThreadContext context, final ByteList num) { - return RubyFloat.newFloat(context.runtime, new DoubleConverter().parse(num, true, true)); - } - - private IRubyObject createBigDecimal(final ThreadContext context, final ByteList num) { - final Ruby runtime = context.runtime; - return runtime.getKernel().callMethod(context, "BigDecimal", runtime.newString(num)); - } - - private IRubyObject createCustomDecimal(final ThreadContext context, final ByteList num) { - return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); - } - - /** - * A string parsing session. - * - *

Once a ParserSession is instantiated, the source string should not - * change until the parsing is complete. The ParserSession object assumes - * the source {@link RubyString} is still associated to its original - * {@link ByteList}, which in turn must still be bound to the same - * byte[] value (and on the same offset). - */ - // Ragel uses lots of fall-through - @SuppressWarnings("fallthrough") - private static class ParserSession { - private final ParserConfig config; - private final RuntimeInfo info; - private final ByteList byteList; - private final ByteList view; - private final byte[] data; - private final StringDecoder decoder; - private int currentNesting = 0; - private int emittedDeprecations = 0; - - private ParserSession(ParserConfig config, RubyString source, ThreadContext context, RuntimeInfo info) { - this.config = config; - this.info = info; - this.byteList = source.getByteList(); - this.data = byteList.unsafeBytes(); - this.view = new ByteList(data, false); - this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); - } - - private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { - RubyString msg = context.runtime.newString("unexpected token at '") - .cat(data, absStart, Math.min(absEnd - absStart, 32)) - .cat((byte)'\''); - return newException(context, Utils.M_PARSER_ERROR, msg); - } - - private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { - return parsingError(context, "unexpected token at '", absStart, absEnd); - } - - -// line 360 "ParserConfig.rl" - - - -// line 326 "ParserConfig.java" -private static byte[] init__JSON_value_actions_0() -{ - return new byte [] { - 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, - 5, 1, 6, 1, 7, 1, 8, 1, 9 - }; -} - -private static final byte _JSON_value_actions[] = init__JSON_value_actions_0(); - - -private static byte[] init__JSON_value_key_offsets_0() -{ - return new byte [] { - 0, 0, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 - }; -} - -private static final byte _JSON_value_key_offsets[] = init__JSON_value_key_offsets_0(); - - -private static char[] init__JSON_value_trans_keys_0() -{ - return new char [] { - 34, 45, 73, 78, 91, 102, 110, 116, 123, 48, 57, 110, - 102, 105, 110, 105, 116, 121, 97, 78, 97, 108, 115, 101, - 117, 108, 108, 114, 117, 101, 0 - }; -} - -private static final char _JSON_value_trans_keys[] = init__JSON_value_trans_keys_0(); - - -private static byte[] init__JSON_value_single_lengths_0() -{ - return new byte [] { - 0, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 - }; -} - -private static final byte _JSON_value_single_lengths[] = init__JSON_value_single_lengths_0(); - - -private static byte[] init__JSON_value_range_lengths_0() -{ - return new byte [] { - 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_value_range_lengths[] = init__JSON_value_range_lengths_0(); - - -private static byte[] init__JSON_value_index_offsets_0() -{ - return new byte [] { - 0, 0, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, - 31, 33, 35, 37, 39, 41, 43, 45, 47, 49 - }; -} - -private static final byte _JSON_value_index_offsets[] = init__JSON_value_index_offsets_0(); - - -private static byte[] init__JSON_value_trans_targs_0() -{ - return new byte [] { - 21, 21, 2, 9, 21, 11, 15, 18, 21, 21, 0, 3, - 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, 21, - 0, 10, 0, 21, 0, 12, 0, 13, 0, 14, 0, 21, - 0, 16, 0, 17, 0, 21, 0, 19, 0, 20, 0, 21, - 0, 0, 0 - }; -} - -private static final byte _JSON_value_trans_targs[] = init__JSON_value_trans_targs_0(); - - -private static byte[] init__JSON_value_trans_actions_0() -{ - return new byte [] { - 13, 11, 0, 0, 15, 0, 0, 0, 17, 11, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, - 0, 0, 0 - }; -} - -private static final byte _JSON_value_trans_actions[] = init__JSON_value_trans_actions_0(); - - -private static byte[] init__JSON_value_from_state_actions_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 19 - }; -} - -private static final byte _JSON_value_from_state_actions[] = init__JSON_value_from_state_actions_0(); - - -static final int JSON_value_start = 1; -static final int JSON_value_first_final = 21; -static final int JSON_value_error = 0; - -static final int JSON_value_en_main = 1; - - -// line 466 "ParserConfig.rl" - - - void parseValue(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject result = null; - - -// line 448 "ParserConfig.java" - { - cs = JSON_value_start; - } - -// line 473 "ParserConfig.rl" - -// line 455 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _acts = _JSON_value_from_state_actions[cs]; - _nacts = (int) _JSON_value_actions[_acts++]; - while ( _nacts-- > 0 ) { - switch ( _JSON_value_actions[_acts++] ) { - case 9: -// line 451 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 487 "ParserConfig.java" - } - } - - _match: do { - _keys = _JSON_value_key_offsets[cs]; - _trans = _JSON_value_index_offsets[cs]; - _klen = _JSON_value_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( data[p] < _JSON_value_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( data[p] > _JSON_value_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_value_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( data[p] < _JSON_value_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( data[p] > _JSON_value_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - cs = _JSON_value_trans_targs[_trans]; - - if ( _JSON_value_trans_actions[_trans] != 0 ) { - _acts = _JSON_value_trans_actions[_trans]; - _nacts = (int) _JSON_value_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_value_actions[_acts++] ) - { - case 0: -// line 368 "ParserConfig.rl" - { - result = context.nil; - } - break; - case 1: -// line 371 "ParserConfig.rl" - { - result = context.fals; - } - break; - case 2: -// line 374 "ParserConfig.rl" - { - result = context.tru; - } - break; - case 3: -// line 377 "ParserConfig.rl" - { - if (config.allowNaN) { - result = getConstant(CONST_NAN); - } else { - throw unexpectedToken(context, p - 2, pe); - } - } - break; - case 4: -// line 384 "ParserConfig.rl" - { - if (config.allowNaN) { - result = getConstant(CONST_INFINITY); - } else { - throw unexpectedToken(context, p - 7, pe); - } - } - break; - case 5: -// line 391 "ParserConfig.rl" - { - if (pe > p + 8 && - absSubSequence(p, p + 9).equals(JSON_MINUS_INFINITY)) { - - if (config.allowNaN) { - result = getConstant(CONST_MINUS_INFINITY); - {p = (( p + 10))-1;} - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - throw unexpectedToken(context, p, pe); - } - } - parseFloat(context, res, p, pe); - if (res.result != null) { - result = res.result; - {p = (( res.p))-1;} - } - parseInteger(context, res, p, pe); - if (res.result != null) { - result = res.result; - {p = (( res.p))-1;} - } - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; - case 6: -// line 417 "ParserConfig.rl" - { - parseString(context, res, p, pe); - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - result = res.result; - {p = (( res.p))-1;} - } - } - break; - case 7: -// line 427 "ParserConfig.rl" - { - currentNesting++; - parseArray(context, res, p, pe); - currentNesting--; - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - result = res.result; - {p = (( res.p))-1;} - } - } - break; - case 8: -// line 439 "ParserConfig.rl" - { - currentNesting++; - parseObject(context, res, p, pe); - currentNesting--; - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - result = res.result; - {p = (( res.p))-1;} - } - } - break; -// line 659 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 474 "ParserConfig.rl" - - if (cs >= JSON_value_first_final && result != null) { - if (config.freeze) { - result.setFrozen(true); - } - res.update(result, p); - } else { - res.update(null, p); - } - } - - -// line 692 "ParserConfig.java" -private static byte[] init__JSON_integer_actions_0() -{ - return new byte [] { - 0, 1, 0 - }; -} - -private static final byte _JSON_integer_actions[] = init__JSON_integer_actions_0(); - - -private static byte[] init__JSON_integer_key_offsets_0() -{ - return new byte [] { - 0, 0, 4, 7, 9, 9 - }; -} - -private static final byte _JSON_integer_key_offsets[] = init__JSON_integer_key_offsets_0(); - - -private static char[] init__JSON_integer_trans_keys_0() -{ - return new char [] { - 45, 48, 49, 57, 48, 49, 57, 48, 57, 48, 57, 0 - }; -} - -private static final char _JSON_integer_trans_keys[] = init__JSON_integer_trans_keys_0(); - - -private static byte[] init__JSON_integer_single_lengths_0() -{ - return new byte [] { - 0, 2, 1, 0, 0, 0 - }; -} - -private static final byte _JSON_integer_single_lengths[] = init__JSON_integer_single_lengths_0(); - - -private static byte[] init__JSON_integer_range_lengths_0() -{ - return new byte [] { - 0, 1, 1, 1, 0, 1 - }; -} - -private static final byte _JSON_integer_range_lengths[] = init__JSON_integer_range_lengths_0(); - - -private static byte[] init__JSON_integer_index_offsets_0() -{ - return new byte [] { - 0, 0, 4, 7, 9, 10 - }; -} - -private static final byte _JSON_integer_index_offsets[] = init__JSON_integer_index_offsets_0(); - - -private static byte[] init__JSON_integer_indicies_0() -{ - return new byte [] { - 0, 2, 3, 1, 2, 3, 1, 1, 4, 1, 3, 4, - 0 - }; -} - -private static final byte _JSON_integer_indicies[] = init__JSON_integer_indicies_0(); - - -private static byte[] init__JSON_integer_trans_targs_0() -{ - return new byte [] { - 2, 0, 3, 5, 4 - }; -} - -private static final byte _JSON_integer_trans_targs[] = init__JSON_integer_trans_targs_0(); - - -private static byte[] init__JSON_integer_trans_actions_0() -{ - return new byte [] { - 0, 0, 0, 0, 1 - }; -} - -private static final byte _JSON_integer_trans_actions[] = init__JSON_integer_trans_actions_0(); - - -static final int JSON_integer_start = 1; -static final int JSON_integer_first_final = 3; -static final int JSON_integer_error = 0; - -static final int JSON_integer_en_main = 1; - - -// line 496 "ParserConfig.rl" - - - void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { - int new_p = parseIntegerInternal(p, pe); - if (new_p == -1) { - res.update(null, p); - return; - } - RubyInteger number = createInteger(context, p, new_p); - res.update(config.onLoad(context, number), new_p + 1); - } - - int parseIntegerInternal(int p, int pe) { - int cs; - - -// line 808 "ParserConfig.java" - { - cs = JSON_integer_start; - } - -// line 512 "ParserConfig.rl" - int memo = p; - -// line 816 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _match: do { - _keys = _JSON_integer_key_offsets[cs]; - _trans = _JSON_integer_index_offsets[cs]; - _klen = _JSON_integer_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( data[p] < _JSON_integer_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( data[p] > _JSON_integer_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_integer_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( data[p] < _JSON_integer_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( data[p] > _JSON_integer_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_integer_indicies[_trans]; - cs = _JSON_integer_trans_targs[_trans]; - - if ( _JSON_integer_trans_actions[_trans] != 0 ) { - _acts = _JSON_integer_trans_actions[_trans]; - _nacts = (int) _JSON_integer_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_integer_actions[_acts++] ) - { - case 0: -// line 490 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 903 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 514 "ParserConfig.rl" - - if (cs < JSON_integer_first_final) { - return -1; - } - - return p; - } - - RubyInteger createInteger(ThreadContext context, int p, int new_p) { - Ruby runtime = context.runtime; - ByteList num = absSubSequence(p, new_p); - return bytesToInum(runtime, num); - } - - RubyInteger bytesToInum(Ruby runtime, ByteList num) { - return ConvertBytes.byteListToInum(runtime, num, 10, true); - } - - -// line 943 "ParserConfig.java" -private static byte[] init__JSON_float_actions_0() -{ - return new byte [] { - 0, 1, 0 - }; -} - -private static final byte _JSON_float_actions[] = init__JSON_float_actions_0(); - - -private static byte[] init__JSON_float_key_offsets_0() -{ - return new byte [] { - 0, 0, 4, 7, 10, 12, 16, 18, 23, 29, 29 - }; -} - -private static final byte _JSON_float_key_offsets[] = init__JSON_float_key_offsets_0(); - - -private static char[] init__JSON_float_trans_keys_0() -{ - return new char [] { - 45, 48, 49, 57, 48, 49, 57, 46, 69, 101, 48, 57, - 43, 45, 48, 57, 48, 57, 46, 69, 101, 48, 57, 69, - 101, 45, 46, 48, 57, 69, 101, 45, 46, 48, 57, 0 - }; -} - -private static final char _JSON_float_trans_keys[] = init__JSON_float_trans_keys_0(); - - -private static byte[] init__JSON_float_single_lengths_0() -{ - return new byte [] { - 0, 2, 1, 3, 0, 2, 0, 3, 2, 0, 2 - }; -} - -private static final byte _JSON_float_single_lengths[] = init__JSON_float_single_lengths_0(); - - -private static byte[] init__JSON_float_range_lengths_0() -{ - return new byte [] { - 0, 1, 1, 0, 1, 1, 1, 1, 2, 0, 2 - }; -} - -private static final byte _JSON_float_range_lengths[] = init__JSON_float_range_lengths_0(); - - -private static byte[] init__JSON_float_index_offsets_0() -{ - return new byte [] { - 0, 0, 4, 7, 11, 13, 17, 19, 24, 29, 30 - }; -} - -private static final byte _JSON_float_index_offsets[] = init__JSON_float_index_offsets_0(); - - -private static byte[] init__JSON_float_indicies_0() -{ - return new byte [] { - 0, 2, 3, 1, 2, 3, 1, 4, 5, 5, 1, 6, - 1, 7, 7, 8, 1, 8, 1, 4, 5, 5, 3, 1, - 5, 5, 1, 6, 9, 1, 1, 1, 1, 8, 9, 0 - }; -} - -private static final byte _JSON_float_indicies[] = init__JSON_float_indicies_0(); - - -private static byte[] init__JSON_float_trans_targs_0() -{ - return new byte [] { - 2, 0, 3, 7, 4, 5, 8, 6, 10, 9 - }; -} - -private static final byte _JSON_float_trans_targs[] = init__JSON_float_trans_targs_0(); - - -private static byte[] init__JSON_float_trans_actions_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 - }; -} - -private static final byte _JSON_float_trans_actions[] = init__JSON_float_trans_actions_0(); - - -static final int JSON_float_start = 1; -static final int JSON_float_first_final = 8; -static final int JSON_float_error = 0; - -static final int JSON_float_en_main = 1; - - -// line 547 "ParserConfig.rl" - - - void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { - int new_p = parseFloatInternal(p, pe); - if (new_p == -1) { - res.update(null, p); - return; - } - final ByteList num = absSubSequence(p, new_p); - IRubyObject number = config.decimalFactory.apply(context, num); - - res.update(config.onLoad(context, number), new_p + 1); - } - - int parseFloatInternal(int p, int pe) { - int cs; - - -// line 1064 "ParserConfig.java" - { - cs = JSON_float_start; - } - -// line 565 "ParserConfig.rl" - int memo = p; - -// line 1072 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _match: do { - _keys = _JSON_float_key_offsets[cs]; - _trans = _JSON_float_index_offsets[cs]; - _klen = _JSON_float_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( data[p] < _JSON_float_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( data[p] > _JSON_float_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_float_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( data[p] < _JSON_float_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( data[p] > _JSON_float_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_float_indicies[_trans]; - cs = _JSON_float_trans_targs[_trans]; - - if ( _JSON_float_trans_actions[_trans] != 0 ) { - _acts = _JSON_float_trans_actions[_trans]; - _nacts = (int) _JSON_float_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_float_actions[_acts++] ) - { - case 0: -// line 538 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 1159 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 567 "ParserConfig.rl" - - if (cs < JSON_float_first_final) { - return -1; - } - - return p; - } - - -// line 1189 "ParserConfig.java" -private static byte[] init__JSON_string_actions_0() -{ - return new byte [] { - 0, 2, 0, 1 - }; -} - -private static final byte _JSON_string_actions[] = init__JSON_string_actions_0(); - - -private static byte[] init__JSON_string_key_offsets_0() -{ - return new byte [] { - 0, 0, 1, 3, 4, 10, 16, 22, 28 - }; -} - -private static final byte _JSON_string_key_offsets[] = init__JSON_string_key_offsets_0(); - - -private static char[] init__JSON_string_trans_keys_0() -{ - return new char [] { - 34, 34, 92, 117, 48, 57, 65, 70, 97, 102, 48, 57, - 65, 70, 97, 102, 48, 57, 65, 70, 97, 102, 48, 57, - 65, 70, 97, 102, 0 - }; -} - -private static final char _JSON_string_trans_keys[] = init__JSON_string_trans_keys_0(); - - -private static byte[] init__JSON_string_single_lengths_0() -{ - return new byte [] { - 0, 1, 2, 1, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_string_single_lengths[] = init__JSON_string_single_lengths_0(); - - -private static byte[] init__JSON_string_range_lengths_0() -{ - return new byte [] { - 0, 0, 0, 0, 3, 3, 3, 3, 0 - }; -} - -private static final byte _JSON_string_range_lengths[] = init__JSON_string_range_lengths_0(); - - -private static byte[] init__JSON_string_index_offsets_0() -{ - return new byte [] { - 0, 0, 2, 5, 7, 11, 15, 19, 23 - }; -} - -private static final byte _JSON_string_index_offsets[] = init__JSON_string_index_offsets_0(); - - -private static byte[] init__JSON_string_indicies_0() -{ - return new byte [] { - 0, 1, 2, 3, 0, 4, 0, 5, 5, 5, 1, 6, - 6, 6, 1, 7, 7, 7, 1, 0, 0, 0, 1, 1, - 0 - }; -} - -private static final byte _JSON_string_indicies[] = init__JSON_string_indicies_0(); - - -private static byte[] init__JSON_string_trans_targs_0() -{ - return new byte [] { - 2, 0, 8, 3, 4, 5, 6, 7 - }; -} - -private static final byte _JSON_string_trans_targs[] = init__JSON_string_trans_targs_0(); - - -private static byte[] init__JSON_string_trans_actions_0() -{ - return new byte [] { - 0, 0, 1, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_string_trans_actions[] = init__JSON_string_trans_actions_0(); - - -static final int JSON_string_start = 1; -static final int JSON_string_first_final = 8; -static final int JSON_string_error = 0; - -static final int JSON_string_en_main = 1; - - -// line 606 "ParserConfig.rl" - - - void parseString(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject result = null; - - -// line 1299 "ParserConfig.java" - { - cs = JSON_string_start; - } - -// line 613 "ParserConfig.rl" - int memo = p; - -// line 1307 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _match: do { - _keys = _JSON_string_key_offsets[cs]; - _trans = _JSON_string_index_offsets[cs]; - _klen = _JSON_string_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( data[p] < _JSON_string_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( data[p] > _JSON_string_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_string_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( data[p] < _JSON_string_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( data[p] > _JSON_string_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_string_indicies[_trans]; - cs = _JSON_string_trans_targs[_trans]; - - if ( _JSON_string_trans_actions[_trans] != 0 ) { - _acts = _JSON_string_trans_actions[_trans]; - _nacts = (int) _JSON_string_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_string_actions[_acts++] ) - { - case 0: -// line 581 "ParserConfig.rl" - { - int offset = byteList.begin(); - ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, - p - offset); - result = context.runtime.newString(decoded); - if (result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - {p = (( p + 1))-1;} - } - } - break; - case 1: -// line 594 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 1409 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 615 "ParserConfig.rl" - - if (cs >= JSON_string_first_final && result != null) { - if (result instanceof RubyString) { - RubyString string = (RubyString)result; - string.setEncoding(UTF8Encoding.INSTANCE); - string.clearCodeRange(); - if (config.freeze) { - string.setFrozen(true); - string = context.runtime.freezeAndDedupString(string); - } - res.update(config.onLoad(context, string), p + 1); - } else { - res.update(config.onLoad(context, result), p + 1); - } - } else { - res.update(null, p + 1); - } - } - - -// line 1450 "ParserConfig.java" -private static byte[] init__JSON_array_actions_0() -{ - return new byte [] { - 0, 1, 0, 1, 1, 1, 2 - }; -} - -private static final byte _JSON_array_actions[] = init__JSON_array_actions_0(); - - -private static byte[] init__JSON_array_cond_offsets_0() -{ - return new byte [] { - 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 6, 6, 6, 6, 6, 8, 11, 16, 19 - }; -} - -private static final byte _JSON_array_cond_offsets[] = init__JSON_array_cond_offsets_0(); - - -private static byte[] init__JSON_array_cond_lengths_0() -{ - return new byte [] { - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 5, 0, 0, 0, 0, 2, 3, 5, 3, 0 - }; -} - -private static final byte _JSON_array_cond_lengths[] = init__JSON_array_cond_lengths_0(); - - -private static int[] init__JSON_array_cond_keys_0() -{ - return new int [] { - 44, 44, 9, 9, 10, 10, 13, 13, 32, 32, 47, 47, - 42, 42, 47, 47, 0, 41, 42, 42, 43,65535, 0, 41, - 42, 42, 43, 46, 47, 47, 48,65535, 0, 9, 10, 10, - 11,65535, 0 - }; -} - -private static final int _JSON_array_cond_keys[] = init__JSON_array_cond_keys_0(); - - -private static byte[] init__JSON_array_cond_spaces_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_array_cond_spaces[] = init__JSON_array_cond_spaces_0(); - - -private static byte[] init__JSON_array_key_offsets_0() -{ - return new byte [] { - 0, 0, 1, 18, 26, 28, 29, 31, 32, 48, 50, 51, - 53, 54, 76, 78, 79, 81, 82, 86, 92, 100, 106 - }; -} - -private static final byte _JSON_array_key_offsets[] = init__JSON_array_key_offsets_0(); - - -private static int[] init__JSON_array_trans_keys_0() -{ - return new int [] { - 91, 13, 32, 34, 45, 47, 73, 78, 91, 93, 102, 110, - 116, 123, 9, 10, 48, 57, 13, 32, 47, 93,65580,131116, - 9, 10, 42, 47, 42, 42, 47, 10, 13, 32, 34, 45, - 47, 73, 78, 91, 102, 110, 116, 123, 9, 10, 48, 57, - 42, 47, 42, 42, 47, 10, 34, 45, 73, 78, 91, 93, - 102, 110, 116, 123,65549,65568,65583,131085,131104,131119, 48, 57, - 65545,65546,131081,131082, 42, 47, 42, 42, 47, 10,65578,65583, - 131114,131119,65578,131114,65536,131071,131072,196607,65578,65583,131114,131119, - 65536,131071,131072,196607,65546,131082,65536,131071,131072,196607, 0 - }; -} - -private static final int _JSON_array_trans_keys[] = init__JSON_array_trans_keys_0(); - - -private static byte[] init__JSON_array_single_lengths_0() -{ - return new byte [] { - 0, 1, 13, 6, 2, 1, 2, 1, 12, 2, 1, 2, - 1, 16, 2, 1, 2, 1, 4, 2, 4, 2, 0 - }; -} - -private static final byte _JSON_array_single_lengths[] = init__JSON_array_single_lengths_0(); - - -private static byte[] init__JSON_array_range_lengths_0() -{ - return new byte [] { - 0, 0, 2, 1, 0, 0, 0, 0, 2, 0, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 2, 2, 2, 0 - }; -} - -private static final byte _JSON_array_range_lengths[] = init__JSON_array_range_lengths_0(); - - -private static short[] init__JSON_array_index_offsets_0() -{ - return new short [] { - 0, 0, 2, 18, 26, 29, 31, 34, 36, 51, 54, 56, - 59, 61, 81, 84, 86, 89, 91, 96, 101, 108, 113 - }; -} - -private static final short _JSON_array_index_offsets[] = init__JSON_array_index_offsets_0(); - - -private static byte[] init__JSON_array_indicies_0() -{ - return new byte [] { - 0, 1, 0, 0, 2, 2, 3, 2, 2, 2, 4, 2, - 2, 2, 2, 0, 2, 1, 5, 5, 6, 4, 7, 8, - 5, 1, 9, 10, 1, 11, 9, 11, 5, 9, 12, 10, - 7, 7, 2, 2, 13, 2, 2, 2, 2, 2, 2, 2, - 7, 2, 1, 14, 15, 1, 16, 14, 16, 7, 14, 17, - 15, 2, 2, 2, 2, 2, 4, 2, 2, 2, 2, 0, - 0, 3, 8, 8, 18, 2, 0, 8, 1, 19, 20, 1, - 21, 19, 21, 0, 19, 22, 20, 19, 20, 23, 24, 1, - 21, 25, 19, 23, 1, 21, 0, 25, 8, 19, 23, 1, - 22, 26, 20, 24, 1, 1, 0 - }; -} - -private static final byte _JSON_array_indicies[] = init__JSON_array_indicies_0(); - - -private static byte[] init__JSON_array_trans_targs_0() -{ - return new byte [] { - 2, 0, 3, 14, 22, 3, 4, 8, 13, 5, 7, 6, - 3, 9, 10, 12, 11, 8, 18, 15, 17, 16, 2, 19, - 21, 20, 13 - }; -} - -private static final byte _JSON_array_trans_targs[] = init__JSON_array_trans_targs_0(); - - -private static byte[] init__JSON_array_trans_actions_0() -{ - return new byte [] { - 0, 0, 3, 0, 5, 0, 0, 0, 0, 0, 0, 1, - 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, - 0, 1, 1 - }; -} - -private static final byte _JSON_array_trans_actions[] = init__JSON_array_trans_actions_0(); - - -static final int JSON_array_start = 1; -static final int JSON_array_first_final = 22; -static final int JSON_array_error = 0; - -static final int JSON_array_en_main = 1; - - -// line 669 "ParserConfig.rl" - - - void parseArray(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - - if (config.maxNesting > 0 && currentNesting > config.maxNesting) { - throw newException(context, Utils.M_NESTING_ERROR, - "nesting of " + currentNesting + " is too deep"); - } - - IRubyObject result = RubyArray.newArray(context.runtime); - - -// line 1633 "ParserConfig.java" - { - cs = JSON_array_start; - } - -// line 682 "ParserConfig.rl" - -// line 1640 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _widec; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _widec = data[p]; - _keys = _JSON_array_cond_offsets[cs]*2 -; _klen = _JSON_array_cond_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys -; int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( _widec < _JSON_array_cond_keys[_mid] ) - _upper = _mid - 2; - else if ( _widec > _JSON_array_cond_keys[_mid+1] ) - _lower = _mid + 2; - else { - switch ( _JSON_array_cond_spaces[_JSON_array_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { - case 0: { - _widec = 65536 + (data[p] - 0); - if ( -// line 640 "ParserConfig.rl" - config.allowTrailingComma ) _widec += 65536; - break; - } - } - break; - } - } - } - - _match: do { - _keys = _JSON_array_key_offsets[cs]; - _trans = _JSON_array_index_offsets[cs]; - _klen = _JSON_array_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( _widec < _JSON_array_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( _widec > _JSON_array_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_array_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( _widec < _JSON_array_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( _widec > _JSON_array_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_array_indicies[_trans]; - cs = _JSON_array_trans_targs[_trans]; - - if ( _JSON_array_trans_actions[_trans] != 0 ) { - _acts = _JSON_array_trans_actions[_trans]; - _nacts = (int) _JSON_array_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_array_actions[_acts++] ) - { - case 0: -// line 321 "ParserConfig.rl" - { - if (!config.allowComments) { - if (config.deprecateComments) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" - ); - } - } else { - throw unexpectedToken(context, p, pe); - } - } - } - break; - case 1: -// line 642 "ParserConfig.rl" - { - parseValue(context, res, p, pe); - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - ((RubyArray)result).append(res.result); - {p = (( res.p))-1;} - } - } - break; - case 2: -// line 653 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 1789 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 683 "ParserConfig.rl" - - if (cs >= JSON_array_first_final) { - res.update(config.onLoad(context, result), p + 1); - } else { - throw unexpectedToken(context, p, pe); - } - } - - -// line 1819 "ParserConfig.java" -private static byte[] init__JSON_object_actions_0() -{ - return new byte [] { - 0, 1, 0, 1, 1, 1, 2, 1, 3 - }; -} - -private static final byte _JSON_object_actions[] = init__JSON_object_actions_0(); - - -private static byte[] init__JSON_object_cond_offsets_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, - 6, 6, 6, 6, 6, 6, 6, 6, 6, 8, 11, 16, - 19, 19, 19, 19, 19, 19, 19, 19, 19 - }; -} - -private static final byte _JSON_object_cond_offsets[] = init__JSON_object_cond_offsets_0(); - - -private static byte[] init__JSON_object_cond_lengths_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 3, - 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_object_cond_lengths[] = init__JSON_object_cond_lengths_0(); - - -private static int[] init__JSON_object_cond_keys_0() -{ - return new int [] { - 9, 9, 10, 10, 13, 13, 32, 32, 44, 44, 47, 47, - 42, 42, 47, 47, 0, 41, 42, 42, 43,65535, 0, 41, - 42, 42, 43, 46, 47, 47, 48,65535, 0, 9, 10, 10, - 11,65535, 0 - }; -} - -private static final int _JSON_object_cond_keys[] = init__JSON_object_cond_keys_0(); - - -private static byte[] init__JSON_object_cond_spaces_0() -{ - return new byte [] { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_object_cond_spaces[] = init__JSON_object_cond_spaces_0(); - - -private static byte[] init__JSON_object_key_offsets_0() -{ - return new byte [] { - 0, 0, 1, 8, 14, 16, 17, 19, 20, 36, 49, 56, - 62, 64, 65, 67, 68, 70, 71, 73, 74, 78, 84, 92, - 98, 100, 101, 103, 104, 106, 107, 109, 110 - }; -} - -private static final byte _JSON_object_key_offsets[] = init__JSON_object_key_offsets_0(); - - -private static int[] init__JSON_object_trans_keys_0() -{ - return new int [] { - 123, 13, 32, 34, 47, 125, 9, 10, 13, 32, 47, 58, - 9, 10, 42, 47, 42, 42, 47, 10, 13, 32, 34, 45, - 47, 73, 78, 91, 102, 110, 116, 123, 9, 10, 48, 57, - 125,65549,65568,65580,65583,131085,131104,131116,131119,65545,65546,131081, - 131082, 13, 32, 44, 47, 125, 9, 10, 13, 32, 34, 47, - 9, 10, 42, 47, 42, 42, 47, 10, 42, 47, 42, 42, - 47, 10,65578,65583,131114,131119,65578,131114,65536,131071,131072,196607, - 65578,65583,131114,131119,65536,131071,131072,196607,65546,131082,65536,131071, - 131072,196607, 42, 47, 42, 42, 47, 10, 42, 47, 42, 42, - 47, 10, 0 - }; -} - -private static final int _JSON_object_trans_keys[] = init__JSON_object_trans_keys_0(); - - -private static byte[] init__JSON_object_single_lengths_0() -{ - return new byte [] { - 0, 1, 5, 4, 2, 1, 2, 1, 12, 9, 5, 4, - 2, 1, 2, 1, 2, 1, 2, 1, 4, 2, 4, 2, - 2, 1, 2, 1, 2, 1, 2, 1, 0 - }; -} - -private static final byte _JSON_object_single_lengths[] = init__JSON_object_single_lengths_0(); - - -private static byte[] init__JSON_object_range_lengths_0() -{ - return new byte [] { - 0, 0, 1, 1, 0, 0, 0, 0, 2, 2, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; -} - -private static final byte _JSON_object_range_lengths[] = init__JSON_object_range_lengths_0(); - - -private static short[] init__JSON_object_index_offsets_0() -{ - return new short [] { - 0, 0, 2, 9, 15, 18, 20, 23, 25, 40, 52, 59, - 65, 68, 70, 73, 75, 78, 80, 83, 85, 90, 95, 102, - 107, 110, 112, 115, 117, 120, 122, 125, 127 - }; -} - -private static final short _JSON_object_index_offsets[] = init__JSON_object_index_offsets_0(); - - -private static byte[] init__JSON_object_indicies_0() -{ - return new byte [] { - 0, 1, 0, 0, 2, 3, 4, 0, 1, 5, 5, 6, - 7, 5, 1, 8, 9, 1, 10, 8, 10, 5, 8, 11, - 9, 7, 7, 12, 12, 13, 12, 12, 12, 12, 12, 12, - 12, 7, 12, 1, 4, 14, 14, 15, 16, 17, 17, 0, - 18, 14, 17, 1, 14, 14, 15, 16, 4, 14, 1, 15, - 15, 2, 19, 15, 1, 20, 21, 1, 22, 20, 22, 15, - 20, 23, 21, 24, 25, 1, 26, 24, 26, 14, 24, 27, - 25, 24, 25, 28, 29, 1, 26, 30, 24, 28, 1, 26, - 14, 30, 17, 24, 28, 1, 27, 31, 25, 29, 1, 32, - 33, 1, 34, 32, 34, 7, 32, 35, 33, 36, 37, 1, - 38, 36, 38, 0, 36, 39, 37, 1, 0 - }; -} - -private static final byte _JSON_object_indicies[] = init__JSON_object_indicies_0(); - - -private static byte[] init__JSON_object_trans_targs_0() -{ - return new byte [] { - 2, 0, 3, 28, 32, 3, 4, 8, 5, 7, 6, 3, - 9, 24, 10, 11, 16, 9, 20, 12, 13, 15, 14, 11, - 17, 19, 18, 10, 21, 23, 22, 9, 25, 27, 26, 8, - 29, 31, 30, 2 - }; -} - -private static final byte _JSON_object_trans_targs[] = init__JSON_object_trans_targs_0(); - - -private static byte[] init__JSON_object_trans_actions_0() -{ - return new byte [] { - 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 1, 1, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, - 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, - 0, 0, 1, 1 - }; -} - -private static final byte _JSON_object_trans_actions[] = init__JSON_object_trans_actions_0(); - - -static final int JSON_object_start = 1; -static final int JSON_object_first_final = 32; -static final int JSON_object_error = 0; - -static final int JSON_object_en_main = 1; - - -// line 754 "ParserConfig.rl" - - - void parseObject(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject lastName = null; - - if (config.maxNesting > 0 && currentNesting > config.maxNesting) { - throw newException(context, Utils.M_NESTING_ERROR, - "nesting of " + currentNesting + " is too deep"); - } - - // this is guaranteed to be a RubyHash due to the earlier - // allocator test at OptionsReader#getClass - IRubyObject result = RubyHash.newHash(context.runtime); - - -// line 2015 "ParserConfig.java" - { - cs = JSON_object_start; - } - -// line 770 "ParserConfig.rl" - -// line 2022 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _widec; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _widec = data[p]; - _keys = _JSON_object_cond_offsets[cs]*2 -; _klen = _JSON_object_cond_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys -; int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( _widec < _JSON_object_cond_keys[_mid] ) - _upper = _mid - 2; - else if ( _widec > _JSON_object_cond_keys[_mid+1] ) - _lower = _mid + 2; - else { - switch ( _JSON_object_cond_spaces[_JSON_object_cond_offsets[cs] + ((_mid - _keys)>>1)] ) { - case 0: { - _widec = 65536 + (data[p] - 0); - if ( -// line 697 "ParserConfig.rl" - config.allowTrailingComma ) _widec += 65536; - break; - } - } - break; - } - } - } - - _match: do { - _keys = _JSON_object_key_offsets[cs]; - _trans = _JSON_object_index_offsets[cs]; - _klen = _JSON_object_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( _widec < _JSON_object_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( _widec > _JSON_object_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_object_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( _widec < _JSON_object_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( _widec > _JSON_object_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_object_indicies[_trans]; - cs = _JSON_object_trans_targs[_trans]; - - if ( _JSON_object_trans_actions[_trans] != 0 ) { - _acts = _JSON_object_trans_actions[_trans]; - _nacts = (int) _JSON_object_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_object_actions[_acts++] ) - { - case 0: -// line 321 "ParserConfig.rl" - { - if (!config.allowComments) { - if (config.deprecateComments) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" - ); - } - } else { - throw unexpectedToken(context, p, pe); - } - } - } - break; - case 1: -// line 699 "ParserConfig.rl" - { - parseValue(context, res, p, pe); - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - ((RubyHash)result).op_aset(context, lastName, res.result); - {p = (( res.p))-1;} - } - } - break; - case 2: -// line 710 "ParserConfig.rl" - { - parseString(context, res, p, pe); - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - RubyString name = (RubyString)res.result; - if (config.symbolizeNames) { - lastName = name.intern(); - } else { - lastName = name; - } - - if (!config.allowDuplicateKey) { - if (((RubyHash)result).hasKey(lastName)) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" - ); - } else { - throw parsingError(context, "duplicate key" + name.inspect(), p, pe); - } - } - } - - {p = (( res.p))-1;} - } - } - break; - case 3: -// line 740 "ParserConfig.rl" - { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } - break; -// line 2203 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 771 "ParserConfig.rl" - - if (cs < JSON_object_first_final) { - res.update(null, p + 1); - return; - } - - res.update(config.onLoad(context, result), p + 1); - } - - -// line 2234 "ParserConfig.java" -private static byte[] init__JSON_actions_0() -{ - return new byte [] { - 0, 1, 0, 1, 1 - }; -} - -private static final byte _JSON_actions[] = init__JSON_actions_0(); - - -private static byte[] init__JSON_key_offsets_0() -{ - return new byte [] { - 0, 0, 16, 18, 19, 21, 22, 24, 25, 27, 28 - }; -} - -private static final byte _JSON_key_offsets[] = init__JSON_key_offsets_0(); - - -private static char[] init__JSON_trans_keys_0() -{ - return new char [] { - 13, 32, 34, 45, 47, 73, 78, 91, 102, 110, 116, 123, - 9, 10, 48, 57, 42, 47, 42, 42, 47, 10, 42, 47, - 42, 42, 47, 10, 13, 32, 47, 9, 10, 0 - }; -} - -private static final char _JSON_trans_keys[] = init__JSON_trans_keys_0(); - - -private static byte[] init__JSON_single_lengths_0() -{ - return new byte [] { - 0, 12, 2, 1, 2, 1, 2, 1, 2, 1, 3 - }; -} - -private static final byte _JSON_single_lengths[] = init__JSON_single_lengths_0(); - - -private static byte[] init__JSON_range_lengths_0() -{ - return new byte [] { - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1 - }; -} - -private static final byte _JSON_range_lengths[] = init__JSON_range_lengths_0(); - - -private static byte[] init__JSON_index_offsets_0() -{ - return new byte [] { - 0, 0, 15, 18, 20, 23, 25, 28, 30, 33, 35 - }; -} - -private static final byte _JSON_index_offsets[] = init__JSON_index_offsets_0(); - - -private static byte[] init__JSON_indicies_0() -{ - return new byte [] { - 0, 0, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, - 0, 2, 1, 4, 5, 1, 6, 4, 6, 7, 4, 8, - 5, 9, 10, 1, 11, 9, 11, 0, 9, 12, 10, 7, - 7, 13, 7, 1, 0 - }; -} - -private static final byte _JSON_indicies[] = init__JSON_indicies_0(); - - -private static byte[] init__JSON_trans_targs_0() -{ - return new byte [] { - 1, 0, 10, 6, 3, 5, 4, 10, 10, 7, 9, 8, - 1, 2 - }; -} - -private static final byte _JSON_trans_targs[] = init__JSON_trans_targs_0(); - - -private static byte[] init__JSON_trans_actions_0() -{ - return new byte [] { - 0, 0, 3, 0, 0, 0, 1, 0, 1, 0, 0, 1, - 1, 0 - }; -} - -private static final byte _JSON_trans_actions[] = init__JSON_trans_actions_0(); - - -static final int JSON_start = 1; -static final int JSON_first_final = 10; -static final int JSON_error = 0; - -static final int JSON_en_main = 1; - - -// line 800 "ParserConfig.rl" - - - public IRubyObject parseImplementation(ThreadContext context) { - int cs; - int p, pe; - IRubyObject result = null; - ParserResult res = new ParserResult(); - - -// line 2349 "ParserConfig.java" - { - cs = JSON_start; - } - -// line 809 "ParserConfig.rl" - p = byteList.begin(); - pe = p + byteList.length(); - -// line 2358 "ParserConfig.java" - { - int _klen; - int _trans = 0; - int _acts; - int _nacts; - int _keys; - int _goto_targ = 0; - - _goto: while (true) { - switch ( _goto_targ ) { - case 0: - if ( p == pe ) { - _goto_targ = 4; - continue _goto; - } - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } -case 1: - _match: do { - _keys = _JSON_key_offsets[cs]; - _trans = _JSON_index_offsets[cs]; - _klen = _JSON_single_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + _klen - 1; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + ((_upper-_lower) >> 1); - if ( data[p] < _JSON_trans_keys[_mid] ) - _upper = _mid - 1; - else if ( data[p] > _JSON_trans_keys[_mid] ) - _lower = _mid + 1; - else { - _trans += (_mid - _keys); - break _match; - } - } - _keys += _klen; - _trans += _klen; - } - - _klen = _JSON_range_lengths[cs]; - if ( _klen > 0 ) { - int _lower = _keys; - int _mid; - int _upper = _keys + (_klen<<1) - 2; - while (true) { - if ( _upper < _lower ) - break; - - _mid = _lower + (((_upper-_lower) >> 1) & ~1); - if ( data[p] < _JSON_trans_keys[_mid] ) - _upper = _mid - 2; - else if ( data[p] > _JSON_trans_keys[_mid+1] ) - _lower = _mid + 2; - else { - _trans += ((_mid - _keys)>>1); - break _match; - } - } - _trans += _klen; - } - } while (false); - - _trans = _JSON_indicies[_trans]; - cs = _JSON_trans_targs[_trans]; - - if ( _JSON_trans_actions[_trans] != 0 ) { - _acts = _JSON_trans_actions[_trans]; - _nacts = (int) _JSON_actions[_acts++]; - while ( _nacts-- > 0 ) - { - switch ( _JSON_actions[_acts++] ) - { - case 0: -// line 321 "ParserConfig.rl" - { - if (!config.allowComments) { - if (config.deprecateComments) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" - ); - } - } else { - throw unexpectedToken(context, p, pe); - } - } - } - break; - case 1: -// line 786 "ParserConfig.rl" - { - parseValue(context, res, p, pe); - if (res.result == null) { - p--; - { p += 1; _goto_targ = 5; if (true) continue _goto;} - } else { - result = res.result; - {p = (( res.p))-1;} - } - } - break; -// line 2468 "ParserConfig.java" - } - } - } - -case 2: - if ( cs == 0 ) { - _goto_targ = 5; - continue _goto; - } - if ( ++p != pe ) { - _goto_targ = 1; - continue _goto; - } -case 4: -case 5: - } - break; } - } - -// line 812 "ParserConfig.rl" - - if (cs >= JSON_first_final && p == pe) { - return result; - } else { - throw unexpectedToken(context, p, pe); - } - } - - public IRubyObject parse(ThreadContext context) { - return parseImplementation(context); - } - - /** - * Updates the "view" bytelist with the new offsets and returns it. - * @param absStart - * @param absEnd - */ - private ByteList absSubSequence(int absStart, int absEnd) { - view.setBegin(absStart); - view.setRealSize(absEnd - absStart); - return view; - } - - /** - * Retrieves a constant directly descended from the JSON module. - * @param name The constant name - */ - private IRubyObject getConstant(String name) { - return config.info.jsonModule.get().getConstant(name); - } - - private RaiseException newException(ThreadContext context, String className, String message) { - return Utils.newException(context, className, message); - } - - private RaiseException newException(ThreadContext context, String className, RubyString message) { - return Utils.newException(context, className, message); - } - - RubyHash.VisitorWithState MATCH_VISITOR = new RubyHash.VisitorWithState() { - @Override - public void visit(ThreadContext context, RubyHash self, IRubyObject pattern, IRubyObject klass, int index, IRubyObject[] state) { - if (pattern.callMethod(context, "===", state[0]).isTrue()) { - state[1] = klass; - throw JumpException.SPECIAL_JUMP; - } - } - }; - } -} diff --git a/java/src/json/ext/ParserConfig.rl b/java/src/json/ext/ParserConfig.rl deleted file mode 100644 index e98c71f8c..000000000 --- a/java/src/json/ext/ParserConfig.rl +++ /dev/null @@ -1,861 +0,0 @@ -/* - * This code is copyrighted work by Daniel Luz . - * - * Distributed under the Ruby license: https://www.ruby-lang.org/en/about/license.txt - */ -package json.ext; - -import org.jcodings.Encoding; -import org.jcodings.specific.ASCIIEncoding; -import org.jcodings.specific.UTF8Encoding; -import org.jruby.Ruby; -import org.jruby.RubyArray; -import org.jruby.RubyClass; -import org.jruby.RubyFloat; -import org.jruby.RubyHash; -import org.jruby.RubyInteger; -import org.jruby.RubyObject; -import org.jruby.RubyProc; -import org.jruby.RubyString; -import org.jruby.anno.JRubyMethod; -import org.jruby.exceptions.JumpException; -import org.jruby.exceptions.RaiseException; -import org.jruby.runtime.Block; -import org.jruby.runtime.Helpers; -import org.jruby.runtime.ObjectAllocator; -import org.jruby.runtime.ThreadContext; -import org.jruby.runtime.Visibility; -import org.jruby.runtime.builtin.IRubyObject; -import org.jruby.util.ByteList; -import org.jruby.util.ConvertBytes; - -import java.util.function.BiFunction; - -import static org.jruby.util.ConvertDouble.DoubleConverter; - -/** - * The JSON::Ext::Parser class. - * - *

This is the JSON parser implemented as a Java class. To use it as the - * standard parser, set - * 

JSON.parser = JSON::Ext::Parser
- * This is performed for you when you include "json/ext". - * - *

This class does not perform the actual parsing, just acts as an interface - * to Ruby code. When the {@link #parse(ThreadContext)} method is invoked, a - * ParserConfig.ParserSession object is instantiated, which handles the process. - * - * @author mernen - */ -public class ParserConfig extends RubyObject { - private final RuntimeInfo info; - private int maxNesting; - private boolean allowNaN; - private boolean allowTrailingComma; - private boolean allowComments; - private boolean deprecateComments; - private boolean allowControlCharacters; - private boolean allowInvalidEscape; - private boolean allowDuplicateKey; - private boolean deprecateDuplicateKey; - private boolean symbolizeNames; - private boolean freeze; - private RubyProc onLoadProc; - private RubyClass decimalClass; - BiFunction decimalFactory; - private RubyHash match_string; - - private static final int DEFAULT_MAX_NESTING = 100; - - private static final ByteList JSON_MINUS_INFINITY = new ByteList(ByteList.plain("-Infinity")); - // constant names in the JSON module containing those values - private static final String CONST_NAN = "NaN"; - private static final String CONST_INFINITY = "Infinity"; - private static final String CONST_MINUS_INFINITY = "MinusInfinity"; - - static final ObjectAllocator ALLOCATOR = ParserConfig::new; - - /** - * Multiple-value return for internal parser methods. - * - *

All the parseStuff methods return instances of - * ParserResult when successful, or null when - * there's a problem with the input data. - */ - static final class ParserResult { - /** - * The result of the successful parsing. Should never be - * null. - */ - IRubyObject result; - /** - * The point where the parser returned. - */ - int p; - - void update(IRubyObject result, int p) { - this.result = result; - this.p = p; - } - } - - public ParserConfig(Ruby runtime, RubyClass metaClass) { - super(runtime, metaClass); - info = RuntimeInfo.forRuntime(runtime); - } - - /** - * ParserConfig.new(source, opts = {}) - * - *

Creates a new JSON::Ext::Parser instance for the string - * source. - * It will be configured by the opts Hash. - * opts can have the following keys: - *

- *

- *
:max_nesting - *
The maximum depth of nesting allowed in the parsed data - * structures. Disable depth checking with :max_nesting => false|nil|0, - * it defaults to 100. - *

- *

:allow_nan - *
If set to true, allow NaN, - * Infinity and -Infinity in defiance of RFC 4627 - * to be parsed by the Parser. This option defaults to false. - *

- *

:allow_trailing_comma - *
If set to true, allow arrays and objects with a trailing - * comma in defiance of RFC 4627 to be parsed by the Parser. - * This option defaults to false. - *

- *

:symbolize_names - *
If set to true, returns symbols for the names (keys) in - * a JSON object. Otherwise strings are returned, which is also the default. - *

- *

:create_additions - *
If set to false, the Parser doesn't create additions - * even if a matching class and create_id was found. This option - * defaults to true. - *

- *

:object_class - *
Defaults to Hash. If another type is provided, it will be used - * instead of Hash to represent JSON objects. The type must respond to - * new without arguments, and return an object that respond to []=. - *

- *

:array_class - *
Defaults to Array. If another type is provided, it will be used - * instead of Hash to represent JSON arrays. The type must respond to - * new without arguments, and return an object that respond to <<. - *

- *

:decimal_class - *
Specifies which class to use instead of the default (Float) when - * parsing decimal numbers. This class must accept a single string argument - * in its constructor. - *
- */ - - @JRubyMethod(name = "new", meta = true) - public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, Block block) { - ParserConfig config = (ParserConfig)((RubyClass)clazz).allocate(); - - config.callInit(arg0, block); - - return config; - } - - @JRubyMethod(name = "new", meta = true) - public static IRubyObject newInstance(IRubyObject clazz, IRubyObject arg0, IRubyObject arg1, Block block) { - ParserConfig config = (ParserConfig)((RubyClass)clazz).allocate(); - - config.callInit(arg0, arg1, block); - - return config; - } - - @JRubyMethod(visibility = Visibility.PRIVATE) - public IRubyObject initialize(ThreadContext context, IRubyObject options) { - checkFrozen(); - Ruby runtime = context.runtime; - - OptionsReader opts = new OptionsReader(context, options); - this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); - this.allowNaN = opts.getBool("allow_nan", false); - if (opts.hasKey("allow_comments")) { - this.allowComments = opts.getBool("allow_comments", false); - this.deprecateComments = false; - } else { - this.allowComments = true; - this.deprecateComments = true; - } - - this.allowControlCharacters = opts.getBool("allow_control_characters", false); - this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); - this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); - this.symbolizeNames = opts.getBool("symbolize_names", false); - if (opts.hasKey("allow_duplicate_key")) { - this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); - this.deprecateDuplicateKey = false; - } else { - this.allowDuplicateKey = false; - this.deprecateDuplicateKey = true; - } - - this.freeze = opts.getBool("freeze", false); - this.onLoadProc = opts.getProc("on_load"); - - this.decimalClass = opts.getClass("decimal_class", null); - - if (decimalClass == null) { - this.decimalFactory = this::createFloat; - } else if (decimalClass == runtime.getClass("BigDecimal")) { - this.decimalFactory = this::createBigDecimal; - } else { - this.decimalFactory = this::createCustomDecimal; - } - - return this; - } - - public IRubyObject onLoad(ThreadContext context, IRubyObject object) { - if (onLoadProc == null) { - return object; - } else { - return onLoadProc.call(context, object); - } - } - - /** - * Checks the given string's encoding. If a non-UTF-8 encoding is detected, - * a converted copy is returned. - * Returns the source string if no conversion is needed. - */ - private RubyString convertEncoding(ThreadContext context, RubyString source) { - Encoding encoding = source.getEncoding(); - if (encoding == ASCIIEncoding.INSTANCE) { - source = (RubyString) source.dup(); - source.setEncoding(UTF8Encoding.INSTANCE); - source.clearCodeRange(); - } else if (encoding != UTF8Encoding.INSTANCE) { - source = (RubyString) source.encode(context, context.runtime.getEncodingService().convertEncodingToRubyEncoding(UTF8Encoding.INSTANCE)); - } - return source; - } - - /** - * Parser#parse() - * - *

Parses the current JSON text source and returns the - * complete data structure as a result. - */ - @JRubyMethod - public IRubyObject parse(ThreadContext context, IRubyObject source) { - return new ParserSession(this, convertEncoding(context, source.convertToString()), context, info).parse(context); - } - - /** - * Queries JSON.create_id. Returns null if it is - * set to nil or false, and a String if not. - */ - private RubyString getCreateId(ThreadContext context) { - IRubyObject v = info.jsonModule.get().callMethod(context, "create_id"); - return v.isTrue() ? v.convertToString() : null; - } - - private RubyFloat createFloat(final ThreadContext context, final ByteList num) { - return RubyFloat.newFloat(context.runtime, new DoubleConverter().parse(num, true, true)); - } - - private IRubyObject createBigDecimal(final ThreadContext context, final ByteList num) { - final Ruby runtime = context.runtime; - return runtime.getKernel().callMethod(context, "BigDecimal", runtime.newString(num)); - } - - private IRubyObject createCustomDecimal(final ThreadContext context, final ByteList num) { - return decimalClass.newInstance(context, context.runtime.newString(num), Block.NULL_BLOCK); - } - - /** - * A string parsing session. - * - *

Once a ParserSession is instantiated, the source string should not - * change until the parsing is complete. The ParserSession object assumes - * the source {@link RubyString} is still associated to its original - * {@link ByteList}, which in turn must still be bound to the same - * byte[] value (and on the same offset). - */ - // Ragel uses lots of fall-through - @SuppressWarnings("fallthrough") - private static class ParserSession { - private final ParserConfig config; - private final RuntimeInfo info; - private final ByteList byteList; - private final ByteList view; - private final byte[] data; - private final StringDecoder decoder; - private int currentNesting = 0; - private int emittedDeprecations = 0; - - private ParserSession(ParserConfig config, RubyString source, ThreadContext context, RuntimeInfo info) { - this.config = config; - this.info = info; - this.byteList = source.getByteList(); - this.data = byteList.unsafeBytes(); - this.view = new ByteList(data, false); - this.decoder = new StringDecoder(config.allowControlCharacters, config.allowInvalidEscape); - } - - private RaiseException parsingError(ThreadContext context, String message, int absStart, int absEnd) { - RubyString msg = context.runtime.newString("unexpected token at '") - .cat(data, absStart, Math.min(absEnd - absStart, 32)) - .cat((byte)'\''); - return newException(context, Utils.M_PARSER_ERROR, msg); - } - - private RaiseException unexpectedToken(ThreadContext context, int absStart, int absEnd) { - return parsingError(context, "unexpected token at '", absStart, absEnd); - } - - %%{ - machine JSON_common; - - action parse_comment { - if (!config.allowComments) { - if (config.deprecateComments) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`" - ); - } - } else { - throw unexpectedToken(context, p, pe); - } - } - } - - cr = '\n'; - cr_neg = [^\n]; - ws = [ \t\r\n]; - c_comment = '/*' ( any* - (any* '*/' any* ) ) '*/' >parse_comment; - cpp_comment = '//' cr_neg* cr >parse_comment; - comment = c_comment | cpp_comment; - ignore = ws | comment; - name_separator = ':'; - value_separator = ','; - Vnull = 'null'; - Vfalse = 'false'; - Vtrue = 'true'; - VNaN = 'NaN'; - VInfinity = 'Infinity'; - VMinusInfinity = '-Infinity'; - begin_value = [nft"\-[{NI] | digit; - begin_object = '{'; - end_object = '}'; - begin_array = '['; - end_array = ']'; - begin_string = '"'; - begin_name = begin_string; - begin_number = digit | '-'; - - }%% - - %%{ - machine JSON_value; - include JSON_common; - - write data; - - action parse_null { - result = context.nil; - } - action parse_false { - result = context.fals; - } - action parse_true { - result = context.tru; - } - action parse_nan { - if (config.allowNaN) { - result = getConstant(CONST_NAN); - } else { - throw unexpectedToken(context, p - 2, pe); - } - } - action parse_infinity { - if (config.allowNaN) { - result = getConstant(CONST_INFINITY); - } else { - throw unexpectedToken(context, p - 7, pe); - } - } - action parse_number { - if (pe > fpc + 8 && - absSubSequence(fpc, fpc + 9).equals(JSON_MINUS_INFINITY)) { - - if (config.allowNaN) { - result = getConstant(CONST_MINUS_INFINITY); - fexec p + 10; - fhold; - fbreak; - } else { - throw unexpectedToken(context, p, pe); - } - } - parseFloat(context, res, fpc, pe); - if (res.result != null) { - result = res.result; - fexec res.p; - } - parseInteger(context, res, fpc, pe); - if (res.result != null) { - result = res.result; - fexec res.p; - } - fhold; - fbreak; - } - action parse_string { - parseString(context, res, fpc, pe); - if (res.result == null) { - fhold; - fbreak; - } else { - result = res.result; - fexec res.p; - } - } - action parse_array { - currentNesting++; - parseArray(context, res, fpc, pe); - currentNesting--; - if (res.result == null) { - fhold; - fbreak; - } else { - result = res.result; - fexec res.p; - } - } - action parse_object { - currentNesting++; - parseObject(context, res, fpc, pe); - currentNesting--; - if (res.result == null) { - fhold; - fbreak; - } else { - result = res.result; - fexec res.p; - } - } - action exit { - fhold; - fbreak; - } - - main := ( Vnull @parse_null | - Vfalse @parse_false | - Vtrue @parse_true | - VNaN @parse_nan | - VInfinity @parse_infinity | - begin_number >parse_number | - begin_string >parse_string | - begin_array >parse_array | - begin_object >parse_object - ) %*exit; - }%% - - void parseValue(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject result = null; - - %% write init; - %% write exec; - - if (cs >= JSON_value_first_final && result != null) { - if (config.freeze) { - result.setFrozen(true); - } - res.update(result, p); - } else { - res.update(null, p); - } - } - - %%{ - machine JSON_integer; - - write data; - - action exit { - fhold; - fbreak; - } - - main := '-'? ( '0' | [1-9][0-9]* ) ( ^[0-9]? @exit ); - }%% - - void parseInteger(ThreadContext context, ParserResult res, int p, int pe) { - int new_p = parseIntegerInternal(p, pe); - if (new_p == -1) { - res.update(null, p); - return; - } - RubyInteger number = createInteger(context, p, new_p); - res.update(config.onLoad(context, number), new_p + 1); - } - - int parseIntegerInternal(int p, int pe) { - int cs; - - %% write init; - int memo = p; - %% write exec; - - if (cs < JSON_integer_first_final) { - return -1; - } - - return p; - } - - RubyInteger createInteger(ThreadContext context, int p, int new_p) { - Ruby runtime = context.runtime; - ByteList num = absSubSequence(p, new_p); - return bytesToInum(runtime, num); - } - - RubyInteger bytesToInum(Ruby runtime, ByteList num) { - return ConvertBytes.byteListToInum(runtime, num, 10, true); - } - - %%{ - machine JSON_float; - include JSON_common; - - write data; - - action exit { - fhold; - fbreak; - } - - main := '-'? - ( ( ( '0' | [1-9][0-9]* ) '.' [0-9]+ ( [Ee] [+\-]?[0-9]+ )? ) - | ( ( '0' | [1-9][0-9]* ) ( [Ee] [+\-]? [0-9]+ ) ) ) - ( ^[0-9Ee.\-]? @exit ); - }%% - - void parseFloat(ThreadContext context, ParserResult res, int p, int pe) { - int new_p = parseFloatInternal(p, pe); - if (new_p == -1) { - res.update(null, p); - return; - } - final ByteList num = absSubSequence(p, new_p); - IRubyObject number = config.decimalFactory.apply(context, num); - - res.update(config.onLoad(context, number), new_p + 1); - } - - int parseFloatInternal(int p, int pe) { - int cs; - - %% write init; - int memo = p; - %% write exec; - - if (cs < JSON_float_first_final) { - return -1; - } - - return p; - } - - %%{ - machine JSON_string; - include JSON_common; - - write data; - - action parse_string { - int offset = byteList.begin(); - ByteList decoded = decoder.decode(context, byteList, memo + 1 - offset, - p - offset); - result = context.runtime.newString(decoded); - if (result == null) { - fhold; - fbreak; - } else { - fexec p + 1; - } - } - - action exit { - fhold; - fbreak; - } - - main := '"' - ( ( ^(["\\]) - | '\\'["\\/bfnrt] - | '\\u'[0-9a-fA-F]{4} - | '\\'^(["\\/bfnrtu]) - )* %parse_string - ) '"' @exit; - }%% - - void parseString(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject result = null; - - %% write init; - int memo = p; - %% write exec; - - if (cs >= JSON_string_first_final && result != null) { - if (result instanceof RubyString) { - RubyString string = (RubyString)result; - string.setEncoding(UTF8Encoding.INSTANCE); - string.clearCodeRange(); - if (config.freeze) { - string.setFrozen(true); - string = context.runtime.freezeAndDedupString(string); - } - res.update(config.onLoad(context, string), p + 1); - } else { - res.update(config.onLoad(context, result), p + 1); - } - } else { - res.update(null, p + 1); - } - } - - %%{ - machine JSON_array; - include JSON_common; - - write data; - - action allow_trailing_comma { config.allowTrailingComma } - - action parse_value { - parseValue(context, res, fpc, pe); - if (res.result == null) { - fhold; - fbreak; - } else { - ((RubyArray)result).append(res.result); - fexec res.p; - } - } - - action exit { - fhold; - fbreak; - } - - next_element = value_separator ignore* begin_value >parse_value; - - main := begin_array - ignore* - ( ( begin_value >parse_value - ignore* ) - ( ignore* - next_element - ignore* )* ( (value_separator ignore*) when allow_trailing_comma )? )? - ignore* - end_array @exit; - }%% - - void parseArray(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - - if (config.maxNesting > 0 && currentNesting > config.maxNesting) { - throw newException(context, Utils.M_NESTING_ERROR, - "nesting of " + currentNesting + " is too deep"); - } - - IRubyObject result = RubyArray.newArray(context.runtime); - - %% write init; - %% write exec; - - if (cs >= JSON_array_first_final) { - res.update(config.onLoad(context, result), p + 1); - } else { - throw unexpectedToken(context, p, pe); - } - } - - %%{ - machine JSON_object; - include JSON_common; - - write data; - - action allow_trailing_comma { config.allowTrailingComma } - - action parse_value { - parseValue(context, res, fpc, pe); - if (res.result == null) { - fhold; - fbreak; - } else { - ((RubyHash)result).op_aset(context, lastName, res.result); - fexec res.p; - } - } - - action parse_name { - parseString(context, res, fpc, pe); - if (res.result == null) { - fhold; - fbreak; - } else { - RubyString name = (RubyString)res.result; - if (config.symbolizeNames) { - lastName = name.intern(); - } else { - lastName = name; - } - - if (!config.allowDuplicateKey) { - if (((RubyHash)result).hasKey(lastName)) { - if (config.deprecateDuplicateKey && emittedDeprecations < 5) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "detected duplicate key " + name.inspect() + " in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`" - ); - } else { - throw parsingError(context, "duplicate key" + name.inspect(), p, pe); - } - } - } - - fexec res.p; - } - } - - action exit { - fhold; - fbreak; - } - - pair = ignore* begin_name >parse_name ignore* name_separator - ignore* begin_value >parse_value; - next_pair = ignore* value_separator pair; - - main := ( - begin_object - (pair (next_pair)*((ignore* value_separator) when allow_trailing_comma)?)? ignore* - end_object - ) @exit; - }%% - - void parseObject(ThreadContext context, ParserResult res, int p, int pe) { - int cs; - IRubyObject lastName = null; - - if (config.maxNesting > 0 && currentNesting > config.maxNesting) { - throw newException(context, Utils.M_NESTING_ERROR, - "nesting of " + currentNesting + " is too deep"); - } - - // this is guaranteed to be a RubyHash due to the earlier - // allocator test at OptionsReader#getClass - IRubyObject result = RubyHash.newHash(context.runtime); - - %% write init; - %% write exec; - - if (cs < JSON_object_first_final) { - res.update(null, p + 1); - return; - } - - res.update(config.onLoad(context, result), p + 1); - } - - %%{ - machine JSON; - include JSON_common; - - write data; - - action parse_value { - parseValue(context, res, fpc, pe); - if (res.result == null) { - fhold; - fbreak; - } else { - result = res.result; - fexec res.p; - } - } - - main := ignore* - ( begin_value >parse_value) - ignore*; - }%% - - public IRubyObject parseImplementation(ThreadContext context) { - int cs; - int p, pe; - IRubyObject result = null; - ParserResult res = new ParserResult(); - - %% write init; - p = byteList.begin(); - pe = p + byteList.length(); - %% write exec; - - if (cs >= JSON_first_final && p == pe) { - return result; - } else { - throw unexpectedToken(context, p, pe); - } - } - - public IRubyObject parse(ThreadContext context) { - return parseImplementation(context); - } - - /** - * Updates the "view" bytelist with the new offsets and returns it. - * @param absStart - * @param absEnd - */ - private ByteList absSubSequence(int absStart, int absEnd) { - view.setBegin(absStart); - view.setRealSize(absEnd - absStart); - return view; - } - - /** - * Retrieves a constant directly descended from the JSON module. - * @param name The constant name - */ - private IRubyObject getConstant(String name) { - return config.info.jsonModule.get().getConstant(name); - } - - private RaiseException newException(ThreadContext context, String className, String message) { - return Utils.newException(context, className, message); - } - - private RaiseException newException(ThreadContext context, String className, RubyString message) { - return Utils.newException(context, className, message); - } - - RubyHash.VisitorWithState MATCH_VISITOR = new RubyHash.VisitorWithState() { - @Override - public void visit(ThreadContext context, RubyHash self, IRubyObject pattern, IRubyObject klass, int index, IRubyObject[] state) { - if (pattern.callMethod(context, "===", state[0]).isTrue()) { - state[1] = klass; - throw JumpException.SPECIAL_JUMP; - } - } - }; - } -} diff --git a/java/src/json/ext/ParserService.java b/java/src/json/ext/ParserService.java index 88aa9674b..2d49bd1a4 100644 --- a/java/src/json/ext/ParserService.java +++ b/java/src/json/ext/ParserService.java @@ -27,8 +27,8 @@ public boolean basicLoad(Ruby runtime) throws IOException { RubyModule jsonExtModule = info.jsonModule.get().defineModuleUnder("Ext"); RubyClass parserConfigClass = jsonExtModule.defineClassUnder("ParserConfig", runtime.getObject(), - ParserConfig.ALLOCATOR); - parserConfigClass.defineAnnotatedMethods(ParserConfig.class); + Parser.ALLOCATOR); + parserConfigClass.defineAnnotatedMethods(Parser.class); return true; } } diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index 091b09f56..28874acdc 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -186,8 +186,7 @@ private int readHex(ThreadContext context) { } else if (digit >= 'A' && digit <= 'F') { digitValue = 10 + digit - 'A'; } else { - throw new NumberFormatException("Invalid base 16 number " - + src.subSequence(numberStart, numberStart + length)); + throw invalidUnicodeEscape(context, numberStart); } result = result * 16 + digitValue; } @@ -220,4 +219,14 @@ protected RaiseException invalidEscape(ThreadContext context) { return Utils.newException(context, Utils.M_PARSER_ERROR, context.runtime.newString(message)); } + + protected RaiseException invalidUnicodeEscape(ThreadContext context, int escapeStart) { + ByteList message = new ByteList( + ByteList.plain("incomplete unicode character escape sequence at ")); + // Point at the backslash-u that introduced the escape. + int start = Math.max(escapeStart - 2, 0); + message.append(src, start, srcEnd - start); + return Utils.newException(context, Utils.M_PARSER_ERROR, + context.runtime.newString(message)); + } } diff --git a/java/src/json/ext/StringScanner.java b/java/src/json/ext/StringScanner.java new file mode 100644 index 000000000..5ca59d177 --- /dev/null +++ b/java/src/json/ext/StringScanner.java @@ -0,0 +1,140 @@ +package json.ext; + +import java.nio.ByteBuffer; + +/** + * Scans the body of a JSON string for its closing quote, reporting whether the + * body can be copied verbatim (the "plain" fast path) or must be handed to + * {@link StringDecoder} for escape expansion and UTF-8/control validation. + * + *

The default implementation is SWAR (8 bytes per step). A vectorized + * subclass ({@code VectorizedStringScanner}) is loaded reflectively when the + * {@code jruby.json.useVectorizedParser} system property is set and the + * {@code jdk.incubator.vector} module is available; otherwise this SWAR + * implementation is used. Instances are stateless and therefore shared. + */ +class StringScanner { + /** Set in the returned bits when the whole body is plain printable ASCII. */ + static final long PLAIN_BIT = 1L << 32; + /** Returned when no closing quote is found before {@code end}. */ + static final long NOT_FOUND = -1L; + + private static final long HIGH_BITS = 0x8080808080808080L; + private static final long ONES = 0x0101010101010101L; + + private static final long SPACES = 0x2020202020202020L; + private static final long DOUBLE_QUOTES = 0x2222222222222222L; + private static final long BACKSLASHES = 0x5C5C5C5C5C5C5C5CL; + + private static final String VECTORIZED_SCANNER_CLASS = "json.ext.VectorizedStringScanner"; + private static final String USE_VECTORIZED_PARSER_PROP = "jruby.json.useVectorizedParser"; + private static final String USE_VECTORIZED_PARSER_DEFAULT = "false"; + + private static final StringScanner INSTANCE; + + static { + StringScanner scanner = new StringScanner(); + String enable = System.getProperty(USE_VECTORIZED_PARSER_PROP, USE_VECTORIZED_PARSER_DEFAULT); + if ("true".equalsIgnoreCase(enable) || "1".equals(enable)) { + try { + Class vectorized = StringScanner.class.getClassLoader() + .loadClass(VECTORIZED_SCANNER_CLASS); + scanner = (StringScanner) vectorized.getDeclaredConstructor().newInstance(); + } catch (Throwable t) { + // jdk.incubator.vector unavailable (or any load failure): + // keep the SWAR implementation. + scanner = new StringScanner(); + } + } + INSTANCE = scanner; + } + + static StringScanner getInstance() { + return INSTANCE; + } + + /** + * Scans {@code data[start..end)} for the closing quote, honouring backslash + * escapes. + * + * @param chunks a little-endian {@link ByteBuffer} over {@code data}, used + * for the 8-byte SWAR reads (the vectorized subclass reads + * {@code data} directly and ignores it). + * @return packed result: the low 32 bits hold the index of the closing + * quote, or {@code -1} ({@link #NOT_FOUND}) when none is found + * before {@code end}; {@link #PLAIN_BIT} is set when the entire body + * is plain printable ASCII (no escape, no ASCII control character, + * and no non-ASCII byte) and can be copied verbatim. + */ + long scan(byte[] data, ByteBuffer chunks, int start, int end) { + int p = start; + boolean plain = true; + + outer: + while (true) { + // SWAR: skip 8-byte chunks that contain nothing interesting. + while (p + 8 <= end) { + long x = chunks.getLong(p); + // Due to the byte-by-byte handling if we match an interesting byte, + // if we already know this is a non-ASCII-only string, we simply + // look for quotes and backslashes. + long m = plain ? stringScanMask(x) : quoteBackslashMask(x); + if (m == 0) { + p += 8; + } else { + p += Long.numberOfTrailingZeros(m) >>> 3; + break; + } + } + // If we match on a byte above and/or tail handling for <8 remaining bytes. + while (p < end) { + int b = data[p] & 0xFF; + if (b == '"') { + return ((long) p) | (plain ? PLAIN_BIT : 0L); + } + if (b == '\\') { + plain = false; + p += 2; // skip the backslash and the escaped byte + continue outer; + } + if (b < 0x20 || b >= 0x80) { + plain = false; + p++; + continue outer; + } + p++; + } + return NOT_FOUND; + } + } + + /** + * Returns a mask whose high bit (0x80) is set in every lane of {@code x} + * that needs scalar attention: an ASCII control character (< 0x20), a + * double quote, a backslash, or a non-ASCII byte (high bit set). Returns 0 + * when the whole 8-byte chunk is printable ASCII copyable verbatim. + */ + private static long stringScanMask(long x) { + long control = (x - SPACES) & ~x; // bytes < 0x20 (ASCII) + long high = x; // bit 0x80 set iff non-ASCII + long q = x ^ DOUBLE_QUOTES; + long quote = (q - ONES) & ~q; + long s = x ^ BACKSLASHES; + long bslash = (s - ONES) & ~s; + return (control | high | quote | bslash) & HIGH_BITS; + } + + /** + * Like {@link #stringScanMask} but only flags double quotes and backslashes. + * Used once a string is known to require the decoder, so the remaining scan + * for the closing quote still skips clean chunks (including multi-byte + * UTF-8) eight bytes at a time. + */ + private static long quoteBackslashMask(long x) { + long q = x ^ DOUBLE_QUOTES; + long quote = (q - ONES) & ~q; + long s = x ^ BACKSLASHES; + long bslash = (s - ONES) & ~s; + return (quote | bslash) & HIGH_BITS; + } +} diff --git a/java/src/json/ext/VectorizedStringScanner.java b/java/src/json/ext/VectorizedStringScanner.java new file mode 100644 index 000000000..8a9c1e3e7 --- /dev/null +++ b/java/src/json/ext/VectorizedStringScanner.java @@ -0,0 +1,72 @@ +package json.ext; + +import java.nio.ByteBuffer; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.VectorMask; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorSpecies; + +final class VectorizedStringScanner extends StringScanner { + private static final VectorSpecies SP = ByteVector.SPECIES_PREFERRED; + private static final ByteVector ZERO = ByteVector.zero(SP); + private static final ByteVector TWO = ByteVector.broadcast(SP, 2); + private static final ByteVector THIRTY_THREE = ByteVector.broadcast(SP, 33); + private static final ByteVector BACKSLASH = ByteVector.broadcast(SP, '\\'); + private static final ByteVector DQUOTE = ByteVector.broadcast(SP, '"'); + + @Override + long scan(byte[] data, ByteBuffer chunks, int start, int end) { + final int width = SP.length(); + int p = start; + boolean plain = true; + + // The same structure as the StringEncoder. The logic is + // duplicated for maximum inlining. + outer: + while (true) { + while (p + width <= end) { + ByteVector chunk = ByteVector.fromArray(SP, data, p); + VectorMask interesting = + plain ? interestingLanes(chunk) : quoteOrBackslashLanes(chunk); + if (interesting.anyTrue()) { + p += interesting.firstTrue(); + break; + } + p += width; + } + while (p < end) { + int b = data[p] & 0xFF; + if (b == '"') { + return ((long) p) | (plain ? PLAIN_BIT : 0L); + } + if (b == '\\') { + plain = false; + p += 2; // skip the backslash and the escaped byte + continue outer; + } + if (b < 0x20 || b >= 0x80) { + plain = false; + p++; + continue outer; + } + p++; + } + return NOT_FOUND; + } + } + + // Lanes that are control characters, double quotes, backslashes or non-ASCII. + private static VectorMask interestingLanes(ByteVector chunk) { + VectorMask negative = chunk.lt(ZERO); + VectorMask lowOrQuote = chunk.lanewise(VectorOperators.XOR, TWO) + .lt(THIRTY_THREE) + .andNot(negative); + return lowOrQuote.or(chunk.eq(BACKSLASH)).or(negative); + } + + // Lanes that are a double quote or a backslash (non-plain phase). + private static VectorMask quoteOrBackslashLanes(ByteVector chunk) { + return chunk.eq(DQUOTE).or(chunk.eq(BACKSLASH)); + } +} From 525ffbc98c10e603c71694f3490dfcdb1d28b8b9 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Tue, 16 Jun 2026 17:38:59 -0500 Subject: [PATCH 376/506] Presize the RubyHash when decoding an object. --- java/src/json/ext/Parser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 092ff4d41..c710a02d8 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -545,7 +545,7 @@ private IRubyObject decodeObject(int count) { final Ruby runtime = context.runtime; int base = valueTop - count; int limit = valueTop; - RubyHash hash = RubyHash.newHash(runtime); + RubyHash hash = new RubyHash(runtime, count / 2); for (int i = base; i < limit; i += 2) { // We use RubyHash#fastASet because all object keys have already been // frozen and deduplicated. From 85ab2784e60016594ceb304d8a6a9772bde57c6f Mon Sep 17 00:00:00 2001 From: "Thomas E. Enebo" Date: Tue, 16 Jun 2026 21:00:00 -0500 Subject: [PATCH 377/506] Use jruby_new_parser parseNumbers --- java/src/json/ext/Parser.java | 142 +++---- java/src/json/ext/Ryu.java | 770 ++++++++++++++++++++++++++++++++++ 2 files changed, 832 insertions(+), 80 deletions(-) create mode 100644 java/src/json/ext/Ryu.java diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index c710a02d8..05c394b20 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -34,6 +34,7 @@ import java.util.Arrays; import java.util.function.BiFunction; +import static json.ext.Ryu.ryuS2dFromParts; import static org.jruby.util.ConvertDouble.DoubleConverter; /** @@ -318,7 +319,8 @@ private IRubyObject run() { case VALUE: { eatWhitespace(); IRubyObject value; - switch (peek()) { + byte c = peek(); + switch (c) { case 'n': if (matchKeyword("null")) { value = context.nil; break; } throw unexpectedToken(cursor, end); @@ -345,12 +347,12 @@ private IRubyObject run() { if (config.allowNaN && matchKeyword("Infinity")) { value = getConstant(CONST_MINUS_INFINITY); } else { - value = parseNumber(cursor - 1); + value = parseNumber(cursor - 1, peek(), true); } break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': - value = parseNumber(cursor); + value = parseNumber(cursor, c, false); break; case '"': value = parseString(false); @@ -595,108 +597,84 @@ private void onDuplicateKey(IRubyObject key) { } } - private IRubyObject parseNumber(int numberStart) { - byte first = cursor < end ? data[cursor] : 0; - boolean negative = data[numberStart] == '-'; + private int parseDigits(long value) { + int start = cursor; + byte c = peek(); + for (; c >= '0' && c <= '9'; c = advance()) { + value = value * 10 + (c - '0'); + } + digitsValue = value; + return cursor - start; + } - int intStart = cursor; - int p = scanDigits(intStart); - int intDigits = p - intStart; + private IRubyObject parseNumber(int numberStart, byte first, boolean negative) { + int mantissaDigits = parseDigits(0); long mantissa = digitsValue; - if ((first == '0' && intDigits > 1) || intDigits == 0) { - cursor = p; + if ((first == '0' && mantissaDigits > 1) || mantissaDigits == 0) { throw unexpectedToken(numberStart, end); } boolean integer = true; + int decimal_point_pos = -1; - if (p < end && data[p] == '.') { + if (peek() == '.') { integer = false; - p++; - int fracStart = p; - p = scanDigits(fracStart); - if (p == fracStart) { - cursor = p; + decimal_point_pos = mantissaDigits; + cursor++; + int fracDigits = parseDigits(mantissa); + if (fracDigits == 0) { throw unexpectedToken(numberStart, end); } + mantissaDigits += fracDigits; + mantissa = digitsValue; } - if (p < end && (data[p] == 'e' || data[p] == 'E')) { + int c = peek(); + long exponent = 0; + if (c == 'e' || c == 'E') { integer = false; - p++; - if (p < end && (data[p] == '+' || data[p] == '-')) p++; - int expStart = p; - p = scanDigits(expStart); - if (p == expStart) { - cursor = p; + c = advance(); + boolean negative_exponent = c == '-'; + if (negative_exponent || c == '+') advance(); + + int exponent_digits = parseDigits(0); + long abs_exponent = digitsValue; + if (exponent_digits == 0) { throw unexpectedToken(numberStart, end); } - } - cursor = p; + if (exponent_digits >= 20 || Long.compareUnsigned(abs_exponent, Long.MAX_VALUE) > 0) { + exponent = negative_exponent ? Long.MIN_VALUE : Long.MAX_VALUE; + } else { + exponent = negative_exponent ? -abs_exponent : abs_exponent; + } + } if (integer) { - if (intDigits < MAX_FAST_INTEGER_SIZE) { + if (mantissaDigits < MAX_FAST_INTEGER_SIZE) { return context.runtime.newFixnum(negative ? -mantissa : mantissa); } - return ConvertBytes.byteListToInum(context.runtime, absSubSequence(numberStart, p), 10, true); - } - return config.decimalFactory.apply(context, absSubSequence(numberStart, p)); - } - - private int scanDigits(int p) { - long acc = 0; - while (p + 8 <= end) { - long next8 = chunks.getLong(p); - // Branchless all-eight-are-digits test (simdjson / 0x80.pl): - // each nibble pair resolves to 0x3 iff the byte is '0'..'9'. - long match = (next8 & 0xF0F0F0F0F0F0F0F0L) - | (((next8 + 0x0606060606060606L) & 0xF0F0F0F0F0F0F0F0L) >>> 4); - if (match == 0x3333333333333333L) { - acc = acc * 100000000L + decode8digits(next8); - p += 8; - continue; - } - int consecutive = Long.numberOfTrailingZeros(match ^ 0x3333333333333333L) >>> 3; - if (consecutive >= 4) { - acc = acc * 10000L + decode4digits((int) next8); - p += 4; - consecutive -= 4; - } - while (consecutive > 0) { - acc = acc * 10 + (data[p] - '0'); - p++; - consecutive--; - } - digitsValue = acc; - return p; - } - while (p < end && data[p] >= '0' && data[p] <= '9') { - acc = acc * 10 + (data[p] - '0'); - p++; + return ConvertBytes.byteListToInum(context.runtime, absSubSequence(numberStart, cursor), 10, true); } - digitsValue = acc; - return p; - } - // Decode eight packed little-endian ASCII digits to their integer value. - // From https://lemire.me/blog/2022/01/21/swar-explained-parsing-eight-digits/ - private static long decode8digits(long val) { - final long mask = 0x000000FF000000FFL; - final long mul1 = 0x000F424000000064L; // 100 + (1000000 << 32) - final long mul2 = 0x0000271000000001L; // 1 + (10000 << 32) - val -= 0x3030303030303030L; - val = (val * 10) + (val >>> 8); - val = (((val & mask) * mul1) + (((val >>> 16) & mask) * mul2)) >>> 32; - return val; + // Adjust exponent based on decimal point position + if (decimal_point_pos >= 0) exponent -= (mantissaDigits - decimal_point_pos); + + return decodeFloat(context, mantissa, mantissaDigits, exponent, negative, numberStart); } - // Decode four packed little-endian ASCII digits to their integer value. - private static long decode4digits(int val) { - val -= 0x30303030; - val = (val * 10) + (val >>> 8); - return ((val & 0xFF) * 100) + ((val >>> 16) & 0xFF); + IRubyObject decodeFloat(ThreadContext context, long mantissa, int mantissa_digits, long exponent, boolean negative, int start) { + if (config.decimalClass != null) return config.decimalFactory.apply(context, absSubSequence(start, cursor)); + if (exponent > Integer.MAX_VALUE) return getConstant(negative ? CONST_MINUS_INFINITY : CONST_INFINITY); + if (exponent < Integer.MIN_VALUE) return RubyFloat.newFloat(context.runtime, negative ? -0.0 : 0.0); + + // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308) + if (mantissa_digits > 17 || mantissa_digits + exponent < -307) { + return RubyFloat.newFloat(context.runtime, Double.parseDouble(new String(data, start, cursor - start))); + } + + return RubyFloat.newFloat(context.runtime, ryuS2dFromParts(mantissa, mantissa_digits, (int) exponent, negative)); } private IRubyObject parseString(boolean isName) { @@ -830,6 +808,10 @@ private static int compareKey(byte[] buf, int off, int len, IRubyObject entry) { private byte peek() { return cursor < end ? data[cursor] : 0; } + private byte advance() { + cursor++; + return peek(); + } private boolean matchKeyword(String keyword) { int len = keyword.length(); diff --git a/java/src/json/ext/Ryu.java b/java/src/json/ext/Ryu.java new file mode 100644 index 000000000..570cd24c4 --- /dev/null +++ b/java/src/json/ext/Ryu.java @@ -0,0 +1,770 @@ +// Copyright 2018 Ulf Adams +// +// The contents of this file may be used under the terms of the Apache License, +// Version 2.0. +// +// Alternatively, the contents of this file may be used under the terms of +// the Boost Software License, Version 1.0. +// +// Unless required by applicable law or agreed to in writing, this software +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. +// +// --- +// +// Apache License +// Version 2.0, January 2004 +// http://www.apache.org/licenses/ +// +// TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +// +// 1. Definitions. +// +// "License" shall mean the terms and conditions for use, reproduction, +// and distribution as defined by Sections 1 through 9 of this document. +// +// "Licensor" shall mean the copyright owner or entity authorized by +// the copyright owner that is granting the License. +// +// "Legal Entity" shall mean the union of the acting entity and all +// other entities that control, are controlled by, or are under common +// control with that entity. For the purposes of this definition, +// "control" means (i) the power, direct or indirect, to cause the +// direction or management of such entity, whether by contract or +// otherwise, or (ii) ownership of fifty percent (50%) or more of the +// outstanding shares, or (iii) beneficial ownership of such entity. +// +// "You" (or "Your") shall mean an individual or Legal Entity +// exercising permissions granted by this License. +// +// "Source" form shall mean the preferred form for making modifications, +// including but not limited to software source code, documentation +// source, and configuration files. +// +// "Object" form shall mean any form resulting from mechanical +// transformation or translation of a Source form, including but +// not limited to compiled object code, generated documentation, +// and conversions to other media types. +// +// "Work" shall mean the work of authorship, whether in Source or +// Object form, made available under the License, as indicated by a +// copyright notice that is included in or attached to the work +// (an example is provided in the Appendix below). +// +// "Derivative Works" shall mean any work, whether in Source or Object +// form, that is based on (or derived from) the Work and for which the +// editorial revisions, annotations, elaborations, or other modifications +// represent, as a whole, an original work of authorship. For the purposes +// of this License, Derivative Works shall not include works that remain +// separable from, or merely link (or bind by name) to the interfaces of, +// the Work and Derivative Works thereof. +// +// "Contribution" shall mean any work of authorship, including +// the original version of the Work and any modifications or additions +// to that Work or Derivative Works thereof, that is intentionally +// submitted to Licensor for inclusion in the Work by the copyright owner +// or by an individual or Legal Entity authorized to submit on behalf of +// the copyright owner. For the purposes of this definition, "submitted" +// means any form of electronic, verbal, or written communication sent +// to the Licensor or its representatives, including but not limited to +// communication on electronic mailing lists, source code control systems, +// and issue tracking systems that are managed by, or on behalf of, the +// Licensor for the purpose of discussing and improving the Work, but +// excluding communication that is conspicuously marked or otherwise +// designated in writing by the copyright owner as "Not a Contribution." +// +// "Contributor" shall mean Licensor and any individual or Legal Entity +// on behalf of whom a Contribution has been received by Licensor and +// subsequently incorporated within the Work. +// +// 2. Grant of Copyright License. Subject to the terms and conditions of +// this License, each Contributor hereby grants to You a perpetual, +// worldwide, non-exclusive, no-charge, royalty-free, irrevocable +// copyright license to reproduce, prepare Derivative Works of, +// publicly display, publicly perform, sublicense, and distribute the +// Work and such Derivative Works in Source or Object form. +// +// 3. Grant of Patent License. Subject to the terms and conditions of +// this License, each Contributor hereby grants to You a perpetual, +// worldwide, non-exclusive, no-charge, royalty-free, irrevocable +// (except as stated in this section) patent license to make, have made, +// use, offer to sell, sell, import, and otherwise transfer the Work, +// where such license applies only to those patent claims licensable +// by such Contributor that are necessarily infringed by their +// Contribution(s) alone or by combination of their Contribution(s) +// with the Work to which such Contribution(s) was submitted. If You +// institute patent litigation against any entity (including a +// cross-claim or counterclaim in a lawsuit) alleging that the Work +// or a Contribution incorporated within the Work constitutes direct +// or contributory patent infringement, then any patent licenses +// granted to You under this License for that Work shall terminate +// as of the date such litigation is filed. +// +// 4. Redistribution. You may reproduce and distribute copies of the +// Work or Derivative Works thereof in any medium, with or without +// modifications, and in Source or Object form, provided that You +// meet the following conditions: +// +// (a) You must give any other recipients of the Work or +// Derivative Works a copy of this License; and +// +// (b) You must cause any modified files to carry prominent notices +// stating that You changed the files; and +// +// (c) You must retain, in the Source form of any Derivative Works +// that You distribute, all copyright, patent, trademark, and +// attribution notices from the Source form of the Work, +// excluding those notices that do not pertain to any part of +// the Derivative Works; and +// +// (d) If the Work includes a "NOTICE" text file as part of its +// distribution, then any Derivative Works that You distribute must +// include a readable copy of the attribution notices contained +// within such NOTICE file, excluding those notices that do not +// pertain to any part of the Derivative Works, in at least one +// of the following places: within a NOTICE text file distributed +// as part of the Derivative Works; within the Source form or +// documentation, if provided along with the Derivative Works; or, +// within a display generated by the Derivative Works, if and +// wherever such third-party notices normally appear. The contents +// of the NOTICE file are for informational purposes only and +// do not modify the License. You may add Your own attribution +// notices within Derivative Works that You distribute, alongside +// or as an addendum to the NOTICE text from the Work, provided +// that such additional attribution notices cannot be construed +// as modifying the License. +// +// You may add Your own copyright statement to Your modifications and +// may provide additional or different license terms and conditions +// for use, reproduction, or distribution of Your modifications, or +// for any such Derivative Works as a whole, provided Your use, +// reproduction, and distribution of the Work otherwise complies with +// the conditions stated in this License. +// +// 5. Submission of Contributions. Unless You explicitly state otherwise, +// any Contribution intentionally submitted for inclusion in the Work +// by You to the Licensor shall be under the terms and conditions of +// this License, without any additional terms or conditions. +// Notwithstanding the above, nothing herein shall supersede or modify +// the terms of any separate license agreement you may have executed +// with Licensor regarding such Contributions. +// +// 6. Trademarks. This License does not grant permission to use the trade +// names, trademarks, service marks, or product names of the Licensor, +// except as required for reasonable and customary use in describing the +// origin of the Work and reproducing the content of the NOTICE file. +// +// 7. Disclaimer of Warranty. Unless required by applicable law or +// agreed to in writing, Licensor provides the Work (and each +// Contributor provides its Contributions) on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied, including, without limitation, any warranties or conditions +// of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +// PARTICULAR PURPOSE. You are solely responsible for determining the +// appropriateness of using or redistributing the Work and assume any +// risks associated with Your exercise of permissions under this License. +// +// 8. Limitation of Liability. In no event and under no legal theory, +// whether in tort (including negligence), contract, or otherwise, +// unless required by applicable law (such as deliberate and grossly +// negligent acts) or agreed to in writing, shall any Contributor be +// liable to You for damages, including any direct, indirect, special, +// incidental, or consequential damages of any character arising as a +// result of this License or out of the use or inability to use the +// Work (including but not limited to damages for loss of goodwill, +// work stoppage, computer failure or malfunction, or any and all +// other commercial damages or losses), even if such Contributor +// has been advised of the possibility of such damages. +// +// 9. Accepting Warranty or Additional Liability. While redistributing +// the Work or Derivative Works thereof, You may choose to offer, +// and charge a fee for, acceptance of support, warranty, indemnity, +// or other liability obligations and/or rights consistent with this +// License. However, in accepting such obligations, You may act only +// on Your own behalf and on Your sole responsibility, not on behalf +// of any other Contributor, and only if You agree to indemnify, +// defend, and hold each Contributor harmless for any liability +// incurred by, or claims asserted against, such Contributor by reason +// of your accepting any such warranty or additional liability. +// +// END OF TERMS AND CONDITIONS +// +// APPENDIX: How to apply the Apache License to your work. +// +// To apply the Apache License to your work, attach the following +// boilerplate notice, with the fields enclosed by brackets "[]" +// replaced with your own identifying information. (Don't include +// the brackets!) The text should be enclosed in the appropriate +// comment syntax for the file format. We also recommend that a +// file or class name and description of purpose be included on the +// same "printed page" as the copyright notice for easier +// identification within third-party archives. +// +// Copyright [yyyy] [name of copyright owner] +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// --- +// +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// --- +// Minimal Ryu implementation adapted for Ruby JSON gem by Josef Šimánek +// Optimized for pre-extracted mantissa/exponent from JSON parsing +// This is a stripped-down version containing only what's needed for +// converting decimal mantissa+exponent to IEEE 754 double precision. +package json.ext; + +// Note: This is manually translated from ext/json/ext/vendor/ryu.h +public class Ryu { + static final int DOUBLE_POW5_INV_BITCOUNT = 125; + static final int DOUBLE_POW5_BITCOUNT = 125; + static long[][] DOUBLE_POW5_INV_SPLIT = { + { 1L, 2305843009213693952L}, { -7378697629483820646L, 1844674407370955161L}, + { 5165088340638674453L, 1475739525896764129L}, { 7821419487252849886L, 1180591620717411303L}, + { 8824922364862649494L, 1888946593147858085L}, { 7059937891890119595L, 1511157274518286468L}, + { -5420096130713635294L, 1208925819614629174L}, { -8672153809141816470L, 1934281311383406679L}, + { -6937723047313453176L, 1547425049106725343L}, { -1860829623108852217L, 1237940039285380274L}, + { -2977327396974163548L, 1980704062856608439L}, { -2381861917579330838L, 1584563250285286751L}, + { 9162556910162266299L, 1267650600228229401L}, { 7281393426775805432L, 2028240960365167042L}, + { -1553582888063176301L, 1622592768292133633L}, { 2446482504291369283L, 1298074214633706907L}, + { 7603720821608101175L, 2076918743413931051L}, { 2393627842544570617L, 1661534994731144841L}, + { -1774446540706253830L, 1329227995784915872L}, { -6528463279871916451L, 2126764793255865396L}, + { 5845275820328197809L, 1701411834604692317L}, { -2702476973221262399L, 1361129467683753853L}, + { 3054734472329800808L, 2177807148294006166L}, { -1245561236878069677L, 1742245718635204932L}, + { 6382248639981364905L, 1393796574908163946L}, { 2832900194486363201L, 2230074519853062314L}, + { 5955668970331000884L, 1784059615882449851L}, { 1075186361522890384L, 1427247692705959881L}, + { -5658399451047196032L, 2283596308329535809L}, { -4526719560837756825L, 1826877046663628647L}, + { 3757321980813615186L, 1461501637330902918L}, { -8062188859574838821L, 1169201309864722334L}, + { 5547241898389809503L, 1870722095783555735L}, { 4437793518711847602L, 1496577676626844588L}, + { -7517811629256252888L, 1197262141301475670L}, { -960452162584273651L, 1915619426082361072L}, + { 6610335899416401726L, 1532495540865888858L}, { -5779777724692609589L, 1225996432692711086L}, + { -5558295544766265019L, 1961594292308337738L}, { -757287621071101692L, 1569275433846670190L}, + { -4295178911598791677L, 1255420347077336152L}, { 7885109000409574610L, 2008672555323737844L}, + { -8449308058639981605L, 1606938044258990275L}, { 7997948812055656009L, 1285550435407192220L}, + { -5650025974420502002L, 2056880696651507552L}, { 2858676849947419045L, 1645504557321206042L}, + { -5091756149525885410L, 1316403645856964833L}, { -768112209757596011L, 2106245833371143733L}, + { 3074859046935833515L, 1684996666696914987L}, { -4918810391935153834L, 1347997333357531989L}, + { -7870096627096246135L, 2156795733372051183L}, { -2606728486935086585L, 1725436586697640946L}, + { 8982663654677661702L, 1380349269358112757L}, { -385133411483382570L, 2208558830972980411L}, + { -7686804358670526703L, 1766847064778384329L}, { -6149443486936421362L, 1413477651822707463L}, + { -2460411949614453533L, 2261564242916331941L}, { 9099716884534168143L, 1809251394333065553L}, + { -3788272936598396455L, 1447401115466452442L}, { 4348079280205103483L, 1157920892373161954L}, + { -4111119595897565398L, 1852673427797059126L}, { 7779150767507678651L, 1482138742237647301L}, + { 2533971799264232598L, 1185710993790117841L}, { -3324342750661048490L, 1897137590064188545L}, + { -6348823015270749115L, 1517710072051350836L}, { 5988988032009131678L, 1214168057641080669L}, + { -1485665593011120286L, 1942668892225729070L}, { -4877881289150806552L, 1554135113780583256L}, + { 7165741412905085728L, 1243308091024466605L}, { -6981557813061414451L, 1989292945639146568L}, + { -1895897435707221237L, 1591434356511317254L}, { -1516717948565776990L, 1273147485209053803L}, + { 4951948911778577463L, 2037035976334486086L}, { 272210314680951647L, 1629628781067588869L}, + { 3907117066486671641L, 1303703024854071095L}, { 6251387306378674625L, 2085924839766513752L}, + { -2377587784380880946L, 1668739871813211001L}, { 9165976216721026213L, 1334991897450568801L}, + { 7286864317269821294L, 2135987035920910082L}, { -1549206175667963611L, 1708789628736728065L}, + { -4928713755276281212L, 1367031702989382452L}, { 6871453250525591353L, 2187250724783011924L}, + { 9186511415162383406L, 1749800579826409539L}, { -7408186126837734568L, 1399840463861127631L}, + { -8163748988198464986L, 2239744742177804210L}, { 8226396068408869304L, 1791795793742243368L}, + { -4486929589498635526L, 1433436634993794694L}, { -7179087343197816842L, 2293498615990071511L}, + { 5324776569667477496L, 1834798892792057209L}, { 7949170070475892320L, 1467839114233645767L}, + { -1019361573103106790L, 1174271291386916613L}, { 5747719112518849781L, 1878834066219066582L}, + { -2780522339468740821L, 1503067252975253265L}, { -5913766686316902980L, 1202453802380202612L}, + { 5295368560860596524L, 1923926083808324180L}, { 4236294848688477220L, 1539140867046659344L}, + { 7078384693692692099L, 1231312693637327475L}, { -7121328563801244258L, 1970100309819723960L}, + { 9060332407926645887L, 1576080247855779168L}, { -3819780517884414260L, 1260864198284623334L}, + { -6111648828615062817L, 2017382717255397335L}, { -8578667877633960576L, 1613906173804317868L}, + { -3173585487365258138L, 1291124939043454294L}, { -5077736779784413021L, 2065799902469526871L}, + { 7005857020398200553L, 1652639921975621497L}, { -1774012013165260204L, 1322111937580497197L}, + { -6527768035806326650L, 2115379100128795516L}, { 5845832015580669650L, 1692303280103036413L}, + { -6391380831761195250L, 1353842624082429130L}, { 841837113407818570L, 2166148198531886609L}, + { 4362818505468165179L, 1732918558825509287L}, { -3888442825109288503L, 1386334847060407429L}, + { -6221508520174861605L, 2218135755296651887L}, { 2401490813343931363L, 1774508604237321510L}, + { 1921192650675145090L, 1419606883389857208L}, { -615440573661678179L, 2271371013423771532L}, + { 6886345170554478103L, 1817096810739017226L}, { 1819727321701672159L, 1453677448591213781L}, + { -2233566957380572596L, 1162941958872971024L}, { -3573707131808916153L, 1860707134196753639L}, + { -2858965705447132922L, 1488565707357402911L}, { 8780873879868024632L, 1190852565885922329L}, + { 2981351763563108441L, 1905364105417475727L}, { -4993616218633333894L, 1524291284333980581L}, + { 7073153469319063855L, 1219433027467184465L}, { -7129698522799049449L, 1951092843947495144L}, + { -5703758818239239559L, 1560874275157996115L}, { -8252355869333301970L, 1248699420126396892L}, + { 1553625868034358140L, 1997919072202235028L}, { 8621598323911307159L, 1598335257761788022L}, + { -481418970354774919L, 1278668206209430417L}, { -4459619167309550194L, 2045869129935088668L}, + { 121653480894270168L, 1636695303948070935L}, { 97322784715416134L, 1309356243158456748L}, + { -3533632359197244509L, 2094969989053530796L}, { 8241140556867935363L, 1675975991242824637L}, + { -785785183989472356L, 1340780792994259709L}, { -1257256294383155770L, 2145249268790815535L}, + { -4695153850248434939L, 1716199415032652428L}, { -66774265456837628L, 1372959532026121942L}, + { -3796187639472850528L, 2196735251241795108L}, { 652398703163629901L, 1757388200993436087L}, + { -6856778666952916726L, 1405910560794748869L}, { 7475898206584884855L, 2249456897271598191L}, + { 2291369750525997561L, 1799565517817278553L}, { 9211793429904618695L, 1439652414253822842L}, + { -18525771120251381L, 2303443862806116547L}, { 7363877012587619542L, 1842755090244893238L}, + { -5176944834155635336L, 1474204072195914590L}, { -7830904682066418592L, 1179363257756731672L}, + { 2227947767661371545L, 1886981212410770676L}, { -1906990600612813087L, 1509584969928616540L}, + { -5214941295232160793L, 1207667975942893232L}, { 6413489186596184024L, 1932268761508629172L}, + { -2247906280206873427L, 1545815009206903337L}, { 5580372605318321905L, 1236652007365522670L}, + { 8928596168509315048L, 1978643211784836272L}, { -235820694676368608L, 1582914569427869017L}, + { 7190041073742725760L, 1266331655542295214L}, { 436019273762630246L, 2026130648867672343L}, + { 7727513048493924843L, 1620904519094137874L}, { -8575384820172501418L, 1296723615275310299L}, + { 4726128361433549347L, 2074757784440496479L}, { 7470251503888749801L, 1659806227552397183L}, + { -5091845241114731129L, 1327844982041917746L}, { -4457603571041659483L, 2124551971267068394L}, + { -3566082856833327587L, 1699641577013654715L}, { -6542215100208572392L, 1359713261610923772L}, + { 4289851098633925465L, 2175541218577478036L}, { -257467935834769951L, 1740432974861982428L}, + { 3483374466074094362L, 1392346379889585943L}, { 1884050330976640656L, 2227754207823337509L}, + { 5196589079523222848L, 1782203366258670007L}, { -3221426365865242368L, 1425762693006936005L}, + { 5913764258841343181L, 2281220308811097609L}, { 8420360221814984868L, 1824976247048878087L}, + { -642409452031832752L, 1459980997639102469L}, { -513927561625466201L, 1167984798111281975L}, + { -8200981728084566569L, 1868775676978051161L}, { 4507261061758077715L, 1495020541582440929L}, + { 7295157664148372495L, 1196016433265952743L}, { 7982903447895485668L, 1913626293225524389L}, + { -8371072500651252758L, 1530901034580419511L}, { 4371188443704728763L, 1224720827664335609L}, + { -4074144934298164949L, 1959553324262936974L}, { -3259315947438531959L, 1567642659410349579L}, + { -2607452757950825567L, 1254114127528279663L}, { 3206773216762499739L, 2006582604045247462L}, + { -4813279056073820855L, 1605266083236197969L}, { -3850623244859056684L, 1284212866588958375L}, + { 4907049252451240275L, 2054740586542333401L}, { 236290587219081897L, 1643792469233866721L}, + { -3500316344966644806L, 1315033975387093376L}, { -1911157337204721366L, 2104054360619349402L}, + { 5849771759720043554L, 1683243488495479522L}, { -2698880221707785803L, 1346594790796383617L}, + { -8007557169474367609L, 2154551665274213788L}, { -2716696920837583764L, 1723641332219371030L}, + { -5862706351411977334L, 1378913065775496824L}, { 9066413911450387881L, 2206260905240794919L}, + { -7504264129807330988L, 1765008724192635935L}, { 8753983955121776503L, 1412006979354108748L}, + { -8129718560256619535L, 2259211166966573997L}, { 874922781278525018L, 1807368933573259198L}, + { 8078635854506640661L, 1445895146858607358L}, { -4605137760620418441L, 1156716117486885886L}, + { -3678871602250759182L, 1850745787979017418L}, { 746251532941302978L, 1480596630383213935L}, + { 597001226353042382L, 1184477304306571148L}, { -2734146852577042512L, 1895163686890513836L}, + { 8880728962164096960L, 1516130949512411069L}, { -7652812089236363725L, 1212904759609928855L}, + { -1176452898552450990L, 1940647615375886168L}, { 2748186495899949531L, 1552518092300708935L}, + { 2198549196719959625L, 1242014473840567148L}, { -171670099989974923L, 1987223158144907436L}, + { -7516033709475800585L, 1589778526515925949L}, { -6012826967580640468L, 1271822821212740759L}, + { 8826220925580526867L, 2034916513940385215L}, { 7060976740464421494L, 1627933211152308172L}, + { -1729916237112283451L, 1302346568921846537L}, { -6457214794121563846L, 2083754510274954460L}, + { -8855120650039161400L, 1667003608219963568L}, { -3394747705289418796L, 1333602886575970854L}, + { -5431596328463070074L, 2133764618521553367L}, { 3033420566713364587L, 1707011694817242694L}, + { 6116085268112601993L, 1365609355853794155L}, { -8661007644729388428L, 2184974969366070648L}, + { -3239457301041600419L, 1747979975492856518L}, { 1097782973908629988L, 1398383980394285215L}, + { 1756452758253807981L, 2237414368630856344L}, { 5094511021344956708L, 1789931494904685075L}, + { 4075608817075965366L, 1431945195923748060L}, { 6520974107321544586L, 2291112313477996896L}, + { 1527430471115325346L, 1832889850782397517L}, { -6156753252591560370L, 1466311880625918013L}, + { -1236053787331337972L, 1173049504500734410L}, { 9090360384495590213L, 1876879207201175057L}, + { -106409321887348476L, 1501503365760940045L}, { -3774476272251789104L, 1201202692608752036L}, + { -2349813220860952243L, 1921924308174003258L}, { 1809498238053148529L, 1537539446539202607L}, + { -5931099039041301823L, 1230031557231362085L}, { 1578287981759648052L, 1968050491570179337L}, + { -6116067244076102204L, 1574440393256143469L}, { -4892853795260881763L, 1259552314604914775L}, + { 3239480371808320148L, 2015283703367863641L}, { -1097764517295254205L, 1612226962694290912L}, + { 6500486015647617283L, 1289781570155432730L}, { -8045966448673363964L, 2063650512248692368L}, + { -2747424344196780848L, 1650920409798953894L}, { -2197939475357424678L, 1320736327839163115L}, + { 7551343283653851484L, 2113178124542660985L}, { 6041074626923081187L, 1690542499634128788L}, + { -6235186742687266020L, 1352433999707303030L}, { 1091747655926105338L, 2163894399531684849L}, + { 4562746939482794594L, 1731115519625347879L}, { 7339546366328145998L, 1384892415700278303L}, + { 8053925371383123274L, 2215827865120445285L}, { 6443140297106498619L, 1772662292096356228L}, + { -5913534206540532074L, 1418129833677084982L}, { 5295740528502789974L, 2269007733883335972L}, + { -3142105206681588667L, 1815206187106668777L}, { 4865013464138549713L, 1452164949685335022L}, + { -3486686858172980876L, 1161731959748268017L}, { 9178696285890871890L, 1858771135597228828L}, + { -3725089415513033457L, 1487016908477783062L}, { 4398626097073393881L, 1189613526782226450L}, + { 7037801755317430209L, 1903381642851562320L}, { 5630241404253944167L, 1522705314281249856L}, + { 814844308661245011L, 1218164251424999885L}, { 1303750893857992017L, 1949062802279999816L}, + { -2646348099655516710L, 1559250241823999852L}, { 5261619149759407279L, 1247400193459199882L}, + { -6338804619352589647L, 1995840309534719811L}, { 5997002748743659252L, 1596672247627775849L}, + { 8486951013736837725L, 1277337798102220679L}, { 2511075177753209390L, 2043740476963553087L}, + { -5369837487281253134L, 1634992381570842469L}, { -4295869989825002507L, 1307993905256673975L}, + { 4194654460505726958L, 2092790248410678361L}, { -333625246337328757L, 1674232198728542688L}, + { 3422448617672047318L, 1339385758982834151L}, { -1902779841208544938L, 2143017214372534641L}, + { -8900921502450656597L, 1714413771498027713L}, { -3431388387218614954L, 1371531017198422170L}, + { 5577825024675947042L, 2194449627517475473L}, { -6605786424484973336L, 1755559702013980378L}, + { -1595280324846068345L, 1404447761611184302L}, { -6241797334495619676L, 2247116418577894884L}, + { -4993437867596495741L, 1797693134862315907L}, { 3383947335406624054L, 1438154507889852726L}, + { -1964381892833222160L, 2301047212623764361L}, { -8950203143750398374L, 1840837770099011489L}, + { -7160162515000318699L, 1472670216079209191L}, { 5339916432225476010L, 1178136172863367353L}, + { 4854517476818851293L, 1885017876581387765L}, { 3883613981455081034L, 1508014301265110212L}, + { -4271806444319755819L, 1206411441012088169L}, { -6834890310911609310L, 1930258305619341071L}, + { 5600134195496443521L, 1544206644495472857L}, { -2898590273086665829L, 1235365315596378285L}, + { 6430302007287065643L, 1976584504954205257L}, { -2234456023654168132L, 1581267603963364205L}, + { -5476913633665244829L, 1265014083170691364L}, { -8763061813864391727L, 2024022533073106183L}, + { -3321100636349603058L, 1619218026458484946L}, { 8411165935146048523L, 1295374421166787957L}, + { -1299529762733963656L, 2072599073866860731L}, { -8418321439670991571L, 1658079259093488585L}, + { 8022738107230848036L, 1326463407274790868L}, { 9147032156827446534L, 2122341451639665389L}, + { -7439769533505684065L, 1697873161311732311L}, { 5116230817421183718L, 1358298529049385849L}, + { -2882077136351837022L, 2173277646479017358L}, { 1383687105660440706L, 1738622117183213887L}, + { -6271747944955468082L, 1390897693746571109L}, { 8411947361780802685L, 2225436309994513775L}, + { 6729557889424642148L, 1780349047995611020L}, { 5383646311539713719L, 1424279238396488816L}, + { 1235136468979721303L, 2278846781434382106L}, { -2701239639558133281L, 1823077425147505684L}, + { -2160991711646506624L, 1458461940118004547L}, { 5649904260166615347L, 1166769552094403638L}, + { 5350498001524674232L, 1866831283351045821L}, { 591049586477829062L, 1493465026680836657L}, + { -6905857960301557397L, 1194772021344669325L}, { 18673707743239135L, 1911635234151470921L}, + { -3674409848547319015L, 1529308187321176736L}, { 8128518565387875758L, 1223446549856941389L}, + { 1937583260394870242L, 1957514479771106223L}, { 8928764237799716840L, 1566011583816884978L}, + { -3925035053985957497L, 1252809267053507982L}, { 8477339172590109297L, 2004494827285612772L}, + { -596826291411733209L, 1603595861828490217L}, { 6901236596354434079L, 1282876689462792174L}, + { -26067890058636443L, 2052602703140467478L}, { 3668494502695001169L, 1642082162512373983L}, + { -8133250842069730034L, 1313665730009899186L}, { 9122891541139893884L, 2101865168015838698L}, + { -3769733211313815862L, 1681492134412670958L}, { 673562245690857633L, 1345193707530136767L}, + }; + static long[][] DOUBLE_POW5_SPLIT = { + { 0L, 1152921504606846976L}, { 0L, 1441151880758558720L}, + { 0L, 1801439850948198400L}, { 0L, 2251799813685248000L}, + { 0L, 1407374883553280000L}, { 0L, 1759218604441600000L}, + { 0L, 2199023255552000000L}, { 0L, 1374389534720000000L}, + { 0L, 1717986918400000000L}, { 0L, 2147483648000000000L}, + { 0L, 1342177280000000000L}, { 0L, 1677721600000000000L}, + { 0L, 2097152000000000000L}, { 0L, 1310720000000000000L}, + { 0L, 1638400000000000000L}, { 0L, 2048000000000000000L}, + { 0L, 1280000000000000000L}, { 0L, 1600000000000000000L}, + { 0L, 2000000000000000000L}, { 0L, 1250000000000000000L}, + { 0L, 1562500000000000000L}, { 0L, 1953125000000000000L}, + { 0L, 1220703125000000000L}, { 0L, 1525878906250000000L}, + { 0L, 1907348632812500000L}, { 0L, 1192092895507812500L}, + { 0L, 1490116119384765625L}, { 4611686018427387904L, 1862645149230957031L}, + { -8646911284551352320L, 1164153218269348144L}, { -6196953087261802496L, 1455191522836685180L}, + { -3134505340649865216L, 1818989403545856475L}, { -3918131675812331520L, 2273736754432320594L}, + { -4754675306596401152L, 1421085471520200371L}, { -5943344133245501440L, 1776356839400250464L}, + { -2817494148129488896L, 2220446049250313080L}, { -8678462870222012416L, 1387778780781445675L}, + { 7598665485932036096L, 1734723475976807094L}, { 274959820560269312L, 2168404344971008868L}, + { -9051522149004607488L, 1355252715606880542L}, { 2520655369026404352L, 1694065894508600678L}, + { -6072552825571770368L, 2117582368135750847L}, { -3795345515982356480L, 1323488980084844279L}, + { -4744181894977945600L, 1654361225106055349L}, { 3293144668132343808L, 2067951531382569187L}, + { -247627591630979072L, 1292469707114105741L}, { 8913837547316051968L, 1615587133892632177L}, + { -2692761121137098752L, 2019483917365790221L}, { -6294661719138074624L, 1262177448353618888L}, + { -3256641130495205376L, 1577721810442023610L}, { -8682487431546394624L, 1972152263052529513L}, + { -814868626289108736L, 1232595164407830945L}, { 8204786253993389888L, 1540743955509788682L}, + { 1032610780636961552L, 1925929944387235853L}, { 2951224747111794922L, 1203706215242022408L}, + { 3689030933889743652L, 1504632769052528010L}, { -4612083369492596243L, 1880790961315660012L}, + { -576709096719178700L, 1175494350822287507L}, { -720886370898973375L, 1469367938527859384L}, + { 3710578054803671186L, 1836709923159824231L}, { 26536550077201078L, 2295887403949780289L}, + { -6900943683842831182L, 1434929627468612680L}, { -4014493586376151074L, 1793662034335765850L}, + { 8816941072311974870L, 2242077542919707313L}, { -1406940857446097563L, 1401298464324817070L}, + { -6370362090235009857L, 1751623080406021338L}, { 5872105442488401391L, 2189528850507526673L}, + { -3247463126085830987L, 1368455531567204170L}, { -8671014926034676638L, 1710569414459005213L}, + { -1615396620688569989L, 2138211768073756516L}, { 1296220121283337709L, 1336382355046097823L}, + { -2991410866823215768L, 1670477943807622278L}, { -8350949601956407614L, 2088097429759527848L}, + { 6309871544845715001L, 1305060893599704905L}, { -5947718624225019960L, 1631326116999631131L}, + { -7434648280281274950L, 2039157646249538914L}, { -6952498184389490796L, 1274473528905961821L}, + { 532749306367912313L, 1593091911132452277L}, { 5277622651387278295L, 1991364888915565346L}, + { 7910200175544436838L, 1244603055572228341L}, { -3947307835851617664L, 1555753819465285426L}, + { 8900923260467641632L, 1944692274331606783L}, { -5966138008276193740L, 1215432671457254239L}, + { -7457672510345242175L, 1519290839321567799L}, { 9124653435777998898L, 1899113549151959749L}, + { 8008751406574943263L, 1186945968219974843L}, { 5399253239791291175L, 1483682460274968554L}, + { -2474305487115661840L, 1854603075343710692L}, { 759402079766405302L, 1159126922089819183L}, + { -3662433418719381276L, 1448908652612273978L}, { -9189727791826614499L, 1811135815765342473L}, + { -2263787702928492316L, 2263919769706678091L}, { 7808504722524468110L, 1414949856066673807L}, + { 5148944884728197234L, 1768687320083342259L}, { 1824495087482858639L, 2210859150104177824L}, + { 1140309429676786649L, 1381786968815111140L}, { 1425386787095983311L, 1727233711018888925L}, + { 6393419502297367043L, 2159042138773611156L}, { -5227484847918921406L, 1349401336733506972L}, + { -1922670041471263854L, 1686751670916883715L}, { -2403337551839079817L, 2108439588646104644L}, + { 803757039314269066L, 1317774742903815403L}, { -3606989719284551571L, 1647218428629769253L}, + { 4714634887749086344L, 2059023035787211567L}, { -8582568241225290795L, 1286889397367007229L}, + { -1504838264676837686L, 1608611746708759036L}, { 2730638187581340797L, 2010764683385948796L}, + { -7516723169616437810L, 1256727927116217997L}, { -172531925165771454L, 1570909908895272496L}, + { 4396021111970173586L, 1963637386119090621L}, { 5053356204195052443L, 1227273366324431638L}, + { -2906676781610960254L, 1534091707905539547L}, { -3633345977013700317L, 1917614634881924434L}, + { -4576684244847256650L, 1198509146801202771L}, { -5720855306059070813L, 1498136433501503464L}, + { -2539383114146450612L, 1872670541876879330L}, { -3892957455555225585L, 1170419088673049581L}, + { 4357175217410743827L, 1463023860841311977L}, { -8388589033518733928L, 1828779826051639971L}, + { 7961007781811134206L, 2285974782564549964L}, { -4247742173222816929L, 1428734239102843727L}, + { -5309677716528521161L, 1785917798878554659L}, { -6637097145660651452L, 2232397248598193324L}, + { -1842342706824213205L, 1395248280373870827L}, { -2302928383530266507L, 1744060350467338534L}, + { -7490346497840221037L, 2180075438084173168L}, { 6847748484918331612L, 1362547148802608230L}, + { -663686430706861293L, 1703183936003260287L}, { -829608038383576617L, 2128979920004075359L}, + { -518505023989735386L, 1330612450002547099L}, { -648131279987169232L, 1663265562503183874L}, + { -5421850118411349444L, 2079081953128979843L}, { 5834715712847682405L, 1299426220705612402L}, + { -1929977395795172801L, 1624282775882015502L}, { -7024157763171353905L, 2030353469852519378L}, + { -6695941611195790143L, 1268970918657824611L}, { -8369927013994737679L, 1586213648322280764L}, + { -5850722749066034194L, 1982767060402850955L}, { 5566670318688504437L, 1239229412751781847L}, + { 2346651879933242642L, 1549036765939727309L}, { 7545000868343941206L, 1936295957424659136L}, + { 4715625542714963254L, 1210184973390411960L}, { 5894531928393704067L, 1512731216738014950L}, + { -1855207126362645724L, 1890914020922518687L}, { -1159504453976653577L, 1181821263076574179L}, + { -1449380567470816972L, 1477276578845717724L}, { 2799960309088866689L, 1846595723557147156L}, + { -7473396843674234127L, 1154122327223216972L}, { -4730060036165404755L, 1442652909029021215L}, + { -5912575045206755944L, 1803316136286276519L}, { -7390718806508444929L, 2254145170357845649L}, + { -7513235640390177L, 1408840731473653530L}, { -4621077562977875625L, 1761050914342066913L}, + { 3447025083132431277L, 2201313642927583642L}, { 6766076695385157452L, 1375821026829739776L}, + { 8457595869231446815L, 1719776283537174720L}, { -7874749237170243097L, 2149720354421468400L}, + { 6607496772837067824L, 1343575221513417750L}, { -964001070808441028L, 1679469026891772187L}, + { -1205001338510551285L, 2099336283614715234L}, { -3058968845782788505L, 1312085177259197021L}, + { 5399660979626290177L, 1640106471573996277L}, { -7085481830749300991L, 2050133089467495346L}, + { -6734269153432007072L, 1281333180917184591L}, { -8417836441790008839L, 1601666476146480739L}, + { 7924448521472040567L, 2002083095183100924L}, { -4270591710934750454L, 1251301934489438077L}, + { 3885132398186337741L, 1564127418111797597L}, { -8978642557549241536L, 1955159272639746996L}, + { -3305808589254582008L, 1221974545399841872L}, { 479425281859160394L, 1527468181749802341L}, + { 5210967620751338397L, 1909335227187252926L}, { -1354831255457801406L, 1193334516992033078L}, + { -6305225087749639662L, 1491668146240041348L}, { -3269845341259661673L, 1864585182800051685L}, + { -6655339356714676450L, 1165365739250032303L}, { -8319174195893345562L, 1456707174062540379L}, + { 8047776328842869663L, 1820883967578175474L}, { 836348374198811271L, 2276104959472719343L}, + { 7440246761515338900L, 1422565599670449589L}, { -4534749603387990086L, 1778206999588061986L}, + { 8166621051047176104L, 2222758749485077483L}, { 2798295147690791113L, 1389224218428173427L}, + { -1113817083813899013L, 1736530273035216783L}, { -1392271354767373766L, 2170662841294020979L}, + { 8353202440125167204L, 1356664275808763112L}, { -8005241023553092611L, 1695830344760953890L}, + { 3828506775840797949L, 2119787930951192363L}, { 86973725686804766L, 1324867456844495227L}, + { -4502968861318881947L, 1656084321055619033L}, { 3594660960206173375L, 2070105401319523792L}, + { 2246663100128858359L, 1293815875824702370L}, { -6415043161693702859L, 1617269844780877962L}, + { 5816254103165035138L, 2021587305976097453L}, { 5941001823691840913L, 1263492066235060908L}, + { 7426252279614801142L, 1579365082793826135L}, { 4671129331091113523L, 1974206353492282669L}, + { 5225298841145639904L, 1233878970932676668L}, { 6531623551432049880L, 1542348713665845835L}, + { 3552843420862674446L, 1927935892082307294L}, { -2391158880388216375L, 1204959932551442058L}, + { -7600634618912658373L, 1506199915689302573L}, { -277421236786047158L, 1882749894611628216L}, + { -7090917300632361330L, 1176718684132267635L}, { -8863646625790451662L, 1470898355165334544L}, + { -6467872263810676674L, 1838622943956668180L}, { -3473154311335957938L, 2298278679945835225L}, + { 2440964573842414192L, 1436424174966147016L}, { 3051205717303017741L, 1795530218707683770L}, + { -5409364890226003632L, 2244412773384604712L}, { 8148361989677217490L, 1402757983365377945L}, + { -3649605568185641850L, 1753447479206722431L}, { -4562006960232052312L, 2191809349008403039L}, + { -2851254350145032695L, 1369880843130251899L}, { -3564067937681290869L, 1712351053912814874L}, + { -9066770940529001490L, 2140438817391018593L}, { -1055045819403238027L, 1337774260869386620L}, + { 3292878744173340370L, 1672217826086733276L}, { 4116098430216675462L, 2090272282608416595L}, + { 266718509671728212L, 1306420176630260372L}, { 333398137089660265L, 1633025220787825465L}, + { 5028433689789463235L, 2041281525984781831L}, { -8386443989950055238L, 1275800953740488644L}, + { -5871368969010181144L, 1594751192175610805L}, { 1884160825592049379L, 1993438990219513507L}, + { -1128242493218663091L, 1245899368887195941L}, { 7813068920331446945L, 1557374211108994927L}, + { 5154650131986920777L, 1946717763886243659L}, { 915813323278131534L, 1216698602428902287L}, + { -3466919364329723487L, 1520873253036127858L}, { -8945335223839542262L, 1901091566295159823L}, + { -5590834514899713914L, 1188182228934474889L}, { 2234828893230133415L, 1485227786168093612L}, + { 2793536116537666769L, 1856534732710117015L}, { 8663489100477123587L, 1160334207943823134L}, + { 1605989338741628675L, 1450417759929778918L}, { -7215885363427739964L, 1813022199912223647L}, + { -9019856704284674954L, 2266277749890279559L}, { -5637410440177921847L, 1416423593681424724L}, + { -2435077031795014404L, 1770529492101780905L}, { 6179525747111007803L, 2213161865127226132L}, + { -5361168444910395931L, 1383226165704516332L}, { -2089774537710607010L, 1729032707130645415L}, + { -2612218172138258762L, 2161290883913306769L}, { 2979049660840976177L, 1350806802445816731L}, + { -887873942376167682L, 1688508503057270913L}, { 8113529608884566205L, 2110635628821588642L}, + { -8764102049729309834L, 1319147268013492901L}, { -1731755525306861484L, 1648934085016866126L}, + { -6776380425060964759L, 2061167606271082658L}, { -6541080774876796927L, 1288229753919426661L}, + { 1047021068258779650L, 1610287192399283327L}, { -3302909683103913342L, 2012858990499104158L}, + { 4853210475701136017L, 1258036869061940099L}, { 1454827076199032118L, 1572546086327425124L}, + { 1818533845248790147L, 1965682607909281405L}, { 3442426662494187794L, 1228551629943300878L}, + { -4920338708737041066L, 1535689537429126097L}, { 3072948650933474476L, 1919611921786407622L}, + { -2691093111593966357L, 1199757451116504763L}, { -3363866389492457946L, 1499696813895630954L}, + { -8816519005292960336L, 1874621017369538693L}, { 8324733676974063502L, 1171638135855961683L}, + { 5794231077790191473L, 1464547669819952104L}, { 7242788847237739342L, 1830684587274940130L}, + { -169885977807601630L, 2288355734093675162L}, { -2412021745343444971L, 1430222333808546976L}, + { 1596658836748081690L, 1787777917260683721L}, { 6607509564362490017L, 2234722396575854651L}, + { 1823850468512862308L, 1396701497859909157L}, { 6891499104068465790L, 1745876872324886446L}, + { -608998156769193571L, 2182346090406108057L}, { 4231062170446641922L, 1363966306503817536L}, + { 5288827713058302403L, 1704957883129771920L}, { 6611034641322878003L, 2131197353912214900L}, + { -5091475386027977056L, 1331998346195134312L}, { -1752658214107583416L, 1664997932743917890L}, + { -6802508786061867174L, 2081247415929897363L}, { 4971804045566108824L, 1300779634956185852L}, + { 6214755056957636030L, 1625974543695232315L}, { 3156757802769657134L, 2032468179619040394L}, + { 6584659645158423613L, 1270292612261900246L}, { -992547480406746292L, 1587865765327375307L}, + { -1240684350508432865L, 1984832206659219134L}, { 6142101308573311315L, 1240520129162011959L}, + { 3065940617289251240L, 1550650161452514949L}, { 8444111790038951954L, 1938312701815643686L}, + { 665883850346957067L, 1211445438634777304L}, { 832354812933696334L, 1514306798293471630L}, + { -8182928520687655390L, 1892883497866839537L}, { -502644307002396715L, 1183052186166774710L}, + { -5239991402180383798L, 1478815232708468388L}, { -1938303234298091843L, 1848519040885585485L}, + { -5823125539863695306L, 1155324400553490928L}, { -2667220906402231229L, 1444155500691863660L}, + { 1277659885424598868L, 1805194375864829576L}, { 1597074856780748586L, 2256492969831036970L}, + { 5609857803915355770L, 1410308106144398106L}, { -2211049781960581095L, 1762885132680497632L}, + { 1847873790976661535L, 2203606415850622041L}, { -5762607908280668397L, 1377254009906638775L}, + { -7203259885350835496L, 1721567512383298469L}, { 219297180166231438L, 2151959390479123087L}, + { 7054589765244976505L, 1344974619049451929L}, { -5016820848725943081L, 1681218273811814911L}, + { -6271026060907428851L, 2101522842264768639L}, { -3919391288067143032L, 1313451776415480399L}, + { -4899239110083928790L, 1641814720519350499L}, { -6124048887604910988L, 2052268400649188124L}, + { -1521687545539375415L, 1282667750405742577L}, { 7321262604930556539L, 1603334688007178222L}, + { -71793780691580134L, 2004168360008972777L}, { 4566814905495150320L, 1252605225005607986L}, + { -3514853404985837908L, 1565756531257009982L}, { -9005252774659685289L, 1957195664071262478L}, + { 1289246043478778550L, 1223247290044539049L}, { 6223243572775861092L, 1529059112555673811L}, + { 3167368447542438461L, 1911323890694592264L}, { 1979605279714024038L, 1194577431684120165L}, + { 7086192618069917952L, 1493221789605150206L}, { -365631264267378368L, 1866527237006437757L}, + { -4840205558594499384L, 1166579523129023598L}, { 7784801107039039482L, 1458224403911279498L}, + { 507629346944023544L, 1822780504889099373L}, { 5246222702107417334L, 2278475631111374216L}, + { 3278889188817135834L, 1424047269444608885L}, { 8710297504448807696L, 1780059086805761106L}, + }; + + // IEEE 754 double precision constants + static final int DOUBLE_MANTISSA_BITS = 52; + static final int DOUBLE_EXPONENT_BITS = 11; + static final int DOUBLE_EXPONENT_BIAS = 1023; + + // Helper: floor(log2(value)) using ryu_leading_zeros64 + static int floorLog2(long value) { + return 63 - Long.numberOfLeadingZeros(value); + } + + // Helper: log2(5^e) approximation + static int log2pow5(int e) { + return (e * 1217359) >> 19; + } + + // Helper: ceil(log2(5^e)) + static int ceilLog2pow5(int e) { + return log2pow5(e) + 1; + } + + // Helper: convert uint64 bits to double + static double int64Bits2Double(long bits) { + return Double.longBitsToDouble(bits); + } + + // Check if value is multiple of 2^p + static boolean multipleOfPowerOf2(long value) { + return (value & (value - 1)) == 0; + } + + // Count how many times value is divisible by 5 + // Uses modular inverse to avoid expensive division + static int pow5Factor(long value) { + final long mInv5 = -3689348814741910323L; + final long nDiv5 = 3689348814741910323L; + int count = 0; + for (;;) { + value *= mInv5; + if (Long.compareUnsigned(value, nDiv5) > 0) { + break; + } + ++count; + } + return count; + } + + // Check if value is multiple of 5^p + // Optimized: uses modular inverse instead of division + static boolean multipleOfPowerOf5(long value, int p) { + return pow5Factor(value) >= p; + } + + // Fallback for systems without 128-bit integers + static long umul128(long a, long b, long[] productHi) { + long aLo = Integer.toUnsignedLong((int)a); + long aHi = Integer.toUnsignedLong((int)(a >> 32)); + long bLo = Integer.toUnsignedLong((int)b); + long bHi = Integer.toUnsignedLong((int)(b >> 32)); + + long b00 = aLo * bLo; + long b01 = aLo * bHi; + long b10 = aHi * bLo; + long b11 = aHi * bHi; + + long b00Lo = Integer.toUnsignedLong((int)b00); + long b00Hi = Integer.toUnsignedLong((int)(b00 >> 32)); + + long mid1 = b10 + b00Hi; + long mid1Lo = Integer.toUnsignedLong((int)(mid1)); + long mid1Hi = Integer.toUnsignedLong((int)(mid1 >> 32)); + + long mid2 = b01 + mid1Lo; + long mid2Lo = Integer.toUnsignedLong((int)(mid2)); + long mid2Hi = Integer.toUnsignedLong((int)(mid2 >> 32)); + + long pHi = b11 + mid1Hi + mid2Hi; + long pLo = (mid2Lo << 32) | b00Lo; + + productHi[0] = pHi; + return pLo; + } + + static long shiftright128(long lo, long hi, int dist) { + return (hi << (64 - dist)) | (lo >>> dist); + } + + static long mulShift64(long m, long[] mul, int j) { + long[] high1Buf = new long[1]; + long low1 = umul128(m, mul[1], high1Buf); + long high1 = high1Buf[0]; + long[] high0Buf = high1Buf; // reuse alloc since we extracted high1 into its own long + umul128(m, mul[0], high0Buf); + long high0 = high0Buf[0]; + long sum = high0 + low1; + if (Long.compareUnsigned(sum, high0) < 0) { + ++high1; + } + return shiftright128(sum, high1, j - 64); + } + + // Precomputed powers of ten from 10^0 to 10^22. These + // can be represented exactly using the double type. + private static final double[] POWER_OF_TEN = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; + + // Main conversion function: decimal mantissa+exponent to IEEE 754 double + // Optimized for JSON parsing with fast paths for edge cases + public static double ryuS2dFromParts(long m10, int m10digits, int e10, boolean signedM) { + // From Daniel Lemire's fast_double_parser which is an extraction of part of 'How to Read + // Floating Point Numbers Accurately' by Willian D. Clinger: https://www.cs.uoregon.edu/Reports/TR-1990-001.pdf + if (-22 <= e10 && e10 <= 22 && Long.compareUnsigned(m10, 9007199254740991L) <= 0) { + double value = (double) m10; // Lossless conversion for 0 <= i <= 2^53 - 1. + + if (e10 < 0) { + value /= POWER_OF_TEN[-e10]; + } else { + value *= POWER_OF_TEN[e10]; + } + + return signedM ? -value : value; + } + + // Fast path: handle zero explicitly (e.g., "0.0", "0e0") or overflow/underflow early + if (m10 == 0 || m10digits + e10 <= -324) return signedM ? -0.0 : 0.0; + + // Overflow to infinity + if (m10digits + e10 >= 310) return Double.POSITIVE_INFINITY; + + // Convert decimal to binary: m10 * 10^e10 = m2 * 2^e2 + int e2; + long m2; + boolean trailingZeros; + + if (e10 >= 0) { + // Positive exponent: multiply by 5^e10 and adjust binary exponent + e2 = floorLog2(m10) + e10 + log2pow5(e10) - (DOUBLE_MANTISSA_BITS + 1); + int j = e2 - e10 - ceilLog2pow5(e10) + DOUBLE_POW5_BITCOUNT; + m2 = mulShift64(m10, DOUBLE_POW5_SPLIT[e10], j); + trailingZeros = e2 < e10 || (e2 - e10 < 64 && multipleOfPowerOf2(m10)); + } else { + // Negative exponent: divide by 5^(-e10) + e2 = floorLog2(m10) + e10 - ceilLog2pow5(-e10) - (DOUBLE_MANTISSA_BITS + 1); + int j = e2 - e10 + ceilLog2pow5(-e10) - 1 + DOUBLE_POW5_INV_BITCOUNT; + m2 = mulShift64(m10, DOUBLE_POW5_INV_SPLIT[-e10], j); + trailingZeros = multipleOfPowerOf5(m10, -e10); + } + + // Compute IEEE 754 exponent + int ieee_e2 = Math.max(0, e2 + DOUBLE_EXPONENT_BIAS + floorLog2(m2)); + int signedML = signedM ? 1 : 0; + + if (ieee_e2 > 0x7fe) { + // Overflow to infinity + return int64Bits2Double((signedML << 63) | 0x7ff0000000000000L); + } + + // Compute shift amount for rounding + int shift = (ieee_e2 == 0 ? 1 : ieee_e2) - e2 - DOUBLE_EXPONENT_BIAS - DOUBLE_MANTISSA_BITS; + + // IEEE 754 round-to-even (banker's rounding) + trailingZeros &= (m2 & ((1L << (shift - 1)) - 1)) == 0; + long lastRemovedBit = (m2 >> (shift - 1)) & 1; + int roundUp = (lastRemovedBit != 0) && (!trailingZeros || (((m2 >> shift) & 1) != 0)) ? 1 : 0; + + long ieee_m2 = (m2 >> shift) + roundUp; + ieee_m2 &= (1L << DOUBLE_MANTISSA_BITS) - 1; + + if (ieee_m2 == 0 && roundUp == 1) { + ieee_e2++; + } + + // Pack sign, exponent, and mantissa into IEEE 754 format + // Match original Ryu: group sign+exponent, then shift and add mantissa + long ieee = (((signedML << DOUBLE_EXPONENT_BITS) | (long)ieee_e2) << DOUBLE_MANTISSA_BITS) | ieee_m2; + return int64Bits2Double(ieee); + } +} From bbe22dbaaa0f877bbbaba1ce400abfcb5185013d Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 17 Jun 2026 09:25:31 -0500 Subject: [PATCH 378/506] Add Ryu.class to the jar file. --- Rakefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Rakefile b/Rakefile index 20f1b3d8c..c11d7b8d0 100644 --- a/Rakefile +++ b/Rakefile @@ -166,6 +166,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' "json/ext/ByteList*.class", "json/ext/OptionsReader*.class", "json/ext/Parser*.class", + "json/ext/Ryu*.class", "json/ext/RuntimeInfo*.class", "json/ext/StringDecoder*.class", "json/ext/*StringScanner*.class", From 05bfcb8eb831860a04e86f5a4b18971976749d7a Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 17 Jun 2026 09:28:32 -0500 Subject: [PATCH 379/506] Simplify the fast-path plain string building. --- java/src/json/ext/Parser.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 05c394b20..9ef0554ff 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -726,21 +726,19 @@ private IRubyObject parseString(boolean isName) { return internedKey(buf, off, len); } - ByteList content; + RubyString string; if (plain) { - // Plain ASCII string, skip the decoder. - content = new ByteList(data, contentStart, q - contentStart, true); + string = RubyString.newString(context.runtime, data, contentStart, + q - contentStart, UTF8Encoding.INSTANCE); } else { - content = decoder.decode(context, byteList, - contentStart - begin, q - begin); + ByteList content = decoder.decode(context, byteList, + contentStart - begin, q - begin); + string = context.runtime.newString(content); + string.setEncoding(UTF8Encoding.INSTANCE); + string.clearCodeRange(); } - RubyString string = context.runtime.newString(content); - string.setEncoding(UTF8Encoding.INSTANCE); - string.clearCodeRange(); - if (config.freeze) { - string.setFrozen(true); return context.runtime.freezeAndDedupString(string); } From a842a5c1f6eab272d75c76a466128f48b0842252 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 17 Jun 2026 09:36:17 -0500 Subject: [PATCH 380/506] Undo the pre-sizing of the RubyHash in decodeObject for compatablity with JRuby versions. --- java/src/json/ext/Parser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 9ef0554ff..c54d1a7f9 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -547,7 +547,7 @@ private IRubyObject decodeObject(int count) { final Ruby runtime = context.runtime; int base = valueTop - count; int limit = valueTop; - RubyHash hash = new RubyHash(runtime, count / 2); + RubyHash hash = RubyHash.newHash(runtime); for (int i = base; i < limit; i += 2) { // We use RubyHash#fastASet because all object keys have already been // frozen and deduplicated. From e954a6fe4eb9fc9d8b1ae9c5fe6268ace8ffcd07 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 18 Jun 2026 20:33:37 -0500 Subject: [PATCH 381/506] Update the json_minefield_parser_test.rb to remove JRUBY_PENDING as the new JRuby parser successfully parsed those files. --- test/json/json_minefield_parser_test.rb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/test/json/json_minefield_parser_test.rb b/test/json/json_minefield_parser_test.rb index 715903255..e6dcb54b8 100644 --- a/test/json/json_minefield_parser_test.rb +++ b/test/json/json_minefield_parser_test.rb @@ -11,22 +11,10 @@ class << self private def define_test(name, &block) - if RUBY_ENGINE == 'jruby' && JRUBY_PENDING.include?(name) - define_method("test_#{name}") do - pend("#{name} doesn't pass on JRuby", &block) - end - else - define_method("test_#{name}", &block) - end + define_method("test_#{name}", &block) end end - JRUBY_PENDING = %w( - n_structure_open_array_object - n_structure_100000_opening_arrays - n_object_trailing_comment_slash_open - ).freeze - INVALID_ENCODING_TESTS = %w( i_string_truncated-utf-8 i_string_overlong_sequence_6_bytes_null From 97b2eee92cd15f0f487c887588701a2a38171c3c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 09:30:58 +0200 Subject: [PATCH 382/506] Remove reference to ragel --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e1592ac68..cb57ab693 100644 --- a/README.md +++ b/README.md @@ -306,5 +306,3 @@ The latest version of this library can be downloaded at Online Documentation should be located at * https://www.rubydoc.info/gems/json - -[Ragel]: http://www.colm.net/open-source/ragel/ From 21c2bbea1c57de3b5e4e24288684044d5cbcf7de Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 10:18:25 +0200 Subject: [PATCH 383/506] Implement ResumableParser#parsed_bytes This is intended to make it easier to securely parse untrusted inputs. --- ext/json/ext/parser/parser.c | 69 +++++++++++++++++++++++++++--- test/json/resumable_parser_test.rb | 25 ++++++++++- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 7b028635c..59cdc4273 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2162,6 +2162,9 @@ typedef struct JSON_ResumableParserStruct { rvalue_stack value_stack; json_frame_stack frames; VALUE buffer; + size_t parsed_bytes; + size_t incomplete_bytes; + bool complete; bool in_use; } JSON_ResumableParser; @@ -2282,6 +2285,18 @@ static inline JSON_ResumableParser *cResumableParser_get(VALUE self) * * An incomplete document is buffered in full and there is no size limit, so when reading * from an untrusted source the caller is responsible for bounding how much data is fed. + * For example: + * + * loop do + * if parser.parsed_bytes > DOCUMENT_MAX_SIZE + * raise "document too large" + * end + * + * parser << read_chunk + * while parser.parse + * process(parser.value) + * end + * end */ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) { @@ -2398,6 +2413,13 @@ static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) static VALUE cResumableParser_parse(VALUE self) { JSON_ResumableParser *parser = ResumableParser_acquire(self, true); + + if (parser->complete) { + parser->parsed_bytes = 0; + parser->incomplete_bytes = 0; + parser->complete = false; + } + if (!parser->buffer) { parser->in_use = false; return Qfalse; @@ -2427,20 +2449,28 @@ static VALUE cResumableParser_parse(VALUE self) .config = &parser->config, }; int status; - bool complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status); - parser->in_use = false; + const char *initial_cursor = parser->state.cursor; + parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status); if (status) { - complete = false; VALUE error_source = rb_ivar_get(rb_errinfo(), i_at_eos); if (error_source == self) { - complete = false; // is an EOS error raised by ourself + parser->complete = false; // is an EOS error raised by ourself rb_set_errinfo(Qnil); + status = 0; } else { - rb_jump_tag(status); // reraise + parser->complete = true; // a parse error is considered complete } } + + parser->parsed_bytes += parser->state.cursor - initial_cursor; + parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor; + + parser->in_use = false; + if (status) { + rb_jump_tag(status); // reraise + } RB_GC_GUARD(Vsource); - return complete ? Qtrue : Qfalse; + return parser->complete ? Qtrue : Qfalse; } /* @@ -2498,6 +2528,9 @@ static VALUE cResumableParser_clear(VALUE self) { JSON_ResumableParser *parser = ResumableParser_acquire(self, false); parser->buffer = 0; + parser->complete = true; + parser->parsed_bytes = 0; + parser->incomplete_bytes = 0; parser->frames.head = 0; parser->value_stack.head = 0; parser->state.name_cache.length = 0; @@ -2633,6 +2666,29 @@ static VALUE cResumableParser_eos_p(VALUE self) return eos(&parser->state) ? Qtrue : Qfalse; } +/* + * call-seq: parsed_bytes -> integer + * + * Returns the number of bytes parsed since the start of the current partial value. + * This is intended to be used for securing against untrusted input: + * + * loop do + * if parser.parsed_bytes > DOCUMENT_MAX_SIZE + * raise "document too large" + * end + * + * parser << read_chunk + * while parser.parse + * process(parser.value) + * end + * end + */ +static VALUE cResumableParser_parsed_bytes(VALUE self) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes); +} + void Init_parser(void) { #ifdef HAVE_RB_EXT_RACTOR_SAFE @@ -2669,6 +2725,7 @@ void Init_parser(void) rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0); rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0); rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0); + rb_define_method(cResumableParser, "parsed_bytes", cResumableParser_parsed_bytes, 0); rb_global_variable(&CNaN); CNaN = rb_const_get(mJSON, rb_intern("NaN")); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index a720679c2..3e22a18e0 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -156,7 +156,7 @@ def test_parse_byte_by_byte_string end def test_parse_byte_by_byte_numbers - assert_resumed_parsing('123 ') + assert_resumed_parsing('123 ', trailing_bytes: 1) end def test_nul_byte_is_a_syntax_error @@ -364,6 +364,26 @@ def test_buffer_shrink parser.value end + def test_parsed_bytes + chunk = '[1, 2, 3, 4, tru' + @parser << chunk + refute @parser.parse + assert_equal chunk.bytesize, @parser.parsed_bytes + + @parser << 'e][]' + assert @parser.parse + assert_equal chunk.bytesize + 2, @parser.parsed_bytes + + assert @parser.parse + assert_equal 2, @parser.parsed_bytes + + @parser << chunk + refute @parser.parse + assert_equal chunk.bytesize, @parser.parsed_bytes + @parser.clear + assert_equal 0, @parser.parsed_bytes + end + private def assert_parse_error(json) @@ -389,7 +409,7 @@ def assert_partial_value(expected, json) end end - def assert_resumed_parsing(json, parser = @parser) + def assert_resumed_parsing(json, parser = @parser, trailing_bytes: 0) expected = JSON.parse(json) last_parsed_byte_index = 0 @@ -402,6 +422,7 @@ def assert_resumed_parsing(json, parser = @parser) assert_equal expected, actual remaining_bytes = (json.bytesize - last_parsed_byte_index) assert_equal 0, remaining_bytes, "unconsumed bytes: #{actual.inspect}, remaining: #{json.byteslice(-1, remaining_bytes).inspect}" + assert_equal json.bytesize - trailing_bytes, parser.parsed_bytes end def assert_parse_stream(expected, json, parser = @parser) From b2372115afeeb6cef346db131c3819f5e309f449 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 10:34:15 +0200 Subject: [PATCH 384/506] ResumableParser: Don't compute lines and columns on parse error Fix: https://github.com/ruby/json/issues/1022 They can't always be accurate because we don't always keep the full document in the buffer. As such it's better never to compute them than to sometimes provide wrong coordinates. In theory we could keep the number of lines since the start of the parse, but that's more book keeping for little utility. Anyway, these are useful to find a syntax error in a file, not so much in a stream of documents. --- ext/json/ext/parser/parser.c | 32 +++++++++++++++++++----------- test/json/resumable_parser_test.rb | 14 +++++++++++-- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 59cdc4273..a39b62205 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -627,7 +627,7 @@ static void emit_parse_warning(const char *message, JSON_ParserState *state) #define PARSE_ERROR_FRAGMENT_LEN 32 -static VALUE build_parse_error_message(const char *format, JSON_ParserState *state, long line, long column) +static VALUE build_parse_error_message(const char *format, JSON_ParserState *state) { unsigned char buffer[PARSE_ERROR_FRAGMENT_LEN + 3]; @@ -661,9 +661,7 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta } } - VALUE message = rb_enc_sprintf(enc_utf8, format, ptr); - rb_str_catf(message, " at line %ld column %ld", line, column); - return message; + return rb_enc_sprintf(enc_utf8, format, ptr); } static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos) @@ -679,10 +677,15 @@ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos) { - long line, column; - cursor_position(state, &line, &column); - VALUE message = build_parse_error_message(format, state, line, column); - rb_exc_raise(parse_error_new(state, message, line, column, eos)); + VALUE message = build_parse_error_message(format, state); + if (state->parser) { // line and columns can't be accurate in resumable + rb_exc_raise(parse_error_new(state, message, 0, 0, eos)); + } else { + long line, column; + cursor_position(state, &line, &column); + rb_str_catf(message, " at line %ld column %ld", line, column); + rb_exc_raise(parse_error_new(state, message, line, column, eos)); + } } NORETURN(static) void raise_eos_error(const char *format, JSON_ParserState *state) @@ -1172,10 +1175,15 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d rb_inspect(duplicate_key) ); - long line, column; - cursor_position(state, &line, &column); - rb_str_concat(message, build_parse_error_message("", state, line, column)) ; - rb_exc_raise(parse_error_new(state, message, line, column, false)); + rb_str_concat(message, build_parse_error_message("", state)); + if (state->parser) { // line and columns can't be accurate in resumable + rb_exc_raise(parse_error_new(state, message, 0, 0, false)); + } else { + long line, column; + cursor_position(state, &line, &column); + rb_str_catf(message, " at line %ld column %ld", line, column); + rb_exc_raise(parse_error_new(state, message, line, column, false)); + } } NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 3e22a18e0..1a981e6b8 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -384,14 +384,24 @@ def test_parsed_bytes assert_equal 0, @parser.parsed_bytes end + def test_parse_error_message + error = assert_parse_error("\n\n[plop\nfoo", "unexpected character: 'plop'") + assert_equal 0, error.line + assert_equal 0, error.column + end + private - def assert_parse_error(json) + def assert_parse_error(json, expected_error_message = nil) parser = new_parser parser << json - assert_raise(JSON::ParserError, "expected a parse error for #{json.inspect}") do + error = assert_raise(JSON::ParserError, "expected a parse error for #{json.inspect}") do parser.parse end + if expected_error_message + assert_equal expected_error_message, error.message + end + error end def assert_incomplete(json) From 181396ca3d89d3247454ec698fc9a438580e2514 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 13:30:25 +0200 Subject: [PATCH 385/506] Don't omit `test_parse_error_snippet` on TruffleRuby --- test/json/json_parser_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index c891dd7c2..9000d1152 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -862,7 +862,7 @@ def test_parse_error_incomplete_hash end def test_parse_error_snippet - omit "JRuby errors don't contain positions" unless RUBY_ENGINE == "ruby" + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" error = assert_raise(JSON::ParserError) { JSON.parse("あああああああああああああああああああああああ") } assert_equal "unexpected character: 'ああああああああああ' at line 1 column 1", error.message From 4bd1e9bce64c82b6e829f4583a62bab485529a00 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 11:51:52 +0200 Subject: [PATCH 386/506] ResumableParser: use throw rather than raise for handled EOS Since the exception will be swallowed, building a message and backtrace is just a waste of time. --- ext/json/ext/parser/parser.c | 39 ++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a39b62205..2c5c4a5a4 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -677,10 +677,17 @@ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos) { - VALUE message = build_parse_error_message(format, state); - if (state->parser) { // line and columns can't be accurate in resumable - rb_exc_raise(parse_error_new(state, message, 0, 0, eos)); + if (state->parser) { + if (eos) { + // the error will be swallowed by ResumableParser#parse, so no + // point building a message or backtrace. + rb_throw_obj(state->parser, state->parser); + } else { + // line and columns can't be accurate in resumable + rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos)); + } } else { + VALUE message = build_parse_error_message(format, state); long line, column; cursor_position(state, &line, &column); rb_str_catf(message, " at line %ld column %ld", line, column); @@ -2379,14 +2386,25 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) struct json_parse_any_args { JSON_ParserState *state; JSON_ParserConfig *config; + VALUE parser; }; -static VALUE json_parse_any_resumable_safe(VALUE _args) +static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args)) { struct json_parse_any_args *args = (struct json_parse_any_args *)_args; return (VALUE)json_parse_any(args->state, args->config, true); } +static VALUE json_parse_any_resumable_safe(VALUE _args) +{ + struct json_parse_any_args *args = (struct json_parse_any_args *)_args; + VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args); + if (result == args->parser) { + return (VALUE)false; + } + return result; +} + static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) { JSON_ResumableParser *parser = cResumableParser_get(self); @@ -2455,25 +2473,20 @@ static VALUE cResumableParser_parse(VALUE self) struct json_parse_any_args args = { .state = &parser->state, .config = &parser->config, + .parser = self, }; int status; const char *initial_cursor = parser->state.cursor; parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status); + if (status) { - VALUE error_source = rb_ivar_get(rb_errinfo(), i_at_eos); - if (error_source == self) { - parser->complete = false; // is an EOS error raised by ourself - rb_set_errinfo(Qnil); - status = 0; - } else { - parser->complete = true; // a parse error is considered complete - } + parser->complete = true; // a parse error is considered complete } parser->parsed_bytes += parser->state.cursor - initial_cursor; parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor; - parser->in_use = false; + if (status) { rb_jump_tag(status); // reraise } From 9d8efcb08b90b1543765e50783efaabc6f921ad6 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 12:46:23 +0200 Subject: [PATCH 387/506] Workaround TruffleRuby buggy `rb_catch_obj` implementation Somehow on TruffleRuby `rb_catch_obj` straight out doesn't call the passed function, acting as a noop. --- ext/json/ext/parser/extconf.rb | 2 ++ ext/json/ext/parser/parser.c | 47 +++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index bd537c9c5..8c5bdb66e 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -2,6 +2,8 @@ require 'mkmf' $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" +$defs << "-DJSON_WORKAROUND_RB_CATCH_BUG" if RUBY_ENGINE == 'truffleruby' + have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 have_func("rb_hash_new_capa", "ruby.h") # RUBY_VERSION >= 3.2 diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 2c5c4a5a4..057367bc5 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -5,7 +5,7 @@ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity; -static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column, i_at_eos; +static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column; static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, @@ -669,19 +669,46 @@ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); rb_ivar_set(exc, i_at_column, LONG2NUM(column)); - if (eos && state->parser) { - rb_ivar_set(exc, i_at_eos, state->parser); - } return exc; } +#ifdef JSON_WORKAROUND_RB_CATCH_BUG +#define JSON_CATCH_FUNC_ARGLIST(yielded_arg, func_args) VALUE func_args + +NORETURN(static) void parser_throw_eos(VALUE parser) +{ + VALUE exc = rb_exc_new_str(eParserError, rb_utf8_str_new_cstr("EOS")); + rb_ivar_set(exc, rb_intern("@resumable_parser_eos"), parser); + rb_exc_raise(exc); +} + +static VALUE parser_catch_eos(VALUE parser, VALUE (*func)(VALUE args), VALUE func_args) +{ + int status; + VALUE result = rb_protect(func, func_args, &status); + if (status) { + VALUE error_source = rb_ivar_get(rb_errinfo(), rb_intern("@resumable_parser_eos")); + if (error_source == parser) { + rb_set_errinfo(Qnil); + return parser; + } + rb_jump_tag(status); + } + return result; +} +#else +#define JSON_CATCH_FUNC_ARGLIST RB_BLOCK_CALL_FUNC_ARGLIST +#define parser_throw_eos(parser) rb_throw_obj(parser, parser) +#define parser_catch_eos(parser, func, func_args) rb_catch_obj(parser, func, func_args) +#endif + NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos) { if (state->parser) { if (eos) { // the error will be swallowed by ResumableParser#parse, so no // point building a message or backtrace. - rb_throw_obj(state->parser, state->parser); + parser_throw_eos(state->parser); } else { // line and columns can't be accurate in resumable rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos)); @@ -2389,7 +2416,7 @@ struct json_parse_any_args { VALUE parser; }; -static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args)) +static VALUE json_parse_any_resumable_safe0(JSON_CATCH_FUNC_ARGLIST(yielded_arg, _args)) { struct json_parse_any_args *args = (struct json_parse_any_args *)_args; return (VALUE)json_parse_any(args->state, args->config, true); @@ -2398,11 +2425,8 @@ static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_a static VALUE json_parse_any_resumable_safe(VALUE _args) { struct json_parse_any_args *args = (struct json_parse_any_args *)_args; - VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args); - if (result == args->parser) { - return (VALUE)false; - } - return result; + VALUE result = parser_catch_eos(args->parser, json_parse_any_resumable_safe0, _args); + return result == args->parser ? Qfalse : result; } static JSON_ResumableParser *ResumableParser_acquire(VALUE self, bool lock) @@ -2778,7 +2802,6 @@ void Init_parser(void) i_encode = rb_intern("encode"); i_at_line = rb_intern("@line"); i_at_column = rb_intern("@column"); - i_at_eos = rb_intern("@eos"); binary_encindex = rb_ascii8bit_encindex(); utf8_encindex = rb_utf8_encindex(); From f08c66338cd0155a44dbda81d889f6db9ba4d205 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 14:06:47 +0200 Subject: [PATCH 388/506] ResumableParser: accept only keyword arguments Fix: https://github.com/ruby/json/pull/1016#issuecomment-4744487710 `json` takes option hashes across the board, mostly because its API predates the introduction of keyword arguments. I'd like to change that to only take keyword arguments and error when an unknown argument is passed, but I'm not yet sure of the backward compatibility consequences, so it might wait for the next major. But in the meantime, `ResumableParser` being a new API, it can safely use keyword arguments. --- ext/json/ext/parser/parser.c | 46 +++++++++++++++++++++--------- test/json/resumable_parser_test.rb | 20 +++++++++++-- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 057367bc5..e4dc3fb01 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1966,6 +1966,8 @@ static VALUE convert_encoding(VALUE source) struct parser_config_init_args { JSON_ParserConfig *config; VALUE self; + VALUE unknown_keywords; + bool strict; }; static void parser_config_wb_write(VALUE self, VALUE *dest, VALUE val) @@ -2019,27 +2021,43 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) } } } + else if (args->strict) { + if (!args->unknown_keywords) { + args->unknown_keywords = rb_obj_hide(rb_ary_new()); + } + rb_ary_push(args->unknown_keywords, key); + } return ST_CONTINUE; } -static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self) +static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self, bool strict) { config->max_nesting = 100; struct parser_config_init_args args = { .config = config, .self = self, + .strict = strict, }; - if (!NIL_P(opts)) { - Check_Type(opts, T_HASH); - if (RHASH_SIZE(opts) > 0) { - // We assume in most cases few keys are set so it's faster to go over - // the provided keys than to check all possible keys. - rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args); - } + if (NIL_P(opts)) return; + Check_Type(opts, T_HASH); + if (RHASH_SIZE(opts) == 0) return; + + // We assume in most cases few keys are set so it's faster to go over + // the provided keys than to check all possible keys. + rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args); + if (RB_UNLIKELY(args.unknown_keywords)) { + if (RARRAY_LEN(args.unknown_keywords) == 1) { + rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0)); + } + else { + VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", ")); + rb_raise(rb_eArgError, "unknown keywords: %s", RSTRING_PTR(keywords)); + RB_GC_GUARD(keywords); + } } } @@ -2057,7 +2075,7 @@ static VALUE cParserConfig_initialize(VALUE self, VALUE opts) rb_check_frozen(self); GET_PARSER_CONFIG; - parser_config_init(config, opts, self); + parser_config_init(config, opts, self, false); return self; } @@ -2153,7 +2171,7 @@ static VALUE cParser_m_parse(VALUE klass, VALUE Vsource, VALUE opts) { JSON_ParserConfig _config = {0}; JSON_ParserConfig *config = &_config; - parser_config_init(config, opts, false); + parser_config_init(config, opts, Qfalse, false); return cParser_parse(config, Vsource); } @@ -2342,12 +2360,14 @@ static inline JSON_ResumableParser *cResumableParser_get(VALUE self) */ static VALUE cResumableParser_initialize(int argc, VALUE *argv, VALUE self) { - rb_check_arity(argc, 0, 1); rb_check_frozen(self); + + VALUE opts = Qfalse; + rb_scan_args_kw(RB_SCAN_ARGS_LAST_HASH_KEYWORDS, argc, argv, "0:", &opts); JSON_ResumableParser *parser = cResumableParser_get(self); - VALUE opts = argc > 0 ? argv[0] : Qnil; - parser_config_init(&parser->config, opts, self); + opts = argc > 0 ? argv[0] : Qnil; + parser_config_init(&parser->config, opts, self, true); return self; } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 1a981e6b8..800502f4d 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -9,6 +9,22 @@ def setup @parser = new_parser end + def test_keyword_arguments + new_parser + new_parser({}) + new_parser(allow_nan: true) + + error = assert_raise(ArgumentError) do + new_parser(doesnt_exist: true, allow_nan: true) + end + assert_equal "unknown keyword: doesnt_exist", error.message + + error = assert_raise(ArgumentError) do + new_parser(doesnt_exist: true, allow_nan: true, a: 1, b: 2) + end + assert_equal "unknown keywords: doesnt_exist, a, b", error.message + end + def test_value refute_predicate @parser, :value? assert_raise(ArgumentError) { @parser.value } @@ -444,7 +460,7 @@ def assert_parse_stream(expected, json, parser = @parser) assert_equal(expected, actual) end - def new_parser(options = nil) - JSON::ResumableParser.new(options) + def new_parser(...) + JSON::ResumableParser.new(...) end end From b30a8f852dada93093a215b1519cc3bde526316c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 19 Jun 2026 15:52:20 +0200 Subject: [PATCH 389/506] ResumableParser: eagerly drop the buffer when reaching EOS No point to wait for the next call to `<<` to remove the reference. --- ext/json/ext/parser/parser.c | 48 ++++++++++++++++++++++-------- test/json/resumable_parser_test.rb | 2 +- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e4dc3fb01..948ea20f9 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -3,7 +3,7 @@ #include "../simd/simd.h" static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; -static VALUE CNaN, CInfinity, CMinusInfinity; +static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string; static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column; @@ -843,7 +843,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) } ALWAYS_INLINE(static) void -json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config) +json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments) { while (true) { switch (peek(state)) { @@ -874,6 +874,10 @@ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config) state->cursor++; break; case '/': + if (!include_comments) { + return; + } + json_eat_comments(state, config); break; @@ -1587,7 +1591,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_UNREACHABLE_RETURN(false); JSON_PHASE_VALUE: { - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); VALUE value; const char *value_start = state->cursor; @@ -1684,7 +1688,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '[': { state->cursor++; - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); const char next = peek(state); if (next == ']') { @@ -1713,7 +1717,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '{': { state->cursor++; - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); if (peek(state) == '}') { state->cursor++; @@ -1769,7 +1773,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_PHASE_OBJECT_KEY: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); const char *start = state->cursor; @@ -1804,7 +1808,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_PHASE_OBJECT_COLON: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); if (RB_LIKELY(peek(state) == ':')) { state->cursor++; @@ -1827,14 +1831,14 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_PHASE_ARRAY_COMMA: { JSON_ASSERT(frame->type == JSON_FRAME_ARRAY); - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); const char next_char = peek(state); if (RB_LIKELY(next_char == ',')) { state->cursor++; if (config->allow_trailing_comma) { - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); if (peek(state) == ']') { // Trailing comma: stay in COMMA to close on the next iteration. goto JSON_PHASE_ARRAY_COMMA; @@ -1873,12 +1877,12 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_PHASE_OBJECT_COMMA: { JSON_ASSERT(frame->type == JSON_FRAME_OBJECT); - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); const char next_char = peek(state); if (RB_LIKELY(next_char == ',')) { state->cursor++; - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); if (config->allow_trailing_comma) { if (peek(state) == '}') { @@ -1926,7 +1930,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo static void json_ensure_eof(JSON_ParserState *state, JSON_ParserConfig *config) { - json_eat_whitespace(state, config); + json_eat_whitespace(state, config, true); if (!eos(state)) { raise_syntax_error("unexpected token at end of stream %s", state); } @@ -2216,6 +2220,14 @@ static VALUE cJSON_parser_s_allocate(VALUE klass) return TypedData_Make_Struct(klass, JSON_ParserConfig, &JSON_ParserConfig_type, config); } +static void json_str_clear(VALUE str) +{ + if (RB_OBJ_FROZEN_RAW(str)) { + return; + } + rb_str_replace(str, JSON_empty_string); +} + typedef struct JSON_ResumableParserStruct { JSON_ParserConfig config; JSON_ParserState state; @@ -2394,6 +2406,9 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) const size_t remaining = parser->state.end - parser->state.cursor; if (!remaining) { + if (parser->buffer) { + json_str_clear(parser->buffer); + } parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str)); offset = 0; } else { @@ -2529,6 +2544,12 @@ static VALUE cResumableParser_parse(VALUE self) parser->parsed_bytes += parser->state.cursor - initial_cursor; parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor; + + json_eat_whitespace(&parser->state, &parser->config, false); + if (eos(&parser->state)) { + json_str_clear(parser->buffer); + parser->buffer = Qfalse; + } parser->in_use = false; if (status) { @@ -2804,6 +2825,9 @@ void Init_parser(void) rb_global_variable(&Encoding_UTF_8); Encoding_UTF_8 = rb_const_get(rb_path2class("Encoding"), rb_intern("UTF_8")); + rb_global_variable(&JSON_empty_string); + JSON_empty_string = rb_obj_hide(rb_utf8_str_new("", 0)); + sym_max_nesting = ID2SYM(rb_intern("max_nesting")); sym_allow_nan = ID2SYM(rb_intern("allow_nan")); sym_allow_trailing_comma = ID2SYM(rb_intern("allow_trailing_comma")); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 800502f4d..e93b2314c 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -223,7 +223,7 @@ def test_eos @parser << ' ' assert @parser.parse assert_equal 123, @parser.value - refute_predicate @parser, :eos? + assert_predicate @parser, :eos? refute @parser.parse assert_predicate @parser, :eos? From 98925145d2ac1f87a6bb3e5add1fb5dd2a9901f8 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 20 Jun 2026 15:23:57 +0200 Subject: [PATCH 390/506] Simplify parser_config_init --- ext/json/ext/parser/parser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 948ea20f9..1a407d875 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2059,8 +2059,7 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self } else { VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", ")); - rb_raise(rb_eArgError, "unknown keywords: %s", RSTRING_PTR(keywords)); - RB_GC_GUARD(keywords); + rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords); } } } From 2afd1a9a61de2e58ff5a65c724de22d91c37db2e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 21 Jun 2026 10:19:20 +0200 Subject: [PATCH 391/506] Cleanup the `rb_catch_obj` workaround Followup: https://github.com/ruby/json/pull/1031 Now that the bug has been fixed upstream, we can only apply it on unpatched version. Also refactor the workaround to expose the exact same API as `rb_catch_obj`, making it easier to ignore and remove later. --- ext/json/ext/json.h | 32 ++++++++++++++++++++++++++++++ ext/json/ext/parser/extconf.rb | 7 ++++++- ext/json/ext/parser/parser.c | 36 +++------------------------------- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index a29d37bee..f39e795a4 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -144,4 +144,36 @@ static inline void *ruby_xrealloc2_sized(void *ptr, size_t new_elems, size_t ele #define JSON_CPU_LITTLE_ENDIAN_64BITS 0 #endif +#ifdef JSON_TRUFFLERUBY_RB_CATCH_BUG + +#undef RB_BLOCK_CALL_FUNC_ARGLIST +#define RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, func_args) VALUE func_args + +NORETURN(static inline) void json_rb_throw_obj(VALUE tag, VALUE obj) +{ + VALUE exc = rb_exc_new_str(rb_eException, rb_utf8_str_new_cstr("throw_workaround")); + rb_ivar_set(exc, rb_intern("@throw_tag"), tag); + rb_ivar_set(exc, rb_intern("@throw_obj"), obj); + rb_exc_raise(exc); +} +#define rb_throw_obj json_rb_throw_obj + +static inline VALUE json_rb_catch_obj(VALUE tag, VALUE (*func)(VALUE args), VALUE func_args) +{ + int status; + VALUE result = rb_protect(func, func_args, &status); + if (status) { + VALUE exc = rb_errinfo(); + if (tag == rb_ivar_get(exc, rb_intern("@throw_tag"))) { + rb_set_errinfo(Qnil); + return rb_ivar_get(exc, rb_intern("@throw_obj")); + } + rb_jump_tag(status); + } + return result; +} +#define rb_catch_obj json_rb_catch_obj + +#endif // JSON_TRUFFLERUBY_RB_CATCH_BUG + #endif // _JSON_H_ diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 8c5bdb66e..61ca092a2 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -2,7 +2,12 @@ require 'mkmf' $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" -$defs << "-DJSON_WORKAROUND_RB_CATCH_BUG" if RUBY_ENGINE == 'truffleruby' + +if RUBY_ENGINE == 'truffleruby' && RUBY_VERSION < '4.0' + # Ref: https://github.com/truffleruby/truffleruby/issues/4329 + # Ref: https://github.com/truffleruby/truffleruby/pull/4333 + $defs << "-DJSON_TRUFFLERUBY_RB_CATCH_BUG" +end have_func("rb_enc_interned_str", "ruby/encoding.h") # RUBY_VERSION >= 3.0 have_func("rb_str_to_interned_str", "ruby.h") # RUBY_VERSION >= 3.0 diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1a407d875..c774c7915 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -672,43 +672,13 @@ static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, return exc; } -#ifdef JSON_WORKAROUND_RB_CATCH_BUG -#define JSON_CATCH_FUNC_ARGLIST(yielded_arg, func_args) VALUE func_args - -NORETURN(static) void parser_throw_eos(VALUE parser) -{ - VALUE exc = rb_exc_new_str(eParserError, rb_utf8_str_new_cstr("EOS")); - rb_ivar_set(exc, rb_intern("@resumable_parser_eos"), parser); - rb_exc_raise(exc); -} - -static VALUE parser_catch_eos(VALUE parser, VALUE (*func)(VALUE args), VALUE func_args) -{ - int status; - VALUE result = rb_protect(func, func_args, &status); - if (status) { - VALUE error_source = rb_ivar_get(rb_errinfo(), rb_intern("@resumable_parser_eos")); - if (error_source == parser) { - rb_set_errinfo(Qnil); - return parser; - } - rb_jump_tag(status); - } - return result; -} -#else -#define JSON_CATCH_FUNC_ARGLIST RB_BLOCK_CALL_FUNC_ARGLIST -#define parser_throw_eos(parser) rb_throw_obj(parser, parser) -#define parser_catch_eos(parser, func, func_args) rb_catch_obj(parser, func, func_args) -#endif - NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *state, bool eos) { if (state->parser) { if (eos) { // the error will be swallowed by ResumableParser#parse, so no // point building a message or backtrace. - parser_throw_eos(state->parser); + rb_throw_obj(state->parser, state->parser); } else { // line and columns can't be accurate in resumable rb_exc_raise(parse_error_new(state, build_parse_error_message(format, state), 0, 0, eos)); @@ -2450,7 +2420,7 @@ struct json_parse_any_args { VALUE parser; }; -static VALUE json_parse_any_resumable_safe0(JSON_CATCH_FUNC_ARGLIST(yielded_arg, _args)) +static VALUE json_parse_any_resumable_safe0(RB_BLOCK_CALL_FUNC_ARGLIST(yielded_arg, _args)) { struct json_parse_any_args *args = (struct json_parse_any_args *)_args; return (VALUE)json_parse_any(args->state, args->config, true); @@ -2459,7 +2429,7 @@ static VALUE json_parse_any_resumable_safe0(JSON_CATCH_FUNC_ARGLIST(yielded_arg, static VALUE json_parse_any_resumable_safe(VALUE _args) { struct json_parse_any_args *args = (struct json_parse_any_args *)_args; - VALUE result = parser_catch_eos(args->parser, json_parse_any_resumable_safe0, _args); + VALUE result = rb_catch_obj(args->parser, json_parse_any_resumable_safe0, _args); return result == args->parser ? Qfalse : result; } From 7c8af4b9ef509f6d989a0da1d1cdfdd7650ae10b Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Sun, 21 Jun 2026 13:24:51 +0200 Subject: [PATCH 392/506] Update extconf.rb guard to use RUBY_ENGINE_VERSION --- ext/json/ext/parser/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 61ca092a2..9e141a726 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -3,7 +3,7 @@ $defs << "-DJSON_DEBUG" if ENV.fetch("JSON_DEBUG", "0") != "0" -if RUBY_ENGINE == 'truffleruby' && RUBY_VERSION < '4.0' +if RUBY_ENGINE == 'truffleruby' && RUBY_ENGINE_VERSION < '40.0' # Ref: https://github.com/truffleruby/truffleruby/issues/4329 # Ref: https://github.com/truffleruby/truffleruby/pull/4333 $defs << "-DJSON_TRUFFLERUBY_RB_CATCH_BUG" From 532065c259a87c78a0a2af1e6ea0177aeca85539 Mon Sep 17 00:00:00 2001 From: Masataka Pocke Kuwabara Date: Mon, 22 Jun 2026 16:28:00 +0900 Subject: [PATCH 393/506] Preserve UTF-8 encoding when reallocating a frozen ResumableParser buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON::ResumableParser#<<` raises `Encoding::CompatibilityError` when the first chunk is a frozen multibyte UTF-8 string and more data is fed after a partial parse. ## Reproduction ```ruby require "json" parser = JSON::ResumableParser.new parser << '{"message":"日本'.freeze # frozen + multibyte (UTF-8) parser.parse # => false (incomplete) parser << '語のつづき"}'.freeze # => Encoding::CompatibilityError: BINARY and UTF-8 ``` The error only surfaces when the first chunk is both frozen and multibyte. A frozen ASCII-only first chunk stays ASCII-compatible and a mutable first chunk keeps its encoding, so neither triggers it. This is common in streaming use cases where chunks arrive as frozen UTF-8 (e.g. gRPC/protobuf string fields), and is easily hit with multibyte (e.g. Japanese) payloads. ## Root cause In `cResumableParser_feed`, the first feed adopts a frozen input string directly as the buffer (`parser->buffer = str`), keeping its UTF-8 encoding. When a later feed needs to reallocate that frozen buffer, it allocates the new buffer with `rb_str_buf_new()`, which returns an ASCII-8BIT (BINARY) string and does not carry over the original UTF-8 encoding. The subsequent `rb_str_append(parser->buffer, str)` then appends a multibyte UTF-8 chunk to a BINARY buffer that already holds non-ASCII bytes, and the two encodings are incompatible. `convert_encoding` always normalizes the input to UTF-8, so the buffer is guaranteed to be UTF-8. Associating the freshly allocated buffer with UTF-8 restores that invariant. This mirrors the existing idiom used for the generator buffer, and also fixes the latent case where a frozen ASCII-only first chunk would silently leave the buffer as BINARY. ## Test Added a regression test feeding frozen multibyte chunks across a partial parse. The byte-by-byte `assert_resumed_parsing` helper cannot cover this: it feeds single ASCII-8BIT bytes via `byte.chr`, which `convert_encoding` turns into a mutable dup, so the buffer never becomes frozen and the reallocation path is never taken. The `.freeze` calls are kept explicit so the test still exercises the frozen path even if the file's `frozen_string_literal` magic comment is ever removed. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01PpeQSEFkF1X1Uuzjx9BsYe --- ext/json/ext/parser/parser.c | 1 + test/json/resumable_parser_test.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c774c7915..9a86cd58b 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2388,6 +2388,7 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) if (RB_OBJ_FROZEN_RAW(parser->buffer)) { VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str))); + rb_enc_associate_index(new_buffer, utf8_encindex); char *old_ptr = RSTRING_PTR(parser->buffer); memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining); diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index e93b2314c..4cc6c085d 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -205,6 +205,16 @@ def test_rest assert_equal '"unterminated string', @parser.rest end + def test_feed_frozen_multibyte_chunks + @parser << '{"message":"日本'.freeze + refute @parser.parse + @parser << '語のつづき"}'.freeze + assert @parser.parse + value = @parser.value + assert_equal({ "message" => "日本語のつづき" }, value) + assert_equal Encoding::UTF_8, value["message"].encoding + end + def test_eos assert_predicate @parser, :eos? From 1443265cd178fcacfea41ba531b770c2dbf7feef Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 23 Jun 2026 09:41:47 +0900 Subject: [PATCH 394/506] Remove useless executable bits --- .../minefield/i_number_neg_int_huge_exp.json | 0 .../minefield/i_number_pos_double_huge_exp.json | 0 .../minefield/i_number_very_big_negative_int.json | 0 .../minefield/i_string_UTF-8_invalid_sequence.json | 0 ...tring_incomplete_surrogate_and_escape_valid.json | 0 .../i_string_incomplete_surrogate_pair.json | 0 ...i_string_incomplete_surrogates_escape_valid.json | 0 .../i_string_invalid_lonely_surrogate.json | 0 .../minefield/i_string_invalid_surrogate.json | 0 .../i_string_inverted_surrogates_U+1D11E.json | 0 .../i_string_overlong_sequence_6_bytes.json | 0 .../i_string_overlong_sequence_6_bytes_null.json | 0 .../i_structure_UTF-8_BOM_empty_object.json | 0 .../minefield/n_array_comma_and_number.json | 0 .../fixtures/minefield/n_array_double_comma.json | 0 .../n_array_items_separated_by_semicolon.json | 0 .../json/fixtures/minefield/n_array_just_comma.json | 0 .../json/fixtures/minefield/n_array_just_minus.json | 0 .../minefield/n_array_number_and_comma.json | 0 .../n_array_number_and_several_commas.json | 0 .../n_array_spaces_vertical_tab_formfeed.json | 0 .../fixtures/minefield/n_array_star_inside.json | 0 test/json/fixtures/minefield/n_number_+1.json | 0 test/json/fixtures/minefield/n_number_+Inf.json | 0 test/json/fixtures/minefield/n_number_-01.json | 0 test/json/fixtures/minefield/n_number_-1.0..json | 0 test/json/fixtures/minefield/n_number_-2..json | 0 test/json/fixtures/minefield/n_number_-NaN.json | 0 test/json/fixtures/minefield/n_number_.2e-3.json | 0 test/json/fixtures/minefield/n_number_0.1.2.json | 0 .../fixtures/minefield/n_number_0_capital_E.json | 0 test/json/fixtures/minefield/n_number_1.0e+.json | 0 test/json/fixtures/minefield/n_number_1.0e-.json | 0 test/json/fixtures/minefield/n_number_1.0e.json | 0 test/json/fixtures/minefield/n_number_1_000.json | 0 test/json/fixtures/minefield/n_number_1eE2.json | 0 test/json/fixtures/minefield/n_number_2.e+3.json | 0 test/json/fixtures/minefield/n_number_2.e-3.json | 0 test/json/fixtures/minefield/n_number_2.e3.json | 0 test/json/fixtures/minefield/n_number_Inf.json | 0 test/json/fixtures/minefield/n_number_NaN.json | 0 test/json/fixtures/minefield/n_number_infinity.json | 0 .../fixtures/minefield/n_number_minus_infinity.json | 0 .../n_number_neg_real_without_int_part.json | 0 .../n_number_real_without_fractional_part.json | 0 .../minefield/n_number_starting_with_dot.json | 0 .../minefield/n_number_with_leading_zero.json | 0 .../minefield/n_object_key_with_single_quotes.json | 0 .../fixtures/minefield/n_object_missing_key.json | 0 .../minefield/n_object_missing_semicolon.json | 0 .../fixtures/minefield/n_object_non_string_key.json | 0 ...ject_non_string_key_but_huge_number_instead.json | 0 .../minefield/n_object_repeated_null_null.json | 0 .../minefield/n_object_several_trailing_commas.json | 0 .../fixtures/minefield/n_object_trailing_comma.json | 0 .../minefield/n_object_two_commas_in_a_row.json | 0 test/json/fixtures/minefield/n_single_space.json | 0 .../minefield/n_string_escaped_backslash_bad.json | 0 .../minefield/n_string_incomplete_escape.json | 0 .../n_string_incomplete_escaped_character.json | 0 .../minefield/n_string_incomplete_surrogate.json | 0 ..._string_incomplete_surrogate_escape_invalid.json | 0 .../minefield/n_string_invalid_backslash_esc.json | 0 .../n_string_leading_uescaped_thinspace.json | 0 .../minefield/n_string_single_doublequote.json | 0 .../n_string_single_string_no_double_quotes.json | 0 .../minefield/n_string_unescaped_ctrl_char.json | Bin .../minefield/n_structure_UTF8_BOM_no_data.json | 0 .../minefield/n_structure_angle_bracket_..json | 0 .../minefield/n_structure_angle_bracket_null.json | 0 .../n_structure_array_with_extra_array_close.json | 0 .../n_structure_array_with_unclosed_string.json | 0 .../minefield/n_structure_capitalized_True.json | 0 .../minefield/n_structure_close_unopened_array.json | 0 .../minefield/n_structure_double_array.json | 0 .../minefield/n_structure_incomplete_UTF8_BOM.json | 0 .../n_structure_open_object_close_array.json | 0 .../fixtures/minefield/n_structure_single_star.json | 0 .../n_structure_uescaped_LF_before_string.json | 0 .../minefield/n_structure_unclosed_array.json | 0 .../minefield/n_structure_unclosed_object.json | 0 .../n_structure_whitespace_U+2060_word_joiner.json | 0 .../minefield/n_structure_whitespace_formfeed.json | 0 .../minefield/y_array_arraysWithSpaces.json | 0 test/json/fixtures/minefield/y_array_empty.json | 0 .../minefield/y_array_ending_with_newline.json | 0 .../fixtures/minefield/y_array_heterogeneous.json | 0 .../minefield/y_array_with_leading_space.json | 0 .../minefield/y_array_with_several_null.json | 0 .../minefield/y_array_with_trailing_space.json | 0 test/json/fixtures/minefield/y_number_0e+1.json | 0 test/json/fixtures/minefield/y_number_0e1.json | 0 .../minefield/y_number_double_close_to_zero.json | 0 .../fixtures/minefield/y_number_int_with_exp.json | 0 .../fixtures/minefield/y_number_minus_zero.json | 0 test/json/fixtures/minefield/y_object.json | 0 test/json/fixtures/minefield/y_object_basic.json | 0 .../fixtures/minefield/y_object_duplicated_key.json | 0 .../y_object_duplicated_key_and_value.json | 0 .../json/fixtures/minefield/y_object_empty_key.json | 0 .../y_string_1_2_3_bytes_UTF-8_sequences.json | 0 .../minefield/y_string_accepted_surrogate_pair.json | 0 .../y_string_accepted_surrogate_pairs.json | 0 .../y_string_backslash_and_u_escaped_zero.json | 0 .../minefield/y_string_escaped_noncharacter.json | 0 test/json/fixtures/minefield/y_string_in_array.json | 0 .../y_string_in_array_with_leading_space.json | 0 .../y_string_nonCharacterInUTF-8_U+10FFFF.json | 0 .../y_string_nonCharacterInUTF-8_U+FFFF.json | 0 .../y_string_reservedCharacterInUTF-8_U+1BFFF.json | 0 ...ng_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json | 0 .../minefield/y_string_u+2028_line_sep.json | 0 .../fixtures/minefield/y_string_u+2029_par_sep.json | 0 test/json/fixtures/minefield/y_string_uEscape.json | 0 .../minefield/y_string_unescaped_char_delete.json | 0 .../minefield/y_string_unicodeEscapedBackslash.json | 0 .../y_string_unicode_escaped_double_quote.json | 0 .../minefield/y_string_with_del_character.json | 0 .../fixtures/minefield/y_structure_lonely_int.json | 0 .../minefield/y_structure_lonely_negative_real.json | 0 .../minefield/y_structure_lonely_string.json | 0 .../fixtures/minefield/y_structure_lonely_true.json | 0 122 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 test/json/fixtures/minefield/i_number_neg_int_huge_exp.json mode change 100755 => 100644 test/json/fixtures/minefield/i_number_pos_double_huge_exp.json mode change 100755 => 100644 test/json/fixtures/minefield/i_number_very_big_negative_int.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_invalid_surrogate.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json mode change 100755 => 100644 test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json mode change 100755 => 100644 test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_comma_and_number.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_double_comma.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_just_comma.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_just_minus.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_number_and_comma.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_number_and_several_commas.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json mode change 100755 => 100644 test/json/fixtures/minefield/n_array_star_inside.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_+1.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_+Inf.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_-01.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_-1.0..json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_-2..json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_-NaN.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_.2e-3.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_0.1.2.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_0_capital_E.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_1.0e+.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_1.0e-.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_1.0e.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_1_000.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_1eE2.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_2.e+3.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_2.e-3.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_2.e3.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_Inf.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_NaN.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_infinity.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_minus_infinity.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_neg_real_without_int_part.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_real_without_fractional_part.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_starting_with_dot.json mode change 100755 => 100644 test/json/fixtures/minefield/n_number_with_leading_zero.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_key_with_single_quotes.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_missing_key.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_missing_semicolon.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_non_string_key.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_repeated_null_null.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_several_trailing_commas.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_trailing_comma.json mode change 100755 => 100644 test/json/fixtures/minefield/n_object_two_commas_in_a_row.json mode change 100755 => 100644 test/json/fixtures/minefield/n_single_space.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_escaped_backslash_bad.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_incomplete_escape.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_incomplete_escaped_character.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_incomplete_surrogate.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_invalid_backslash_esc.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_single_doublequote.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json mode change 100755 => 100644 test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_angle_bracket_..json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_angle_bracket_null.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_capitalized_True.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_close_unopened_array.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_double_array.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_open_object_close_array.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_single_star.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_unclosed_array.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_unclosed_object.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json mode change 100755 => 100644 test/json/fixtures/minefield/n_structure_whitespace_formfeed.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_arraysWithSpaces.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_empty.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_ending_with_newline.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_heterogeneous.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_with_leading_space.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_with_several_null.json mode change 100755 => 100644 test/json/fixtures/minefield/y_array_with_trailing_space.json mode change 100755 => 100644 test/json/fixtures/minefield/y_number_0e+1.json mode change 100755 => 100644 test/json/fixtures/minefield/y_number_0e1.json mode change 100755 => 100644 test/json/fixtures/minefield/y_number_double_close_to_zero.json mode change 100755 => 100644 test/json/fixtures/minefield/y_number_int_with_exp.json mode change 100755 => 100644 test/json/fixtures/minefield/y_number_minus_zero.json mode change 100755 => 100644 test/json/fixtures/minefield/y_object.json mode change 100755 => 100644 test/json/fixtures/minefield/y_object_basic.json mode change 100755 => 100644 test/json/fixtures/minefield/y_object_duplicated_key.json mode change 100755 => 100644 test/json/fixtures/minefield/y_object_duplicated_key_and_value.json mode change 100755 => 100644 test/json/fixtures/minefield/y_object_empty_key.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_escaped_noncharacter.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_in_array.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_in_array_with_leading_space.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_u+2028_line_sep.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_u+2029_par_sep.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_uEscape.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_unescaped_char_delete.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json mode change 100755 => 100644 test/json/fixtures/minefield/y_string_with_del_character.json mode change 100755 => 100644 test/json/fixtures/minefield/y_structure_lonely_int.json mode change 100755 => 100644 test/json/fixtures/minefield/y_structure_lonely_negative_real.json mode change 100755 => 100644 test/json/fixtures/minefield/y_structure_lonely_string.json mode change 100755 => 100644 test/json/fixtures/minefield/y_structure_lonely_true.json diff --git a/test/json/fixtures/minefield/i_number_neg_int_huge_exp.json b/test/json/fixtures/minefield/i_number_neg_int_huge_exp.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_number_pos_double_huge_exp.json b/test/json/fixtures/minefield/i_number_pos_double_huge_exp.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_number_very_big_negative_int.json b/test/json/fixtures/minefield/i_number_very_big_negative_int.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json b/test/json/fixtures/minefield/i_string_UTF-8_invalid_sequence.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json b/test/json/fixtures/minefield/i_string_incomplete_surrogate_and_escape_valid.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json b/test/json/fixtures/minefield/i_string_incomplete_surrogate_pair.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json b/test/json/fixtures/minefield/i_string_incomplete_surrogates_escape_valid.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json b/test/json/fixtures/minefield/i_string_invalid_lonely_surrogate.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_invalid_surrogate.json b/test/json/fixtures/minefield/i_string_invalid_surrogate.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json b/test/json/fixtures/minefield/i_string_inverted_surrogates_U+1D11E.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json b/test/json/fixtures/minefield/i_string_overlong_sequence_6_bytes_null.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json b/test/json/fixtures/minefield/i_structure_UTF-8_BOM_empty_object.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_comma_and_number.json b/test/json/fixtures/minefield/n_array_comma_and_number.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_double_comma.json b/test/json/fixtures/minefield/n_array_double_comma.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json b/test/json/fixtures/minefield/n_array_items_separated_by_semicolon.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_just_comma.json b/test/json/fixtures/minefield/n_array_just_comma.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_just_minus.json b/test/json/fixtures/minefield/n_array_just_minus.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_number_and_comma.json b/test/json/fixtures/minefield/n_array_number_and_comma.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_number_and_several_commas.json b/test/json/fixtures/minefield/n_array_number_and_several_commas.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json b/test/json/fixtures/minefield/n_array_spaces_vertical_tab_formfeed.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_array_star_inside.json b/test/json/fixtures/minefield/n_array_star_inside.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_+1.json b/test/json/fixtures/minefield/n_number_+1.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_+Inf.json b/test/json/fixtures/minefield/n_number_+Inf.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_-01.json b/test/json/fixtures/minefield/n_number_-01.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_-1.0..json b/test/json/fixtures/minefield/n_number_-1.0..json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_-2..json b/test/json/fixtures/minefield/n_number_-2..json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_-NaN.json b/test/json/fixtures/minefield/n_number_-NaN.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_.2e-3.json b/test/json/fixtures/minefield/n_number_.2e-3.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_0.1.2.json b/test/json/fixtures/minefield/n_number_0.1.2.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_0_capital_E.json b/test/json/fixtures/minefield/n_number_0_capital_E.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_1.0e+.json b/test/json/fixtures/minefield/n_number_1.0e+.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_1.0e-.json b/test/json/fixtures/minefield/n_number_1.0e-.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_1.0e.json b/test/json/fixtures/minefield/n_number_1.0e.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_1_000.json b/test/json/fixtures/minefield/n_number_1_000.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_1eE2.json b/test/json/fixtures/minefield/n_number_1eE2.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_2.e+3.json b/test/json/fixtures/minefield/n_number_2.e+3.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_2.e-3.json b/test/json/fixtures/minefield/n_number_2.e-3.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_2.e3.json b/test/json/fixtures/minefield/n_number_2.e3.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_Inf.json b/test/json/fixtures/minefield/n_number_Inf.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_NaN.json b/test/json/fixtures/minefield/n_number_NaN.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_infinity.json b/test/json/fixtures/minefield/n_number_infinity.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_minus_infinity.json b/test/json/fixtures/minefield/n_number_minus_infinity.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_neg_real_without_int_part.json b/test/json/fixtures/minefield/n_number_neg_real_without_int_part.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_real_without_fractional_part.json b/test/json/fixtures/minefield/n_number_real_without_fractional_part.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_starting_with_dot.json b/test/json/fixtures/minefield/n_number_starting_with_dot.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_number_with_leading_zero.json b/test/json/fixtures/minefield/n_number_with_leading_zero.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_key_with_single_quotes.json b/test/json/fixtures/minefield/n_object_key_with_single_quotes.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_missing_key.json b/test/json/fixtures/minefield/n_object_missing_key.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_missing_semicolon.json b/test/json/fixtures/minefield/n_object_missing_semicolon.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_non_string_key.json b/test/json/fixtures/minefield/n_object_non_string_key.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json b/test/json/fixtures/minefield/n_object_non_string_key_but_huge_number_instead.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_repeated_null_null.json b/test/json/fixtures/minefield/n_object_repeated_null_null.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_several_trailing_commas.json b/test/json/fixtures/minefield/n_object_several_trailing_commas.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_trailing_comma.json b/test/json/fixtures/minefield/n_object_trailing_comma.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_object_two_commas_in_a_row.json b/test/json/fixtures/minefield/n_object_two_commas_in_a_row.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_single_space.json b/test/json/fixtures/minefield/n_single_space.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_escaped_backslash_bad.json b/test/json/fixtures/minefield/n_string_escaped_backslash_bad.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_incomplete_escape.json b/test/json/fixtures/minefield/n_string_incomplete_escape.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_incomplete_escaped_character.json b/test/json/fixtures/minefield/n_string_incomplete_escaped_character.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_incomplete_surrogate.json b/test/json/fixtures/minefield/n_string_incomplete_surrogate.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json b/test/json/fixtures/minefield/n_string_incomplete_surrogate_escape_invalid.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_invalid_backslash_esc.json b/test/json/fixtures/minefield/n_string_invalid_backslash_esc.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json b/test/json/fixtures/minefield/n_string_leading_uescaped_thinspace.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_single_doublequote.json b/test/json/fixtures/minefield/n_string_single_doublequote.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json b/test/json/fixtures/minefield/n_string_single_string_no_double_quotes.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json b/test/json/fixtures/minefield/n_string_unescaped_ctrl_char.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json b/test/json/fixtures/minefield/n_structure_UTF8_BOM_no_data.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_angle_bracket_..json b/test/json/fixtures/minefield/n_structure_angle_bracket_..json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_angle_bracket_null.json b/test/json/fixtures/minefield/n_structure_angle_bracket_null.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json b/test/json/fixtures/minefield/n_structure_array_with_extra_array_close.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json b/test/json/fixtures/minefield/n_structure_array_with_unclosed_string.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_capitalized_True.json b/test/json/fixtures/minefield/n_structure_capitalized_True.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_close_unopened_array.json b/test/json/fixtures/minefield/n_structure_close_unopened_array.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_double_array.json b/test/json/fixtures/minefield/n_structure_double_array.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json b/test/json/fixtures/minefield/n_structure_incomplete_UTF8_BOM.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_open_object_close_array.json b/test/json/fixtures/minefield/n_structure_open_object_close_array.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_single_star.json b/test/json/fixtures/minefield/n_structure_single_star.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json b/test/json/fixtures/minefield/n_structure_uescaped_LF_before_string.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_unclosed_array.json b/test/json/fixtures/minefield/n_structure_unclosed_array.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_unclosed_object.json b/test/json/fixtures/minefield/n_structure_unclosed_object.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json b/test/json/fixtures/minefield/n_structure_whitespace_U+2060_word_joiner.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/n_structure_whitespace_formfeed.json b/test/json/fixtures/minefield/n_structure_whitespace_formfeed.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_arraysWithSpaces.json b/test/json/fixtures/minefield/y_array_arraysWithSpaces.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_empty.json b/test/json/fixtures/minefield/y_array_empty.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_ending_with_newline.json b/test/json/fixtures/minefield/y_array_ending_with_newline.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_heterogeneous.json b/test/json/fixtures/minefield/y_array_heterogeneous.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_with_leading_space.json b/test/json/fixtures/minefield/y_array_with_leading_space.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_with_several_null.json b/test/json/fixtures/minefield/y_array_with_several_null.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_array_with_trailing_space.json b/test/json/fixtures/minefield/y_array_with_trailing_space.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_number_0e+1.json b/test/json/fixtures/minefield/y_number_0e+1.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_number_0e1.json b/test/json/fixtures/minefield/y_number_0e1.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_number_double_close_to_zero.json b/test/json/fixtures/minefield/y_number_double_close_to_zero.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_number_int_with_exp.json b/test/json/fixtures/minefield/y_number_int_with_exp.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_number_minus_zero.json b/test/json/fixtures/minefield/y_number_minus_zero.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_object.json b/test/json/fixtures/minefield/y_object.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_object_basic.json b/test/json/fixtures/minefield/y_object_basic.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_object_duplicated_key.json b/test/json/fixtures/minefield/y_object_duplicated_key.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_object_duplicated_key_and_value.json b/test/json/fixtures/minefield/y_object_duplicated_key_and_value.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_object_empty_key.json b/test/json/fixtures/minefield/y_object_empty_key.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json b/test/json/fixtures/minefield/y_string_1_2_3_bytes_UTF-8_sequences.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json b/test/json/fixtures/minefield/y_string_accepted_surrogate_pair.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json b/test/json/fixtures/minefield/y_string_accepted_surrogate_pairs.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json b/test/json/fixtures/minefield/y_string_backslash_and_u_escaped_zero.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_escaped_noncharacter.json b/test/json/fixtures/minefield/y_string_escaped_noncharacter.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_in_array.json b/test/json/fixtures/minefield/y_string_in_array.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_in_array_with_leading_space.json b/test/json/fixtures/minefield/y_string_in_array_with_leading_space.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+10FFFF.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json b/test/json/fixtures/minefield/y_string_nonCharacterInUTF-8_U+FFFF.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json b/test/json/fixtures/minefield/y_string_reservedCharacterInUTF-8_U+1BFFF.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json b/test/json/fixtures/minefield/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_u+2028_line_sep.json b/test/json/fixtures/minefield/y_string_u+2028_line_sep.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_u+2029_par_sep.json b/test/json/fixtures/minefield/y_string_u+2029_par_sep.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_uEscape.json b/test/json/fixtures/minefield/y_string_uEscape.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_unescaped_char_delete.json b/test/json/fixtures/minefield/y_string_unescaped_char_delete.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json b/test/json/fixtures/minefield/y_string_unicodeEscapedBackslash.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json b/test/json/fixtures/minefield/y_string_unicode_escaped_double_quote.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_string_with_del_character.json b/test/json/fixtures/minefield/y_string_with_del_character.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_structure_lonely_int.json b/test/json/fixtures/minefield/y_structure_lonely_int.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_structure_lonely_negative_real.json b/test/json/fixtures/minefield/y_structure_lonely_negative_real.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_structure_lonely_string.json b/test/json/fixtures/minefield/y_structure_lonely_string.json old mode 100755 new mode 100644 diff --git a/test/json/fixtures/minefield/y_structure_lonely_true.json b/test/json/fixtures/minefield/y_structure_lonely_true.json old mode 100755 new mode 100644 From 13162921b89c4f662aa018771403457f069f7e26 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 23 Jun 2026 12:45:00 +0200 Subject: [PATCH 395/506] Release 2.20.0 --- CHANGES.md | 6 +++++- lib/json/version.rb | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 21c8fb54e..aa9785f06 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,8 +2,12 @@ ### Unreleased -* Changed the C parser to no longer be recursive, so parsing very deep documents with `max_nesting: false` will no longer +### 2026-06-23 (2.20.0) + +* Both C and Java parsers are no longer recursive, so parsing very deep documents with `max_nesting: false` will no longer result in `SystemStackError stack level too deep` errors. + * The `:max_nesting` option still defaults to `100`. +* Optimized floating point number parsing further by replacing the ryu algorithm by a port of Eisel-Lemire Fast Float. * Added `JSON::ResumableParser` to parse streams of JSON documents. Not yet available on JRuby. * Deprecate default support of JavaScript comments in the parser and add `allow_comments: true` parsing option. * Integrate with Ruby 4.1 `ruby_sized_xfree`. diff --git a/lib/json/version.rb b/lib/json/version.rb index 8efca3d4a..76785011e 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.19.9' + VERSION = '2.20.0' end From e1ee4b0e223955f56429ea4390b1af6dcd8200e1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 25 Jun 2026 14:24:37 +0900 Subject: [PATCH 396/506] Fix wrong call-seq for `JSON::ResumableParser#eos?` --- ext/json/ext/parser/parser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 9a86cd58b..757c58701 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2712,7 +2712,7 @@ static VALUE cResumableParser_rest(VALUE self) } /* - * call-seq: value? -> true or false + * call-seq: eos? -> true or false * * Returns whether the internal buffer has been entirely consumed. */ From fa69e94b666bac2cc64110bba9d1eb1b24375418 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 24 Jun 2026 11:32:29 +0200 Subject: [PATCH 397/506] Declare types as RUBY_TYPED_THREAD_SAFE_FREE Ref: https://github.com/ruby/ruby/pull/17395 --- ext/json/ext/generator/generator.c | 2 +- ext/json/ext/json.h | 4 ++++ ext/json/ext/parser/parser.c | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 08c1fd0db..d4164051f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -739,7 +739,7 @@ static const rb_data_type_t JSON_Generator_State_type = { .dsize = State_memsize, .dcompact = State_compact, }, - .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static void state_init(JSON_Generator_State *state) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index f39e795a4..afddbe2a2 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -31,6 +31,10 @@ /* shims */ +#ifndef RUBY_TYPED_THREAD_SAFE_FREE +#define RUBY_TYPED_THREAD_SAFE_FREE RUBY_TYPED_FREE_IMMEDIATELY +#endif + #ifndef UNDEF_P #define UNDEF_P(val) (val == Qundef) #endif diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 757c58701..b297363f4 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -314,7 +314,7 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = { }, // We deliberately don't declare rvalue_stack as RUBY_TYPED_WB_PROTECTED // because it churns a lot of values so trigering write barriers every time is very costly. - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE, }; static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref) @@ -511,7 +511,7 @@ static const rb_data_type_t JSON_Parser_frame_stack_type = { .dfree = json_frame_stack_free, .dsize = json_frame_stack_memsize, }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, }; static json_frame_stack *json_frame_stack_spill(json_frame_stack *old_stack, VALUE *handle, json_frame_stack **stack_ref) @@ -2180,7 +2180,7 @@ static const rb_data_type_t JSON_ParserConfig_type = { .dsize = JSON_ParserConfig_memsize, .dcompact = JSON_ParserConfig_compact, }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FROZEN_SHAREABLE | RUBY_TYPED_EMBEDDABLE, }; static VALUE cJSON_parser_s_allocate(VALUE klass) @@ -2265,7 +2265,7 @@ static const rb_data_type_t JSON_ResumableParser_type = { // RUBY_TYPED_WB_PROTECTED is deliberately not declared because // this is a superset of JSON_Parser_rvalue_stack_type, so we'd need // to trigger a lot of write barriers. - .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE, + .flags = RUBY_TYPED_THREAD_SAFE_FREE | RUBY_TYPED_EMBEDDABLE, }; static VALUE cResumableParser_allocate(VALUE klass) From 168095ccb80b21c2a1db2c6d0b6bf5682ef64ad9 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 25 Jun 2026 17:25:02 +0900 Subject: [PATCH 398/506] Fix unused-but-set-global warnings in clang-23 --- ext/json/ext/parser/parser.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b297363f4..4d9fa25ba 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -5,7 +5,10 @@ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string; -static ID i_new, i_try_convert, i_uminus, i_encode, i_at_line, i_at_column; +static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column; +#ifndef HAVE_RB_STR_TO_INTERNED_STR +static ID i_uminus; +#endif static VALUE sym_max_nesting, sym_allow_nan, sym_allow_trailing_comma, sym_allow_comments, sym_allow_control_characters, sym_allow_invalid_escape, sym_symbolize_names, @@ -2812,7 +2815,9 @@ void Init_parser(void) i_new = rb_intern("new"); i_try_convert = rb_intern("try_convert"); +#ifndef HAVE_RB_STR_TO_INTERNED_STR i_uminus = rb_intern("-@"); +#endif i_encode = rb_intern("encode"); i_at_line = rb_intern("@line"); i_at_column = rb_intern("@column"); From 724eddaeaa8ff5f0704e994b2d268bb3a6da461f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 26 Jun 2026 02:30:59 +0000 Subject: [PATCH 399/506] Avoid Float out of range warning for clearly underflowing numbers When parsing JSON floats with extremely negative exponents (like 123.456e-789 or 123e-10000000), the parser would fall back to rb_cstr_to_dbl which internally calls strtod. When strtod returns ERANGE due to underflow to 0.0, Ruby emits a "Float out of range" warning, causing noise in the test output. Fix: when mantissa_digits + exponent < -324, the effective value is less than 10^(-324) < DBL_TRUE_MIN/2, so it must round to 0.0 in IEEE 754 round-to-nearest. Return 0.0 directly without going through rb_cstr_to_dbl, avoiding the spurious warning. This fixes warnings introduced by JSONMinefieldParserTest tests (test_i_number_double_huge_neg_exp and test_i_number_real_underflow) added in commit 6507a836c5. --- ext/json/ext/parser/parser.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 4d9fa25ba..a6154c83e 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1133,6 +1133,13 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis } if (RB_UNLIKELY(mantissa_digits > 18 || mantissa_digits + exponent < -307)) { + // If the value is so small that it definitely underflows to 0.0, return early + // to avoid triggering a "Float out of range" warning from rb_cstr_to_dbl. + // When mantissa_digits + exponent < -324, value < 10^(-324) < DBL_TRUE_MIN/2, + // so it rounds to 0 in IEEE 754 round-to-nearest. + if (RB_UNLIKELY(mantissa_digits + exponent < -324)) { + return rb_float_new(negative ? -0.0 : 0.0); + } return json_decode_large_float(start, end - start); } From faa5572b6c8b6d266cbafafbb568f658bbfe0d16 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Fri, 26 Jun 2026 16:33:50 +0300 Subject: [PATCH 400/506] Fix a few inconsistencies in readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cb57ab693..5e6ffe48f 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Both of these behavior can be disabled using the `strict: true` option: ```ruby JSON.generate(Object.new, strict: true) # => Object not allowed in JSON (JSON::GeneratorError) -JSON.generate(Position.new(1, 2)) # => Position not allowed in JSON (JSON::GeneratorError) +JSON.generate(Position.new(1, 2), strict: true) # => Position not allowed in JSON (JSON::GeneratorError) ``` ## JSON::Coder @@ -117,13 +117,13 @@ It is also called for objects that do have a JSON equivalent, but are used as Ha as well as for strings that aren't valid UTF-8: ```ruby -coder = JSON::Combining.new do |object, is_object_key| +coder = JSON::Coder.new do |object, is_object_key| case object when String - if !string.valid_encoding? || string.encoding != Encoding::UTF_8 - Base64.encode64(string) + if !object.valid_encoding? || object.encoding != Encoding::UTF_8 + Base64.encode64(object) else - string + object end else object From 94c1af25b568dd97abe3fcc8a61662019aef3b7c Mon Sep 17 00:00:00 2001 From: tompng Date: Mon, 29 Jun 2026 17:23:57 +0900 Subject: [PATCH 401/506] Fix ResumableParser#partial_value SEGV on incomplete nested containers partial_value rebuilds the open containers on a scratch copy of the value stack. Folding an empty innermost container pops nothing and pushes its result, so head climbs one past the live size; the scratch buffer was sized to exactly the live size, so the push triggered rvalue_stack_grow which reallocated the ALLOCV buffer (a non-malloc pointer) and crashed. A parent fold always reclaims its child's slot, so head exceeds the live size by at most one -- either for the missing-value placeholder or for an empty innermost fold, never both. Size the buffer to capa + 1. Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/json/ext/parser/parser.c | 13 +++++++++---- test/json/resumable_parser_test.rb | 11 +++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a6154c83e..a60f6a3c9 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2625,11 +2625,16 @@ static VALUE cResumableParser_partial_value_body(VALUE self) missing_object_value = 1; } - // Copy the value stack as we need to mutate it. + // Copy the value stack as we need to mutate it. The collapse loop folds each + // open container by popping its entries and pushing the single result, so a + // parent always reclaims its child's slot; head exceeds its live size by at + // most one, either for the missing-value placeholder pushed below or for the + // result of folding an empty innermost container. That one spare slot keeps + // rvalue_stack_push from growing (reallocating) this ALLOCV buffer. long capa = parser.value_stack.head; - parser.value_stack.capa = (capa + missing_object_value); - VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, capa + missing_object_value); - MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, parser.value_stack.capa); + parser.value_stack.capa = capa + 1; + VALUE tmpbuf, *value_stack_buffer = ALLOCV_N(VALUE, tmpbuf, parser.value_stack.capa); + MEMCPY(value_stack_buffer, parser.value_stack.ptr, VALUE, capa); parser.value_stack.ptr = value_stack_buffer; JSON_ParserState *state = &parser.state; diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 4cc6c085d..734d6e220 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -247,6 +247,17 @@ def test_partial_value assert_partial_value([1, { "a" => 1, "b" => { "c" => nil } }], '[1, { "a": 1, "b": { "c"') end + def test_partial_value_collapses_nested_incomplete_containers + # partial_value rebuilds the open containers on a scratch value stack; folding + # an empty inner container pushes a value, so that stack must hold more than its + # live size or the push reallocates the scratch buffer. + assert_partial_value({ "abc" => {} }, '{"abc":{"d') + assert_partial_value({ "a" => { "b" => { "c" => {} } } }, '{"a":{"b":{"c":{"e') + assert_partial_value([1, { "a" => {} }], '[1,{"a":{"d') + assert_partial_value({ "a" => [1, { "b" => [2, { "c" => nil }] }] }, '{"a":[1,{"b":[2,{"c"') + assert_partial_value([1, [2, [3, { "x" => nil }]]], '[1,[2,[3,{"x":[') + end + def test_partial_value_issue_1005 data = <<~JSON [ From ea008e82fed43316fb0672bf84774818e788f022 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Wed, 24 Jun 2026 21:29:24 -0500 Subject: [PATCH 402/506] Add a `sort_keys` option to the generator. Fix: https://github.com/ruby/json/issues/976 Co-Authored-By: Jean Boussier --- ext/json/ext/generator/generator.c | 73 ++++++++++++++++++++++++++- java/src/json/ext/Generator.java | 6 +++ java/src/json/ext/GeneratorState.java | 49 ++++++++++++++++++ lib/json.rb | 6 ++- lib/json/common.rb | 9 ++++ lib/json/ext/generator/state.rb | 1 + lib/json/truffle_ruby/generator.rb | 37 +++++++++++++- test/json/json_generator_test.rb | 55 ++++++++++++++++++++ 8 files changed, 232 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d4164051f..6f473325e 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -34,13 +34,14 @@ typedef struct JSON_Generator_StateStruct { bool ascii_only; bool script_safe; bool strict; + VALUE sort_keys; } JSON_Generator_State; -static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8; +static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc; static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, - sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json; + sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json, sym_sort_keys; #define GET_STATE_TO(self, state) \ @@ -709,6 +710,7 @@ static void State_mark(void *ptr) rb_gc_mark_movable(state->object_nl); rb_gc_mark_movable(state->array_nl); rb_gc_mark_movable(state->as_json); + rb_gc_mark_movable(state->sort_keys); } static void State_compact(void *ptr) @@ -720,6 +722,7 @@ static void State_compact(void *ptr) state->object_nl = rb_gc_location(state->object_nl); state->array_nl = rb_gc_location(state->array_nl); state->as_json = rb_gc_location(state->as_json); + state->sort_keys = rb_gc_location(state->sort_keys); } static size_t State_memsize(const void *ptr) @@ -769,6 +772,7 @@ static void vstate_spill(struct generate_json_data *data) RB_OBJ_WRITTEN(vstate, Qundef, state->object_nl); RB_OBJ_WRITTEN(vstate, Qundef, state->array_nl); RB_OBJ_WRITTEN(vstate, Qundef, state->as_json); + RB_OBJ_WRITTEN(vstate, Qundef, state->sort_keys); } static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj) @@ -1050,6 +1054,11 @@ static inline long increase_depth(struct generate_json_data *data) static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { + if (RB_UNLIKELY(data->state->sort_keys)) { + obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil); + Check_Type(obj, T_HASH); + } + long depth = increase_depth(data); if (RHASH_SIZE(obj) == 0) { @@ -1376,6 +1385,7 @@ static VALUE cState_init_copy(VALUE obj, VALUE orig) RB_OBJ_WRITTEN(obj, Qundef, objState->object_nl); RB_OBJ_WRITTEN(obj, Qundef, objState->array_nl); RB_OBJ_WRITTEN(obj, Qundef, objState->as_json); + RB_OBJ_WRITTEN(obj, Qundef, objState->sort_keys); return obj; } @@ -1722,6 +1732,55 @@ static VALUE cState_ascii_only_set(VALUE self, VALUE enable) return Qnil; } +static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc) +{ + if (!rb_obj_is_proc(proc)) { + rb_raise(rb_eTypeError, "sort_key_proc must be a Proc"); + } + return default_sort_keys_proc = proc; +} + +static VALUE normalize_sort_keys(VALUE value) +{ + if (rb_obj_is_proc(value)) { + return value; + } else if (value == Qtrue) { + return default_sort_keys_proc; + } else if (RTEST(value)) { + rb_raise(rb_eTypeError, "The `sort_keys` argument must be a boolean or a Proc"); + } else { + return Qfalse; + } +} + +/* + * call-seq: sort_keys + * + * Get the value of sort_keys. + */ +static VALUE cState_sort_keys_p(VALUE self) +{ + GET_STATE(self); + return state->sort_keys; +} + +/* + * call-seq: sort_keys=(value) + * + * value is a boolean or a proc. If the value is the boolean true, object keys + * will be sorted lexicographically in ascending order. + * + * If the value is a proc, it receives the entire Hash and must return a Hash + * with its pairs in the desired order, allowing for arbitrary sorting. + */ +static VALUE cState_sort_keys_set(VALUE self, VALUE value) +{ + rb_check_frozen(self); + GET_STATE(self); + RB_OBJ_WRITE(self, &state->sort_keys, normalize_sort_keys(value)); + return Qnil; +} + static VALUE cState_allow_duplicate_key_p(VALUE self) { GET_STATE(self); @@ -1832,6 +1891,9 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; state_write_value(data, &state->as_json, proc); } + else if (key == sym_sort_keys) { + state_write_value(data, &state->sort_keys, normalize_sort_keys(val)); + } return ST_CONTINUE; } @@ -1909,6 +1971,8 @@ void Init_generator(void) VALUE mExt = rb_define_module_under(mJSON, "Ext"); VALUE mGenerator = rb_define_module_under(mExt, "Generator"); + rb_global_variable(&default_sort_keys_proc); + rb_global_variable(&eGeneratorError); eGeneratorError = rb_path2class("JSON::GeneratorError"); @@ -1918,6 +1982,8 @@ void Init_generator(void) cState = rb_define_class_under(mGenerator, "State", rb_cObject); rb_define_alloc_func(cState, cState_s_allocate); rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1); + rb_define_singleton_method(cState, "default_sort_keys_proc=", cState_set_default_sort_keys_proc, 1); + rb_define_method(cState, "initialize", cState_initialize, -1); rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings rb_define_private_method(cState, "_configure", cState_configure, 1); @@ -1957,6 +2023,8 @@ void Init_generator(void) rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1); rb_define_method(cState, "generate", cState_generate, -1); rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1); + rb_define_method(cState, "sort_keys", cState_sort_keys_p, 0); + rb_define_method(cState, "sort_keys=", cState_sort_keys_set, 1); rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); @@ -1986,6 +2054,7 @@ void Init_generator(void) sym_strict = ID2SYM(rb_intern("strict")); sym_as_json = ID2SYM(rb_intern("as_json")); sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); + sym_sort_keys = ID2SYM(rb_intern("sort_keys")); usascii_encindex = rb_usascii_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index ea9e6d083..f1617a051 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -17,6 +17,7 @@ import org.jruby.RubyFloat; import org.jruby.RubyHash; import org.jruby.RubyIO; +import org.jruby.RubyProc; import org.jruby.RubyString; import org.jruby.RubySymbol; import org.jruby.runtime.Helpers; @@ -573,6 +574,11 @@ static void generateHash(ThreadContext context, Session session, RubyHash object return; } + RubyProc sortKeysProc = state.getSortKeysProc(); + if (sortKeysProc != null) { + object = (RubyHash) Helpers.invoke(context, sortKeysProc, "call", object); + } + final ByteList objectNl = state.getObjectNl(); byte[] objectNLBytes = objectNl.unsafeBytes(); final byte[] indent = Utils.repeat(state.getIndent(), depth); diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 11d55559c..2d20ea461 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -37,6 +37,8 @@ public class GeneratorState extends RubyObject { private boolean allowDuplicateKey = false; private boolean deprecateDuplicateKey = true; + private static IRubyObject defaultSortKeyProc; + /** * The indenting unit string. Will be repeated several times for larger * indenting levels. @@ -104,6 +106,13 @@ public class GeneratorState extends RubyObject { private int bufferInitialLength = DEFAULT_BUFFER_INITIAL_LENGTH; static final int DEFAULT_BUFFER_INITIAL_LENGTH = 1024; + /** + * Controls key sorting when generating JSON. null means keys + * are emitted in insertion order; a true value sorts keys lexicographically; + * a {@link RubyProc} is used as a comparator receiving two [key, value] pairs. + */ + private IRubyObject sortKeys; + /** * The current depth (inside a #to_json call) */ @@ -158,6 +167,12 @@ static GeneratorState fromState(ThreadContext context, RuntimeInfo info, return (GeneratorState)klass.newInstance(context, context.nil); } + @JRubyMethod(meta=true, name="default_sort_keys_proc=") + public static IRubyObject setDefaultSortKeyProc(IRubyObject klass, IRubyObject proc) { + defaultSortKeyProc = proc; + return proc; + } + /** * State#initialize(opts = {}) *

@@ -222,6 +237,7 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { this.allowDuplicateKey = orig.allowDuplicateKey; this.deprecateDuplicateKey = orig.deprecateDuplicateKey; + this.sortKeys = orig.sortKeys; return this; } @@ -431,6 +447,23 @@ public boolean strict() { return strict; } + /** + * Returns the proc used to sort the keys of an object, or + * null if keys should not be sorted. The proc receives the + * entire Hash and returns a Hash with its pairs in the desired order. + */ + public RubyProc getSortKeysProc() { + return sortKeys instanceof RubyProc ? (RubyProc) sortKeys : null; + } + + private static IRubyObject normalizeSortKeys(ThreadContext context, IRubyObject value) { + if (value instanceof RubyProc) return value; + if (value != null && value.isTrue()) { + return defaultSortKeyProc; + } + return null; + } + @JRubyMethod(name={"strict","strict?"}) public RubyBoolean strict_get(ThreadContext context) { return RubyBoolean.newBoolean(context, strict); @@ -474,6 +507,18 @@ public IRubyObject buffer_initial_length_set(IRubyObject buffer_initial_length) return buffer_initial_length; } + @JRubyMethod(name="sort_keys") + public IRubyObject sort_keys_get(ThreadContext context) { + return sortKeys == null ? context.getRuntime().getFalse() : sortKeys; + } + + @JRubyMethod(name="sort_keys=") + public IRubyObject sort_keys_set(ThreadContext context, IRubyObject sortKeys) { + checkFrozen(); + this.sortKeys = normalizeSortKeys(context, sortKeys); + return sortKeys; + } + public int getDepth() { return depth; } @@ -568,6 +613,9 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); this.deprecateDuplicateKey = false; } + + sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); + return this; } @@ -596,6 +644,7 @@ public RubyHash to_h(ThreadContext context) { result.op_aset(context, runtime.newSymbol("strict"), strict_get(context)); result.op_aset(context, runtime.newSymbol("depth"), depth_get(context)); result.op_aset(context, runtime.newSymbol("buffer_initial_length"), buffer_initial_length_get(context)); + result.op_aset(context, runtime.newSymbol("sort_keys"), sort_keys_get(context)); if (this.allowDuplicateKey) { if (!this.deprecateDuplicateKey) { diff --git a/lib/json.rb b/lib/json.rb index f8dc4ccc9..d1e94147d 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -408,7 +408,6 @@ # to be inserted after each \JSON object; defaults to the empty \String, ''. # - Option +indent+ (\String) specifies the string (usually spaces) to be # used for indentation; defaults to the empty \String, ''; -# defaults to the empty \String, ''; # has no effect unless options +array_nl+ or +object_nl+ specify newlines. # - Option +space+ (\String) specifies a string (usually a space) to be # inserted after the colon in each \JSON object's pair; @@ -416,6 +415,11 @@ # - Option +space_before+ (\String) specifies a string (usually a space) to be # inserted before the colon in each \JSON object's pair; # defaults to the empty \String, ''. +# - Option +sort_keys+ (boolean or \Proc) controls whether and how the keys of a +# hash are sorted when generating the output; defaults to false. +# When +true+, keys are sorted lexicographically. When a \Proc, it receives +# the entire \Hash and must return a \Hash with its pairs in the desired +# order, allowing for arbitrary sort orders. # # In this example, +obj+ is used first to generate the shortest # \JSON data (no whitespace), then again with all formatting options diff --git a/lib/json/common.rb b/lib/json/common.rb index 230bf0801..fdcb860db 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -155,6 +155,15 @@ def parser=(parser) # :nodoc: # Set the module _generator_ to be used by JSON. def generator=(generator) # :nodoc: old, $VERBOSE = $VERBOSE, nil + + # The default proc used when the +sort_keys+ generation option is +true+. + # It returns a new hash with the entries sorted by their keys. + sort_keys_proc = ->(hash) { hash.sort.to_h } + if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda) + sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc) + end + generator::State.default_sort_keys_proc = sort_keys_proc + @generator = generator if generator.const_defined?(:GeneratorMethods) generator_methods = generator::GeneratorMethods diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index e4f425af6..3c1d2fb94 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -54,6 +54,7 @@ def to_h strict: strict?, depth: depth, buffer_initial_length: buffer_initial_length, + sort_keys: sort_keys } allow_duplicate_key = allow_duplicate_key? diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 4fb1885e0..ee6186fbe 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -111,6 +111,8 @@ def self.valid_utf8?(string) # This class is used to create State instances, that are use to hold data # while generating a JSON text from a Ruby data structure. class State + singleton_class.attr_accessor :default_sort_keys_proc # :nodoc: + def self.generate(obj, opts = nil, io = nil) new(opts).generate(obj, io) end @@ -164,6 +166,7 @@ def initialize(opts = nil) @script_safe = false @strict = false @max_nesting = 100 + @sort_keys = false configure(opts) if opts end @@ -199,6 +202,33 @@ def initialize(opts = nil) # supported by the JSON spec will raise a JSON::GeneratorError attr_accessor :strict + # Controls key sorting in the generated JSON. If set to +true+, object + # keys are sorted by key lexicographically. If set to a Proc, it + # receives the entire Hash and must return a Hash with its pairs in the + # desired order. + attr_reader :sort_keys + + def sort_keys=(value) # :nodoc: + type_error = false + @sort_keys = case value + when Proc + value + when true + State.default_sort_keys_proc + when nil, false + false + else + type_error = true + false + end + + if type_error + raise TypeError, "The `sort_keys` argument must be a boolean or a Proc" + end + + @sort_keys + end + # :stopdoc: attr_reader :buffer_initial_length @@ -285,6 +315,7 @@ def configure(opts) @allow_nan = !!opts[:allow_nan] if opts.key?(:allow_nan) @as_json = opts[:as_json].to_proc if opts[:as_json] @ascii_only = opts[:ascii_only] if opts.key?(:ascii_only) + self.sort_keys = opts[:sort_keys] if opts.key?(:sort_keys) @depth = opts[:depth] || 0 @buffer_initial_length ||= opts[:buffer_initial_length] @@ -349,9 +380,13 @@ def generate(obj, anIO = nil) depth = @depth if @indent.empty? and @space.empty? and @space_before.empty? and @object_nl.empty? and @array_nl.empty? and - !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) + !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) and !@sort_keys result = generate_json(obj, ''.dup) else + if @sort_keys + obj = @sort_keys.call(obj) + end + result = obj.to_json(self) end JSON::TruffleRuby::Generator.valid_utf8?(result) or raise GeneratorError.new( diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 753ee0fbd..ab19704b1 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -190,6 +190,59 @@ def test_generate_pretty_custom JSON end + def test_generate_sort_keys + json = generate({2=>"a", 1=>"b", 3=>"c"}, sort_keys: true) + assert_equal('{"1":"b","2":"a","3":"c"}', json) + + json = generate({2=>"a", 1=>"b", 3=>"c"}, sort_keys: false) + assert_equal('{"2":"a","1":"b","3":"c"}', json) + + json = pretty_generate({2=>"a", 1=>"b", 3=>"c"}, sort_keys: true) + assert_equal(<<~'JSON'.chomp, json) + { + "1": "b", + "2": "a", + "3": "c" + } + JSON + + json = pretty_generate({2=>"a", 1=>"b", 3=>"c"}, sort_keys: false) + assert_equal(<<~'JSON'.chomp, json) + { + "2": "a", + "1": "b", + "3": "c" + } + JSON + + json = pretty_generate({2=>"a", 1=>"b", 3=>"c"}) + assert_equal(<<~'JSON'.chomp, json) + { + "2": "a", + "1": "b", + "3": "c" + } + JSON + end + + def test_generate_sort_keys_with_proc + reverse = ->(hash) { hash.sort.reverse.to_h } + json = generate({2=>"a", 1=>"b", 3=>"c"}, sort_keys: reverse) + assert_equal('{"3":"c","2":"a","1":"b"}', json) + + by_value = ->(hash) { hash.sort_by { |_k, v| v }.to_h } + json = generate({2=>"c", 1=>"a", 3=>"b"}, sort_keys: by_value) + assert_equal('{"1":"a","3":"b","2":"c"}', json) + + state = State.new(sort_keys: reverse) + assert_same reverse, state.to_h[:sort_keys] + assert_equal('{"3":"c","2":"a","1":"b"}', state.generate({2=>"a", 1=>"b", 3=>"c"})) + + # A truthy sort_keys is normalized to the default sorting proc. + state = State.new(sort_keys: true) + assert_instance_of Proc, state.sort_keys + end + def test_generate_custom state = State.new(:space_before => " ", :space => " ", :indent => "", :object_nl => "\n", :array_nl => "") json = generate({1=>{2=>3,4=>[5,6]}}, state) @@ -289,6 +342,7 @@ def test_state_defaults :object_nl => "", :space => "", :space_before => "", + :sort_keys => false, }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) state = JSON::State.new(allow_duplicate_key: true) @@ -307,6 +361,7 @@ def test_state_defaults :object_nl => "", :space => "", :space_before => "", + :sort_keys => false, }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end From 9dbfeb83dfcdbd64af7c87c3511a3ff96b352a0b Mon Sep 17 00:00:00 2001 From: Masataka Pocke Kuwabara Date: Thu, 2 Jul 2026 18:52:53 +0900 Subject: [PATCH 403/506] Fix ResumableParser treating unterminated line comments as complete A `//` line comment split across a feed boundary was treated as fully consumed, so comment body delivered in a later chunk leaked out as parsed JSON values. ## Reproduction ```ruby require "json" parser = JSON::ResumableParser.new(allow_comments: true) documents = [] "[1]//[999]\n[3]".each_char do |char| parser << char documents << parser.value while parser.parse end p documents ``` ## Expected Behavior ``` [[1], [3]] ``` The `[999]` is inside the `//` comment (which runs until the newline), so it is ignored, leaving the two documents `[1]` and `[3]`. This is what feeding the whole string in a single chunk produces. ## Actual Behavior ``` [[1], [999], [3]] ``` The commented-out `[999]` leaked out as a real value. When the comment body does not form valid JSON the split feed raises a ParserError instead. This happens with the default configuration too, since comments are accepted (with a deprecation warning) unless disabled. ## Description A `//` line comment is only terminated by a newline. `json_eat_comments` searched for that newline and, when none was present in the buffer, moved the cursor to the end of the buffer and returned as if the comment had been fully consumed. For the one-shot `JSON.parse` this is correct: the buffer holds the whole document, so a `//` comment with no trailing newline genuinely runs to the end of input. For `ResumableParser`, however, the end of the buffer is only a chunk boundary, not the end of input. Reaching it does not mean the comment is over -- the newline (and possibly more comment body) may still arrive in a later `<<`. Swallowing the buffered bytes as a finished comment loses the "still inside a comment" state, so any comment body delivered in the next chunk was parsed as JSON instead of being ignored. Because the result then depended on where the input happened to be split, the same byte stream could produce different values. The block-comment branch already handles this correctly: when it cannot find the closing `*/` it rewinds to the comment start and raises an EOS-tagged error, which `ResumableParser#parse` swallows into a `false` return so the comment is retried once more input is available. This change makes the line-comment branch behave the same way, but only in resumable mode (detected via `state->parser`, the same discriminator `raise_parse_error` already uses). In non-resumable mode the previous behaviour is preserved: a `//` comment with no trailing newline is still consumed to the end of input. As a consequence, a stream that ends with an unterminated line comment now stays incomplete (`parse` keeps returning `false`) until a newline arrives, which mirrors how a trailing bare number is only considered complete once a following separator is seen. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PpeQSEFkF1X1Uuzjx9BsYe --- ext/json/ext/parser/parser.c | 13 ++++++--- test/json/json_parser_test.rb | 1 + test/json/resumable_parser_test.rb | 42 ++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a60f6a3c9..a4c7dc163 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -779,11 +779,18 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) switch (peek(state)) { case '/': { - state->cursor = memchr(state->cursor, '\n', state->end - state->cursor); - if (!state->cursor) { + const char *newline = memchr(state->cursor, '\n', state->end - state->cursor); + if (!newline) { + // state->parser marks resumable mode, where the buffer end is only a + // chunk boundary: the terminating newline may still arrive, so leave + // the comment unterminated instead of consuming to end as a one-shot + // parse would. + if (state->parser) { + raise_eos_error_at("unterminated comment, expected end of line", state, start); + } state->cursor = state->end; } else { - state->cursor++; + state->cursor = newline + 1; } break; } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 9000d1152..2f79b87cc 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -526,6 +526,7 @@ def test_parse_comments JSON assert_equal({ "key1" => "value1" }, parse(json, allow_comments: true)) assert_equal({}, parse('{} /**/', allow_comments: true)) + assert_equal({}, parse('{} // eol comment ending at eof', allow_comments: true)) assert_raise(ParserError) { parse('{} /* comment not closed', allow_comments: true) } assert_raise(ParserError) { parse('{} /*/', allow_comments: true) } assert_raise(ParserError) { parse('{} /x wrong comment', allow_comments: true) } diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 734d6e220..ee8e2f8d3 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -199,6 +199,48 @@ def test_incomplete_input_at_structural_positions_resumes assert_incomplete "{\"a\":1," end + def test_line_comment_spanning_feed_boundary_is_not_terminated_early + # A `//` line comment is only terminated by a newline. When the newline + # has not arrived yet, the comment must stay incomplete rather than being + # treated as consumed -- otherwise its body, delivered in a later chunk, + # leaks out as parsed values. + values = [] + parser = new_parser(allow_comments: true) + parser << '[1] //' + values << parser.value while parser.parse + + parser << "[2]\n[3]" # [2] belongs to the comment, [3] is a real document + values << parser.value while parser.parse + + assert_equal [[1], [3]], values + end + + def test_line_comment_terminated_by_newline_across_feeds + values = [] + parser = new_parser(allow_comments: true) + parser << '[1] //co' + values << parser.value while parser.parse + + parser << "mment\n[2]" + values << parser.value while parser.parse + + assert_equal [[1], [2]], values + end + + def test_block_comment_spanning_feed_boundary_is_not_terminated_early + # A `/* */` block comment whose closing `*/` has not arrived yet must stay + # incomplete, mirroring the line-comment behaviour above. + values = [] + parser = new_parser(allow_comments: true) + parser << '[1] /*' + values << parser.value while parser.parse + + parser << '[2]*/[3]' # [2] belongs to the comment, [3] is a real document + values << parser.value while parser.parse + + assert_equal [[1], [3]], values + end + def test_rest @parser << '[1, 2, 3, "unterminated string' refute @parser.parse From 0864e83701500ded4db0bece0dd3c78b5f819656 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Fri, 3 Jul 2026 16:33:08 +0900 Subject: [PATCH 404/506] Add JSON::ResumableParser#partial_value? and #empty? There was no single API answering "does the stream end in the middle of a document?" once all parseable fed bytes have been consumed. Callers had to combine two complementary APIs: !parser.rest.empty? || !parser.partial_value.nil? `rest` only reflects unconsumed tokenizer bytes, so it is empty when the stream is truncated exactly on a token boundary (right after a ':' or ','), while `partial_value` is nil when truncation happens mid-token before any container is registered. Neither alone covers all shapes, and `partial_value` materializes the partially built Ruby objects just to test for nil. `partial_value?` answers the same question as `!partial_value.nil?` by looking at the parser's internal value stack directly, without building the partial Ruby object graph. `empty?` is strict: true only when the buffer is fully consumed, no document is under construction and no parsed value awaits retrieval with `value`. It is defined in Ruby as the composition of the three underlying predicates so its definition doubles as documentation: def empty? eos? && !partial_value? && !value? end Co-Authored-By: Claude Fable 5 --- ext/json/ext/parser/parser.c | 36 +++++++++++++ lib/json/ext.rb | 26 ++++++++++ test/json/resumable_parser_test.rb | 83 ++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a4c7dc163..58dd28188 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2744,6 +2744,41 @@ static VALUE cResumableParser_eos_p(VALUE self) return eos(&parser->state) ? Qtrue : Qfalse; } +/* + * call-seq: partial_value? -> true or false + * + * Returns whether a document is currently under construction: an unclosed + * container, a key awaiting its value, etc. + * + * It answers the same question as !partial_value.nil?, but as a + * cheap predicate on the parser's internal state, without materializing the + * partially parsed Ruby objects: + * parser << '{"a":1,' + * parser.parse # => false + * parser.partial_value? # => true + * + * A fully parsed document whose value hasn't been retrieved yet is not under + * construction: #value? returns true and #partial_value? returns false. + */ +static VALUE cResumableParser_partial_value_p(VALUE self) +{ + JSON_ResumableParser *parser = cResumableParser_get(self); + + // Mirror of #value?: values on the stack while the document isn't DONE + // belong to a partially built document. A container whose first key or + // element hasn't been parsed yet has no frame nor value registered (the + // tokenizer rewinds to the container start on EOS), so that state is + // observable through the buffer (#eos?/#rest) instead, keeping this + // predicate consistent with #partial_value returning nil. + if (parser->value_stack.head > 0) { + json_frame *frame = json_frame_stack_peek(&parser->frames); + if (frame->phase != JSON_PHASE_DONE) { + return Qtrue; + } + } + return Qfalse; +} + /* * call-seq: parsed_bytes -> integer * @@ -2800,6 +2835,7 @@ void Init_parser(void) rb_define_method(cResumableParser, "value", cResumableParser_value, 0); rb_define_method(cResumableParser, "value?", cResumableParser_value_p, 0); rb_define_method(cResumableParser, "partial_value", cResumableParser_partial_value, 0); + rb_define_method(cResumableParser, "partial_value?", cResumableParser_partial_value_p, 0); rb_define_method(cResumableParser, "clear", cResumableParser_clear, 0); rb_define_method(cResumableParser, "rest", cResumableParser_rest, 0); rb_define_method(cResumableParser, "eos?", cResumableParser_eos_p, 0); diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 5bacc5e37..28500f149 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -41,5 +41,31 @@ def parse end end + if defined?(ResumableParser) # Not yet available on JRuby + class ResumableParser + # Returns whether the parser is entirely done: no unconsumed bytes in + # the buffer, no document under construction and no parsed value + # awaiting retrieval. + # + # The main use case is detecting a truncated stream once the input is + # exhausted: + # + # loop do + # begin + # parser << socket.readpartial(4096) + # rescue EOFError + # break + # end + # while parser.parse + # process(parser.value) + # end + # end + # warn "stream was truncated" unless parser.empty? + def empty? + eos? && !partial_value? && !value? + end + end + end + JSON_LOADED = true unless defined?(JSON::JSON_LOADED) end diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index ee8e2f8d3..357052c9a 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -281,6 +281,89 @@ def test_eos assert_predicate @parser, :eos? end + def test_empty_predicate + # empty? is defined on the state left after parsing everything that + # could be parsed from the fed bytes, so drain with parse/value first. + { + '' => true, # nothing fed: vacuously empty + '{"a":1}' => true, + '{"a":1}{"b":2}' => true, + '{"a":1} ' => true, # trailing whitespace + '{"a":1}{"b":2' => false, # inside a number token + '{"a":1}{"b":' => false, # right after a colon (token boundary) + '{"a":1}{' => false, # right after an object open + '{"a":1,' => false, # right after a comma (token boundary) + '"abc' => false, # inside a string token + '[1,2' => false, # unclosed array + }.each do |json, expected| + parser = new_parser + parser << json + parser.value while parser.parse + assert_equal expected, parser.empty?, "expected #{json.inspect} to be empty? == #{expected}" + end + end + + def test_empty_predicate_with_undrained_buffer + @parser << '{"a":1}{"b":2}' + assert @parser.parse + refute_predicate @parser, :empty? # second document still in the buffer + assert_equal({ "a" => 1 }, @parser.value) + assert @parser.parse + assert_equal({ "b" => 2 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_empty_predicate_with_pending_value + # A fully parsed document awaiting retrieval with #value is not empty. + @parser << '{"a":1}' + assert @parser.parse + refute_predicate @parser, :empty? + assert_equal({ "a" => 1 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_empty_predicate_across_feeds + @parser << '{"a' # chunk boundary inside a string literal + refute @parser.parse + refute_predicate @parser, :empty? + + @parser << '":1' + refute @parser.parse + refute_predicate @parser, :empty? + + @parser << '}' + assert @parser.parse + refute_predicate @parser, :empty? # value not retrieved yet + assert_equal({ "a" => 1 }, @parser.value) + assert_predicate @parser, :empty? + end + + def test_partial_value_predicate + { + '' => false, + '{"a":1}' => false, + '{"a":1}{"b":2}' => false, + '{"a":1} ' => false, + '{"a":1}{"b":2' => true, # inside a number token + '{"a":1}{"b":' => true, # right after a colon (token boundary) + # The tokenizer rewinds to the token start on EOS, so nothing is + # registered yet for a lone '{' or an unterminated top-level string: + # partial_value returns nil and partial_value? agrees. The truncation + # is still observable through the buffer: eos? is false, rest isn't + # empty. + '{"a":1}{' => false, # right after an object open + '"abc' => false, # inside a string token + '{"a":1,' => true, # right after a comma (token boundary) + '[1,2' => true, # unclosed array + }.each do |json, expected| + parser = new_parser + parser << json + parser.value while parser.parse + assert_equal expected, parser.partial_value?, "expected #{json.inspect} to be partial_value? == #{expected}" + assert_equal !parser.partial_value.nil?, parser.partial_value?, "partial_value?/partial_value mismatch for #{json.inspect}" + end + end + def test_partial_value assert_nil @parser.partial_value assert_partial_value [1, 2, 3], '[1, 2, 3, "unterminated string' From cc010c46889c8516a9a357336b262ce35779985f Mon Sep 17 00:00:00 2001 From: Masataka Pocke Kuwabara Date: Mon, 6 Jul 2026 13:16:42 +0900 Subject: [PATCH 405/506] Fix ResumableParser losing tokens before a feed-boundary suspension When a feed boundary fell right after a consumed token -- a `,` or an opening `[` / `{` -- whose effect was not yet committed to the parser's persistent state, the token was dropped on resume, producing parse errors or silently wrong values. ## Reproduction ```ruby require "json" def parse_in_two(first, second, **opts) parser = JSON::ResumableParser.new(**opts) parser << first parser.parse parser << second parser.parse parser.value rescue JSON::ParserError => e e.message end # a ',' and its closing bracket split across the boundary p parse_in_two("[1,", "]", allow_trailing_comma: true) # a comment right after '[' split across the boundary p parse_in_two("[/*", "*/1]", allow_comments: true) ``` ## Expected Behavior ``` [1] [1] ``` Feeding `[1,]` or `[/**/1]` in a single chunk parses to `[1]`, so splitting it at the comma or inside the comment should make no difference. ## Actual Behavior ``` "unexpected character: ']'" 1 ``` For the comma, the closing bracket that arrived in the later chunk was rejected because the comma had been forgotten. For the bracket, the array's first element leaked out as a bare top-level document (`1` instead of `[1]`), and a following `parse` would raise `unexpected character: ']'` on the orphaned closing bracket. The same shapes affect objects (`{"a":1,` + `}`, `{/*` + `*/"a":1}`), line comments (`[//` + `"x\n1]"`), and comments right after a comma. It happens with the default configuration too, since comments are accepted (with a deprecation warning) unless disabled. ## Description Both bugs share one root cause: the parser consumed a token, then hit a suspension point (end of buffer, possibly inside a comment) before recording the token's effect in the persistent frame/phase state, so resuming had no memory of the token. They differ in where that record lives, so the fix has two parts. On `,`, the comma phases used to eat whitespace and comments before committing the frame's phase, in order to peek for a trailing-comma close. The fix advances the phase to `JSON_PHASE_VALUE` (array) or `JSON_PHASE_OBJECT_KEY` (object) immediately after consuming the comma. Trailing-comma detection moves into those phases: an element/key position that finds `]` / `}` (only reachable after a comma, since an empty container closes inline) hands off to the comma phase to close. On `[` / `{`, the bracket's frame is only pushed after the empty-container check, and pushing it earlier is not an option: an empty container is decoded inline without a frame, which is what keeps `[]` from consuming a nesting level (`[[]]` parses with `max_nesting: 1`). Committing a frame first would break that property or require reworking nesting accounting, affecting the non-resumable parser as well. Instead, widen the rewind: the suspension already rewinds the cursor, it just rewound to the start of the comment, which is after the bracket. `json_eat_comments` gains a `resume_pos` that, in resumable mode only, overrides the rewind target for all three suspension sites (unterminated block comment, unterminated line comment, and a comment marker split at the boundary), and the `[` / `{` cases pass the bracket's own position via the new `json_eat_whitespace_resume_at` variant. Re-reading the bracket and the comment on resume mirrors how strings and numbers already rewind to `value_start` when they hit the end of the buffer. Non-resumable parsing is unaffected in both parts: the same tokens are consumed in the same order, and error positions still point at the comment. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PpeQSEFkF1X1Uuzjx9BsYe --- ext/json/ext/parser/parser.c | 68 ++++++++++++-------- test/json/resumable_parser_test.rb | 99 ++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 24 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 58dd28188..9b7b4be39 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -768,13 +768,18 @@ static const rb_data_type_t JSON_ParserConfig_type; const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`"; NOINLINE(static) void -json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) +json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos) { if (config->on_comment == JSON_RAISE) { raise_syntax_error("unexpected token %s", state); } const char *start = state->cursor; + // An incomplete comment suspends a resumable parse by rewinding the cursor + // and throwing. Callers that already consumed a token not yet committed to + // the frame stack pass resume_pos so the rewind re-reads that token too. + // Non-resumable error positions keep pointing at the comment either way. + const char *rewind_pos = (state->parser && resume_pos) ? resume_pos : start; state->cursor++; switch (peek(state)) { @@ -786,7 +791,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) // the comment unterminated instead of consuming to end as a one-shot // parse would. if (state->parser) { - raise_eos_error_at("unterminated comment, expected end of line", state, start); + raise_eos_error_at("unterminated comment, expected end of line", state, rewind_pos); } state->cursor = state->end; } else { @@ -800,7 +805,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) while (true) { const char *next_match = memchr(state->cursor, '*', state->end - state->cursor); if (!next_match) { - raise_eos_error_at("unterminated comment, expected closing '*/'", state, start); + raise_eos_error_at("unterminated comment, expected closing '*/'", state, rewind_pos); } state->cursor = next_match + 1; @@ -812,7 +817,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) break; } default: - raise_parse_error_at("unexpected token %s", state, start, eos(state)); + raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state)); break; } @@ -823,7 +828,7 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config) } ALWAYS_INLINE(static) void -json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments) +json_eat_whitespace_resume_at(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments, const char *resume_pos) { while (true) { switch (peek(state)) { @@ -858,7 +863,7 @@ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool inc return; } - json_eat_comments(state, config); + json_eat_comments(state, config, resume_pos); break; default: @@ -867,6 +872,12 @@ json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool inc } } +ALWAYS_INLINE(static) void +json_eat_whitespace(JSON_ParserState *state, JSON_ParserConfig *config, bool include_comments) +{ + json_eat_whitespace_resume_at(state, config, include_comments, NULL); +} + static inline VALUE build_string(const char *start, const char *end, bool intern, bool symbolize) { if (symbolize) { @@ -1580,6 +1591,13 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo JSON_PHASE_VALUE: { json_eat_whitespace(state, config, true); + // A trailing comma lands us here expecting an element but finding the + // closing bracket; hand off to ARRAY_COMMA to close. An empty array + // closes inline at '[', so this position is only reached after a ','. + if (config->allow_trailing_comma && frame->type == JSON_FRAME_ARRAY && peek(state) == ']') { + goto JSON_PHASE_ARRAY_COMMA; + } + VALUE value; const char *value_start = state->cursor; @@ -1675,7 +1693,9 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '[': { state->cursor++; - json_eat_whitespace(state, config, true); + // The '[' is consumed but its frame is only pushed below, so a + // comment suspending here must resume from the bracket. + json_eat_whitespace_resume_at(state, config, true, value_start); const char next = peek(state); if (next == ']') { @@ -1704,7 +1724,8 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '{': { state->cursor++; - json_eat_whitespace(state, config, true); + // Same as '[': the frame is only pushed below. + json_eat_whitespace_resume_at(state, config, true, value_start); if (peek(state) == '}') { state->cursor++; @@ -1762,6 +1783,13 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo json_eat_whitespace(state, config, true); + // A trailing comma lands us here expecting a key but finding the closing + // brace; hand off to OBJECT_COMMA to close. An empty object closes inline + // at '{', so this position is only reached after a ','. + if (config->allow_trailing_comma && peek(state) == '}') { + goto JSON_PHASE_OBJECT_COMMA; + } + const char *start = state->cursor; if (RB_LIKELY(peek(state) == '"')) { @@ -1824,13 +1852,10 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo if (RB_LIKELY(next_char == ',')) { state->cursor++; - if (config->allow_trailing_comma) { - json_eat_whitespace(state, config, true); - if (peek(state) == ']') { - // Trailing comma: stay in COMMA to close on the next iteration. - goto JSON_PHASE_ARRAY_COMMA; - } - } + // Commit the phase before eating the whitespace that follows: an + // incomplete comment there would suspend the parse, and a phase not + // yet advanced past the ',' would drop it on resume. A trailing comma + // is recognized in JSON_PHASE_VALUE once the ']' is in the buffer. frame->phase = JSON_PHASE_VALUE; goto JSON_PHASE_VALUE; } else if (next_char == ']') { @@ -1869,15 +1894,10 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo if (RB_LIKELY(next_char == ',')) { state->cursor++; - json_eat_whitespace(state, config, true); - - if (config->allow_trailing_comma) { - if (peek(state) == '}') { - // Trailing comma: stay in COMMA to close on the next iteration. - goto JSON_PHASE_OBJECT_COMMA; - } - } - + // Commit the phase before eating the whitespace that follows: an + // incomplete comment there would suspend the parse, and a phase not + // yet advanced past the ',' would drop it on resume. A trailing comma + // is recognized in JSON_PHASE_OBJECT_KEY once the '}' is in the buffer. frame->phase = JSON_PHASE_OBJECT_KEY; goto JSON_PHASE_OBJECT_KEY; } else if (next_char == '}') { diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 357052c9a..5d7bf38bb 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -241,6 +241,105 @@ def test_block_comment_spanning_feed_boundary_is_not_terminated_early assert_equal [[1], [3]], values end + def test_trailing_comma_split_across_feed_boundary + # With allow_trailing_comma the closing bracket may arrive in a later chunk + # than the comma; consuming the comma must not lose the ability to close. + parser = new_parser(allow_trailing_comma: true) + parser << '[1,' + refute parser.parse + parser << ']' + assert parser.parse + assert_equal [1], parser.value + + parser = new_parser(allow_trailing_comma: true) + parser << '{"a":1,' + refute parser.parse + parser << '}' + assert parser.parse + assert_equal({ "a" => 1 }, parser.value) + + # The boundary can also fall after an inner comma, then after the outer one. + parser = new_parser(allow_trailing_comma: true) + parser << '[[1,' + refute parser.parse + parser << '],]' + assert parser.parse + assert_equal [[1]], parser.value + end + + def test_trailing_comma_byte_by_byte + parser = new_parser(allow_trailing_comma: true) + '[1, 2, ]'.each_char { |c| parser << c; parser.parse } + assert_equal [1, 2], parser.value + + parser = new_parser(allow_trailing_comma: true) + '{ "a": 1, }'.each_char { |c| parser << c; parser.parse } + assert_equal({ "a" => 1 }, parser.value) + end + + def test_comment_after_comma_split_across_feed_boundary + # A comment right after a ',' straddling a feed boundary must not drop the + # comma: the value/key it separates must still be parsed on resume. + # The array case needs allow_trailing_comma: without it the array comma path + # commits its phase before eating the comment, so only the trailing-comma + # path exercises the eat-before-commit bug (the object path always did). + parser = new_parser(allow_comments: true, allow_trailing_comma: true) + parser << '[1,/*' + refute parser.parse + parser << '*/2]' + assert parser.parse + assert_equal [1, 2], parser.value + + parser = new_parser(allow_comments: true) + parser << '{"a":1,/*' + refute parser.parse + parser << '*/"b":2}' + assert parser.parse + assert_equal({ "a" => 1, "b" => 2 }, parser.value) + end + + def test_comment_after_container_open_split_across_feed_boundary + # A comment right after '[' or '{' straddling a feed boundary must not drop + # the opening token: it is consumed before its frame is pushed, so the + # suspension must resume from the bracket, not from inside the comment. + parser = new_parser(allow_comments: true) + parser << '[/*' + refute parser.parse + parser << '*/1]' + assert parser.parse + assert_equal [1], parser.value + + parser = new_parser(allow_comments: true) + parser << '{/*' + refute parser.parse + parser << '*/"a":1}' + assert parser.parse + assert_equal({ "a" => 1 }, parser.value) + + parser = new_parser(allow_comments: true) + parser << '[ /*' + refute parser.parse + parser << '*/ ]' + assert parser.parse + assert_equal [], parser.value + + # The boundary can even split the comment marker itself. + parser = new_parser(allow_comments: true) + parser << '[/' + refute parser.parse + parser << '**/1]' + assert parser.parse + assert_equal [1], parser.value + + # Line comments suspend the same way when their newline hasn't arrived. + parser = new_parser(allow_comments: true) + parser << '[//' + refute parser.parse + parser << "x\n1]" + assert parser.parse + assert_equal [1], parser.value + end + def test_rest @parser << '[1, 2, 3, "unterminated string' refute @parser.parse From d1299d42ac3042f6756af7673ab4ca76178f2d65 Mon Sep 17 00:00:00 2001 From: Masataka Pocke Kuwabara Date: Thu, 9 Jul 2026 15:35:14 +0900 Subject: [PATCH 406/506] Avoid re-decoding an incomplete number on every ResumableParser chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON::ResumableParser` re-decodes an incomplete number on every chunk it receives, so a long number delivered in small chunks is disproportionately slow to parse -- the work is quadratic (a bit worse in practice) in the number's length. ## Background When the parser reads a number that reaches the end of the buffer, it cannot tell whether more digits will arrive in a later chunk, so it rewinds the cursor and returns `false` to wait for more input. Until now it fully decoded the number before that check -- and then discarded the result. For a long run of digits that decode builds a bignum, whose cost grows faster than linearly with the digit count, and it was repeated on every chunk that extended the number. The total cost is therefore quadratic (a bit worse in practice) in the length of the number. The same "rewind and re-process" happens for incomplete strings and comments, but those only re-scan (no value construction), so only numbers pay this decode cost. ## Benchmark Feeding an incomplete token in 128-byte chunks, one iteration being one full chunked feed (`benchmark-ips`; macOS 15.6 arm64; ruby 4.0.0): ```ruby require "json" require "benchmark/ips" def feed_incomplete(token, chunk_size) parser = JSON::ResumableParser.new i = 0 while i < token.bytesize parser << token.byteslice(i, chunk_size) i += chunk_size parser.parse end end CHUNK = 128 sizes = [100, 16_000, 32_000, 64_000, 128_000] Benchmark.ips do |x| x.config(time: 3, warmup: 1) sizes.each do |n| number = "1" * n string = '"' + ("a" * n) x.report("number #{n} digits") { feed_incomplete(number, CHUNK) } x.report("string #{n} chars") { feed_incomplete(string, CHUNK) } end end ``` Per-iteration time, before vs. after this change: | Input (128-byte chunks) | Before | After | Change | |-------------------------|---------:|---------:|-----------------| | number, 100 digits | 949 ns | 414 ns | negligible | | number, 16,000 digits | 16.4 ms | 0.138 ms | ~119x faster | | number, 32,000 digits | 94.8 ms | 0.531 ms | ~178x faster | | number, 64,000 digits | 540.8 ms | 2.05 ms | ~264x faster | | number, 128,000 digits | 3.07 s | 8.13 ms | ~378x faster | | string, 100 chars | 411 ns | 391 ns | ~1x (unchanged) | | string, 16,000 chars | 49.4 μs | 47.7 μs | ~1x (unchanged) | | string, 32,000 chars | 169.7 μs | 165.8 μs | ~1x (unchanged) | | string, 64,000 chars | 621 μs | 616 μs | ~1x (unchanged) | | string, 128,000 chars | 2.38 ms | 2.35 ms | ~1x (unchanged) | The string rows are unchanged, confirming the change is number-specific. ## Fix Defer the decode. `json_parse_number` still scans the digits (cheap, and enough to advance the cursor so the caller can detect an incomplete number), but when the scan reaches the end of the buffer in resumable mode it returns `Qundef` without decoding. The caller already rewinds and returns `false` in exactly that case, so the decoded value was never used there. The number is decoded once, when a following byte proves it complete. Non-resumable `JSON.parse` is unaffected: it passes `resumable = false`, so the early return is never taken and the number is decoded as before. A test checks that large integers and floats split across feeds still decode to the correct value. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PpeQSEFkF1X1Uuzjx9BsYe --- ext/json/ext/parser/parser.c | 16 +++++++++++++--- test/json/resumable_parser_test.rb | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 9b7b4be39..d1c90ff8d 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1457,7 +1457,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat return (int)(state->cursor - start); } -static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start) +static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable) { bool integer = true; const char first_digit = *state->cursor; @@ -1514,6 +1514,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } } + // A number touching the end of the buffer may still grow in a later chunk, + // so the caller will rewind and wait. Decoding it now would build a value + // -- for a long run of digits, an expensive bignum -- only to discard it, + // and repeating that on every resumed chunk is quadratic in the number's + // length. The digit scan above already advanced the cursor, which is all + // the caller needs to detect the incomplete number. + if (RB_UNLIKELY(resumable && eos(state))) { + return Qundef; + } + if (integer) { return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor); } @@ -1638,7 +1648,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '-': { state->cursor++; - value = json_parse_number(state, config, true, value_start); + value = json_parse_number(state, config, true, value_start, resumable); if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) { state->cursor = value_start; @@ -1661,7 +1671,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - value = json_parse_number(state, config, false, value_start); + value = json_parse_number(state, config, false, value_start, resumable); // Top level numbers are ambiguous when parsing streams, we can't // know if we parsed all the digits if we hit EOS. diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 5d7bf38bb..b3c82c167 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -175,6 +175,23 @@ def test_parse_byte_by_byte_numbers assert_resumed_parsing('123 ', trailing_bytes: 1) end + def test_large_numbers_split_across_feeds_are_decoded_correctly + { + '12345678901234567890123456789012345678901234567890 ' => 12345678901234567890123456789012345678901234567890, + '-98765432109876543210987654321 ' => -98765432109876543210987654321, + '3.14159265358979323846264338327950288 ' => 3.14159265358979323846264338327950288, + '-1.5e-300 ' => -1.5e-300, + }.each do |doc, expected| + parser = new_parser + value = nil + doc.each_char do |char| + parser << char + value = parser.value if parser.parse + end + assert_equal expected, value, doc.inspect + end + end + def test_nul_byte_is_a_syntax_error # A NUL byte in a structural position must raise, not stall forever waiting for more input # (peek() returns 0 both at EOS and for a literal NUL byte). From 9be0e8ed39c29f08ad9c72eabdb8aa0851bfa990 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 12 Jul 2026 08:55:36 +0200 Subject: [PATCH 407/506] Release 2.21.0 --- CHANGES.md | 8 ++++++++ lib/json/version.rb | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index aa9785f06..54764e412 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,14 @@ ### Unreleased +### 2026-07-12 (2.21.0) + +* `JSON.generate` now accept a `sort_keys` option, which takes either a boolean or a block. +* Added `#empty?` and `#partial_value?` methods on `JSON::ResumableParser`. +* Numerous correctness and performance fixes for `JSON::ResumableParser`. +* Avoid triggering Ruby's `float out of range` warning when parsing out of range numbers. +* Declare C types with Ruby 4.1 `RUBY_TYPED_THREAD_SAFE_FREE`. + ### 2026-06-23 (2.20.0) * Both C and Java parsers are no longer recursive, so parsing very deep documents with `max_nesting: false` will no longer diff --git a/lib/json/version.rb b/lib/json/version.rb index 76785011e..5c24d0c51 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.20.0' + VERSION = '2.21.0' end From db70b1432157d1c01b556c1367babe72a6a1fe21 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 13 Jul 2026 09:25:04 +0200 Subject: [PATCH 408/506] Rely on Ruby's HAVE_BUILTIN___BUILTIN_CLZLL We don't need to perform out own checks. Fix: https://github.com/ruby/json/pull/1052 --- ext/json/ext/parser/extconf.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index 9e141a726..a0358ccc6 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -15,22 +15,6 @@ have_func("rb_hash_bulk_insert", "ruby.h") # Missing on TruffleRuby have_func("ruby_xfree_sized", "ruby.h") # RUBY_VERSION >= 4.1 -def have_builtin_func(name, check_expr, opt = "", &b) - checking_for checking_message(name.funcall_style, nil, opt) do - if try_compile(< Date: Mon, 13 Jul 2026 09:43:25 +0200 Subject: [PATCH 409/506] Release 2.21.1 --- CHANGES.md | 4 ++++ lib/json/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 54764e412..b13642ae5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ### Unreleased +### 2026-07-13 (2.21.1) + +* Fix a compilation issue on Window and Microsoft Visual C++. + ### 2026-07-12 (2.21.0) * `JSON.generate` now accept a `sort_keys` option, which takes either a boolean or a block. diff --git a/lib/json/version.rb b/lib/json/version.rb index 5c24d0c51..e8483db94 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.21.0' + VERSION = '2.21.1' end From f5fc2454fdf518a3ef7a024ed0d946587f94bd00 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Sun, 19 Jul 2026 21:12:05 -0500 Subject: [PATCH 410/506] Pass '-mno-outline' to the compiler (if supported) to prevent function outlining. --- ext/json/ext/parser/extconf.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ext/json/ext/parser/extconf.rb b/ext/json/ext/parser/extconf.rb index a0358ccc6..18aa546a9 100644 --- a/ext/json/ext/parser/extconf.rb +++ b/ext/json/ext/parser/extconf.rb @@ -30,6 +30,12 @@ append_cflags("-std=c99") +# Disable function outlining on clang to prevent some of the repeated instructions +# in json_eat_whitespace being outlined into function calls. +# Note: This verifies '-mno-outline' is accepted as a valid compiler flag +# and will not pass it if unsupported. +append_cflags("-mno-outline") + if enable_config('parser-use-simd', default=!ENV["JSON_DISABLE_SIMD"]) load __dir__ + "/../simd/conf.rb" end From 499d9bcf22e6a9183f7b765446a458a57ca127ae Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 26 Jul 2026 11:01:14 +0200 Subject: [PATCH 411/506] Fix the parser benchmark bytes report --- benchmark/parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/parser.rb b/benchmark/parser.rb index 54ed32151..0ec73f4c1 100644 --- a/benchmark/parser.rb +++ b/benchmark/parser.rb @@ -21,7 +21,7 @@ end def benchmark_parsing(name, json_output) - puts "== Parsing #{name} (#{json_output.size} bytes)" + puts "== Parsing #{name} (#{json_output.bytesize} bytes)" coder = JSON::Coder.new Benchmark.ips do |x| From 5499aa54a46a8e399e2568e6569b504f0f0e2a41 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 29 Jul 2026 09:48:05 +0200 Subject: [PATCH 412/506] Improve max_nesting documentation --- lib/json.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/json.rb b/lib/json.rb index d1e94147d..74e258424 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -121,9 +121,11 @@ # ====== Input Options # # Option +max_nesting+ (\Integer) specifies the maximum nesting depth allowed; -# defaults to +100+; specify +false+ to disable depth checking. +# defaults to +100+; +# You can set it to +false+ to disable depth checking entirely, but that is dangerous +# when parsing untrusted input. # -# With the default, +false+: +# With the default, +100+: # source = '[0, [1, [2, [3]]]]' # ruby = JSON.parse(source) # ruby # => [0, [1, [2, [3]]]] @@ -384,6 +386,15 @@ # # Raises JSON::NestingError (nesting of 2 is too deep): # JSON.generate(obj, max_nesting: 2) # +# With +false+: +# obj = [] +# obj[0] = obj +# # Raises SystemStackError: stack level too deep +# JSON.generate(obj, max_nesting: false) +# +# Setting +max_nesting+ to +false+ can lead to a stackoverflow and may leave the program +# in an unrecoverable state. It is discouraged. +# # ====== Escaping Options # # Options +script_safe+ (boolean) specifies wether '\u2028', '\u2029' From 2c332bfe2bfb0e754da07e2a0310ef106bf46482 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 30 Jul 2026 08:35:40 +0200 Subject: [PATCH 413/506] Also don't compute ResumableParser cursor position for warning Same reason we don't for errors, since the resumable parser doesn't keep the entire stream in memory, it can't always be accurate, and the buffer may even have been freed. Anyway, for a long lived stream, it wouldn't be very helpful either. Also adds some assertions to ensure the buffer liveness. --- ext/json/ext/parser/parser.c | 15 +++++++++++---- test/json/resumable_parser_test.rb | 13 +++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d1c90ff8d..e577c5883 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -589,6 +589,8 @@ static inline char peek(JSON_ParserState *state) static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out) { + JSON_ASSERT(!state->parser); + JSON_ASSERT(state->cursor); JSON_ASSERT(state->cursor <= state->end); // Redundant but helpful for hardening @@ -621,10 +623,14 @@ static const unsigned int MAX_DEPRECATIONS = 5; static void emit_parse_warning(const char *message, JSON_ParserState *state) { - long line, column; - cursor_position(state, &line, &column); - - VALUE warning = rb_sprintf("%s at line %ld column %ld", message, line, column); + VALUE warning; + if (state->parser) { // line and columns can't be accurate in resumable + warning = rb_utf8_str_new_cstr(message); + } else { + long line, column; + cursor_position(state, &line, &column); + warning = rb_sprintf("%s at line %ld column %ld", message, line, column); + } rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning); } @@ -2566,6 +2572,7 @@ static VALUE cResumableParser_parse(VALUE self) if (eos(&parser->state)) { json_str_clear(parser->buffer); parser->buffer = Qfalse; + parser->state.start = parser->state.cursor = parser->state.end = 0; } parser->in_use = false; diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index b3c82c167..11a5d2eb5 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -668,6 +668,19 @@ def test_parse_error_message assert_equal 0, error.column end + def test_duplicate_key_warning_message + parser = new_parser + parser << (%q({"a":1,"a":2,"pad":") + ("x" * (4 * 1024 * 1024))) + refute parser.parse + + parser << %q(",) + refute parser.parse + + assert_deprecated_warning(/duplicate key "a"/) do + parser.partial_value + end + end + private def assert_parse_error(json, expected_error_message = nil) From 5a32e4367ca5976f62af12887a09685b7109b054 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 31 Jul 2026 08:24:11 +0200 Subject: [PATCH 414/506] Release 2.21.2 --- CHANGES.md | 4 ++++ lib/json/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index b13642ae5..538650c2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ### Unreleased +### 2026-07-31 (2.21.2) + +* Fix a use-after-free bug in `JSON::ResumableParser`. [GHSA-9hj4-r449-hfvc]. + ### 2026-07-13 (2.21.1) * Fix a compilation issue on Window and Microsoft Visual C++. diff --git a/lib/json/version.rb b/lib/json/version.rb index e8483db94..5c66a9d70 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.21.1' + VERSION = '2.21.2' end From 0238abcaf3ec6d101634b9087cf4ec2d0382a77d Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Thu, 6 Aug 2026 07:19:23 -0500 Subject: [PATCH 415/506] Widen the json_decode_integer fast path to more 64-bit integers. When this code was refactored in 9c4db31, the mantissa_digits captured only the length of the mantissa and the negative sign was captured separately. Additionally, the paths diverged between signed and unsigned values. However, MAX_FAST_INTEGER_SIZE stayed at 18 which limited the range of uint64_t and int64_t that took the fast path. This change expands the range of uint64_t and int64_t's that can be returned by the fast paths without falling back to json_decode_large_number. --- ext/json/ext/parser/parser.c | 28 +++++++++++++++++++++++---- test/json/json_parser_test.rb | 31 ++++++++++++++++++++++++++++++ test/json/resumable_parser_test.rb | 21 ++++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index e577c5883..1443095b6 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1082,7 +1082,8 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser return result; } -#define MAX_FAST_INTEGER_SIZE 18 +#define MAX_FAST_INTEGER_SIZE 19 +#define MAX_FAST_UINT64_SIZE 20 #define MAX_NUMBER_STACK_BUFFER 128 typedef VALUE (*json_number_decode_func_t)(const char *ptr); @@ -1117,11 +1118,30 @@ NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len) static inline VALUE json_decode_integer(uint64_t mantissa, int mantissa_digits, bool negative, const char *start, const char *end) { - if (RB_LIKELY(mantissa_digits < MAX_FAST_INTEGER_SIZE)) { - if (negative) { + if (RB_LIKELY(mantissa_digits <= MAX_FAST_INTEGER_SIZE)) { + if (RB_LIKELY(!negative)) { + return UINT64T2NUM(mantissa); + } + + // For a negative number 19 digits in length, we only get half of the range, + // so ensure this negative number is less than INT64_MAX. + // + // Note: This does miss INT64_MIN as it's value is one past INT64_MAX + // when converted to a uint64_t. It will still be parsed correctly by + // falling through to json_decode_large_integer. + if (RB_LIKELY(mantissa <= (uint64_t)INT64_MAX)) { return INT64T2NUM(-((int64_t)mantissa)); } - return UINT64T2NUM(mantissa); + } + + if (!negative && mantissa_digits == MAX_FAST_UINT64_SIZE) { + // Not all 20 digit integers can be safely represented by a uint64_t but + // some can. The memcmp with uint64_max is safe as we've rejected leading + // zeros and we have guaranteed we're comparing it with a 20 digit number. + static const char uint64_max[] = "18446744073709551615"; + if (memcmp(end - MAX_FAST_UINT64_SIZE, uint64_max, MAX_FAST_UINT64_SIZE) <= 0) { + return UINT64T2NUM(mantissa); + } } return json_decode_large_integer(start, end - start); diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 2f79b87cc..f75741fd6 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -132,6 +132,37 @@ def test_parse_numbers capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) } end + INTEGER_FAST_PATH_BOUNDARIES = [ + "999999999999999999", # 18 digits + "1000000000000000000", # narrowest 19 digits + "9999999999999999999", # widest 19 digits, still inside uint64_t + "-999999999999999999", + "-9223372036854775807", # INT64_MAX, the widest negatable accumulator + "-9223372036854775808", # INT64_MIN, one past what negating a uint64_t covers + "-9223372036854775809", + "-9999999999999999999", # widest negative 19 digits + "10000000000000000000", # narrowest 20 digits + "18446744073709551614", + "18446744073709551615", # UINT64_MAX exactly, the last value the range check admits + "18446744073709551616", # 2**64, first value it must reject + "18446744073709551617", + "19999999999999999999", + "99999999999999999999", # widest 20 digits + "-18446744073709551615", # negatives never take the 20 digit path + "-18446744073709551616", + "-99999999999999999999", + "100000000000000000000", # 21 digits, always a bignum + "-100000000000000000000", + ].freeze + + def test_parse_integer_boundaries + INTEGER_FAST_PATH_BOUNDARIES.each do |literal| + expected = Integer(literal, 10) + + assert_equal(expected, parse(literal)) + end + end + def test_parse_bignum bignum = Integer('1234567890' * 10) assert_equal(bignum, JSON.parse(bignum.to_s)) diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 11a5d2eb5..0b8e13560 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -192,6 +192,27 @@ def test_large_numbers_split_across_feeds_are_decoded_correctly end end + def test_integer_boundaries_split_across_feeds + [ + '9999999999999999999', # widest 19 digits + '-9223372036854775808', # INT64_MIN + '18446744073709551615', # UINT64_MAX exactly + '18446744073709551616', # 2**64, wraps the accumulator to 0 + '99999999999999999999', # widest 20 digits + '-18446744073709551616', + '100000000000000000000', # 21 digits + ].each do |literal| + doc = "#{literal} " + parser = new_parser + value = nil + doc.each_char do |char| + parser << char + value = parser.value if parser.parse + end + assert_equal Integer(literal, 10), value, doc.inspect + end + end + def test_nul_byte_is_a_syntax_error # A NUL byte in a structural position must raise, not stall forever waiting for more input # (peek() returns 0 both at EOS and for a literal NUL byte). From 0708dd2bd1e04380ecda0bd1c34a1423d671c668 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 7 Aug 2026 21:00:21 +0200 Subject: [PATCH 416/506] Update CHANGES.md --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 538650c2f..3f455f7c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ ### 2026-07-31 (2.21.2) -* Fix a use-after-free bug in `JSON::ResumableParser`. [GHSA-9hj4-r449-hfvc]. +* Fix a use-after-free bug in `JSON::ResumableParser`. [GHSA-9hj4-r449-hfvc][CVE-2026-71847]. ### 2026-07-13 (2.21.1) From a8be79a41c44aadc0577ca3ed60831ada0d8ca64 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 13:51:51 +0200 Subject: [PATCH 417/506] Move coverage report path to .coverage --- .gitignore | 1 + test/json/test_helper.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index bb35f362a..5865b9834 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.bundle.dSYM *.so coverage +.coverage tags pkg .nfs.* diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index 4c5a91a19..ddc75d3ac 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -22,6 +22,8 @@ add_filter 'lib/json/truffle_ruby' unless RUBY_ENGINE == 'truffleruby' add_filter 'test/' + + coverage_dir '.coverage' end end From 3df735361ec5809dea40386fb7c09d6b4aeb3c81 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 19:38:48 +0200 Subject: [PATCH 418/506] Remove `j` and `jj` --- CHANGES.md | 2 ++ lib/json/common.rb | 30 ------------------------------ 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3f455f7c4..97758091a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Remove `Kernel#j` and `Kernel#jj` + ### 2026-07-31 (2.21.2) * Fix a use-after-free bug in `JSON::ResumableParser`. [GHSA-9hj4-r449-hfvc][CVE-2026-71847]. diff --git a/lib/json/common.rb b/lib/json/common.rb index fdcb860db..a8a83d9c9 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -1136,36 +1136,6 @@ def to_json(state = nil, *) module ::Kernel private - # Outputs _objs_ to STDOUT as JSON strings in the shortest form, that is in - # one line. - def j(*objs) - if RUBY_VERSION >= "3.0" - warn "Kernel#j is deprecated and will be removed in json 3.0.0", uplevel: 1, category: :deprecated - else - warn "Kernel#j is deprecated and will be removed in json 3.0.0", uplevel: 1 - end - - objs.each do |obj| - puts JSON.generate(obj, :allow_nan => true, :max_nesting => false) - end - nil - end - - # Outputs _objs_ to STDOUT as JSON strings in a pretty format, with - # indentation and over many lines. - def jj(*objs) - if RUBY_VERSION >= "3.0" - warn "Kernel#jj is deprecated and will be removed in json 3.0.0", uplevel: 1, category: :deprecated - else - warn "Kernel#jj is deprecated and will be removed in json 3.0.0", uplevel: 1 - end - - objs.each do |obj| - puts JSON.pretty_generate(obj, :allow_nan => true, :max_nesting => false) - end - nil - end - # If _object_ is string-like, parse the string and return the parsed result as # a Ruby data structure. Otherwise, generate a JSON text from the Ruby data # structure object and return it. From e7636af6e67b015b30e9d017bd003d481adfef00 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 19:49:59 +0200 Subject: [PATCH 419/506] Make `allow_duplicate_key: false` the default --- CHANGES.md | 1 + ext/json/ext/generator/generator.c | 25 ++++---------------- ext/json/ext/parser/parser.c | 33 ++++----------------------- java/src/json/ext/Generator.java | 9 ++------ java/src/json/ext/GeneratorState.java | 24 ++----------------- java/src/json/ext/Parser.java | 20 ++-------------- lib/json.rb | 25 ++++++-------------- lib/json/common.rb | 9 ++------ lib/json/truffle_ruby/generator.rb | 16 ++++--------- test/json/json_generator_test.rb | 8 +------ test/json/json_parser_test.rb | 19 --------------- test/json/resumable_parser_test.rb | 13 ----------- 12 files changed, 30 insertions(+), 172 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 97758091a..ee3674f7b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. * Remove `Kernel#j` and `Kernel#jj` ### 2026-07-31 (2.21.2) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 6f473325e..a45fd3928 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -9,12 +9,6 @@ /* ruby api and some helpers */ -enum duplicate_key_action { - JSON_DEPRECATED = 0, - JSON_IGNORE, - JSON_RAISE, -}; - typedef struct JSON_Generator_StateStruct { VALUE indent; VALUE space; @@ -27,8 +21,7 @@ typedef struct JSON_Generator_StateStruct { long depth; long buffer_initial_length; - enum duplicate_key_action on_duplicate_key; - + bool allow_duplicate_key; bool as_json_single_arg; bool allow_nan; bool ascii_only; @@ -956,9 +949,8 @@ json_inspect_hash_with_mixed_keys(struct hash_foreach_arg *arg) arg->mixed_keys_encountered = true; JSON_Generator_State *state = arg->data->state; - if (state->on_duplicate_key != JSON_IGNORE) { - VALUE do_raise = state->on_duplicate_key == JSON_RAISE ? Qtrue : Qfalse; - rb_funcall(mJSON, rb_intern("on_mixed_keys_hash"), 2, arg->hash, do_raise); + if (!state->allow_duplicate_key) { + rb_funcall(mJSON, rb_intern("on_mixed_keys_hash"), 1, arg->hash); } } @@ -1784,14 +1776,7 @@ static VALUE cState_sort_keys_set(VALUE self, VALUE value) static VALUE cState_allow_duplicate_key_p(VALUE self) { GET_STATE(self); - switch (state->on_duplicate_key) { - case JSON_IGNORE: - return Qtrue; - case JSON_DEPRECATED: - return Qnil; - default: - return Qfalse; - } + return state->allow_duplicate_key ? Qtrue : Qfalse; } /* @@ -1885,7 +1870,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_script_safe) { state->script_safe = RTEST(val); } else if (key == sym_escape_slash) { state->script_safe = RTEST(val); } else if (key == sym_strict) { state->strict = RTEST(val); } - else if (key == sym_allow_duplicate_key) { state->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } + else if (key == sym_allow_duplicate_key) { state->allow_duplicate_key = RTEST(val); } else if (key == sym_as_json) { VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 1443095b6..4a6344cb6 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -413,9 +413,9 @@ typedef struct JSON_ParserStruct { VALUE on_load_proc; VALUE decimal_class; ID decimal_method_id; - enum deprecatable_action on_duplicate_key; enum deprecatable_action on_comment; int max_nesting; + bool allow_duplicate_key; bool allow_nan; bool allow_trailing_comma; bool allow_control_characters; @@ -1219,17 +1219,6 @@ static VALUE json_find_duplicated_key(size_t count, const VALUE *pairs) return Qfalse; } -NOINLINE(static) void emit_duplicate_key_warning(JSON_ParserState *state, VALUE duplicate_key) -{ - VALUE message = rb_sprintf( - "detected duplicate key %"PRIsVALUE" in JSON object. This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`", - rb_inspect(duplicate_key) - ); - - emit_parse_warning(RSTRING_PTR(message), state); - RB_GC_GUARD(message); -} - NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE duplicate_key) { VALUE message = rb_sprintf( @@ -1250,23 +1239,9 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) { - switch (config->on_duplicate_key) { - case JSON_IGNORE: - return; - - case JSON_DEPRECATED: - // Only emit the first few deprecations to avoid spamming. - if (state->emitted_deprecations < MAX_DEPRECATIONS) { - state->emitted_deprecations++; - emit_duplicate_key_warning(state, json_find_duplicated_key(count, pairs)); - } - return; - - case JSON_RAISE: - raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); - return; + if (!config->allow_duplicate_key) { + raise_duplicate_key_error(state, json_find_duplicated_key(count, pairs)); } - UNREACHABLE; } static inline VALUE json_decode_object(JSON_ParserState *state, JSON_ParserConfig *config, size_t count) @@ -2038,7 +2013,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); } - else if (key == sym_allow_duplicate_key) { config->on_duplicate_key = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } + else if (key == sym_allow_duplicate_key) { config->allow_duplicate_key = RTEST(val); } else if (key == sym_decimal_class) { if (RTEST(val)) { if (rb_respond_to(val, i_try_convert)) { diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index f1617a051..ce1b5b57e 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -553,13 +553,8 @@ private void report(ThreadContext context, Session session) { final GeneratorState state = session.getState(context); if (!state.getAllowDuplicateKey()) { - if (state.getDeprecateDuplicateKey()) { - IRubyObject args[] = new IRubyObject[]{hash, context.getRuntime().getFalse()}; - info.jsonModule.get().callMethod(context, "on_mixed_keys_hash", args); - } else { - IRubyObject args[] = new IRubyObject[]{hash, context.getRuntime().getTrue()}; - info.jsonModule.get().callMethod(context, "on_mixed_keys_hash", args); - } + IRubyObject args[] = new IRubyObject[]{hash}; + info.jsonModule.get().callMethod(context, "on_mixed_keys_hash", args); } } } diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 2d20ea461..b5b28d2be 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -35,7 +35,6 @@ */ public class GeneratorState extends RubyObject { private boolean allowDuplicateKey = false; - private boolean deprecateDuplicateKey = true; private static IRubyObject defaultSortKeyProc; @@ -236,7 +235,6 @@ public IRubyObject initialize_copy(ThreadContext context, IRubyObject vOrig) { this.depth = orig.depth; this.allowDuplicateKey = orig.allowDuplicateKey; - this.deprecateDuplicateKey = orig.deprecateDuplicateKey; this.sortKeys = orig.sortKeys; return this; @@ -550,8 +548,6 @@ private ByteList prepareByteList(ThreadContext context, IRubyObject value) { public IRubyObject allow_duplicate_key_p(ThreadContext context) { if (allowDuplicateKey) { return context.runtime.getTrue(); - } else if (deprecateDuplicateKey) { - return context.runtime.getNil(); } return context.runtime.getFalse(); } @@ -560,10 +556,6 @@ public boolean getAllowDuplicateKey() { return allowDuplicateKey; } - public boolean getDeprecateDuplicateKey() { - return deprecateDuplicateKey; - } - /** * State#configure(opts) * @@ -608,11 +600,7 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { if (depth < 0) { throw context.runtime.newArgumentError("depth must be >= 0 (got: " + depth + ")"); } - - if (opts.hasKey("allow_duplicate_key")) { - this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); - this.deprecateDuplicateKey = false; - } + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); @@ -645,15 +633,7 @@ public RubyHash to_h(ThreadContext context) { result.op_aset(context, runtime.newSymbol("depth"), depth_get(context)); result.op_aset(context, runtime.newSymbol("buffer_initial_length"), buffer_initial_length_get(context)); result.op_aset(context, runtime.newSymbol("sort_keys"), sort_keys_get(context)); - - if (this.allowDuplicateKey) { - if (!this.deprecateDuplicateKey) { - result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), runtime.getTrue()); - } - } - else { - result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), runtime.getFalse()); - } + result.op_aset(context, runtime.newSymbol("allow_duplicate_key"), allow_duplicate_key_p(context)); for (String name: getInstanceVariableNameList()) { result.op_aset(context, runtime.newSymbol(name.substring(1)), getInstanceVariables().getInstanceVariable(name)); diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index c54d1a7f9..6d5c18e06 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -50,7 +50,6 @@ public class Parser extends RubyObject { private boolean allowControlCharacters; private boolean allowInvalidEscape; private boolean allowDuplicateKey; - private boolean deprecateDuplicateKey; private boolean symbolizeNames; private boolean freeze; private RubyProc onLoadProc; @@ -109,13 +108,7 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); this.symbolizeNames = opts.getBool("symbolize_names", false); - if (opts.hasKey("allow_duplicate_key")) { - this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); - this.deprecateDuplicateKey = false; - } else { - this.allowDuplicateKey = false; - this.deprecateDuplicateKey = true; - } + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); this.freeze = opts.getBool("freeze", false); this.onLoadProc = opts.getProc("on_load"); @@ -585,16 +578,7 @@ private void onDuplicateKey(IRubyObject key) { // match the C parser's message. String keyInspect = key.callMethod(context, "to_s") .callMethod(context, "inspect").asJavaString(); - if (config.deprecateDuplicateKey) { - if (emittedDeprecations < MAX_DEPRECATIONS) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "detected duplicate key " + keyInspect + " in JSON object. " + - "This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`"); - } - } else { - throw newException(Utils.M_PARSER_ERROR, "duplicate key " + keyInspect); - } + throw newException(Utils.M_PARSER_ERROR, "duplicate key " + keyInspect); } private int parseDigits(long value) { diff --git a/lib/json.rb b/lib/json.rb index 74e258424..e544fc773 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -141,18 +141,12 @@ # Option +allow_duplicate_key+ specifies whether duplicate keys in objects # should be ignored or cause an error to be raised: # -# When not specified: -# # The last value is used and a deprecation warning emitted. -# JSON.parse('{"a": 1, "a":2}') => {"a" => 2} -# # warning: detected duplicate keys in JSON object. -# # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` +# When set to +false+, the default: +# JSON.parse('{"a": 1, "a":2}') => duplicate key at line 1 column 1 (JSON::ParserError) # # When set to +true+: # # The last value is used. -# JSON.parse('{"a": 1, "a":2}') => {"a" => 2} -# -# When set to +false+, the future default: -# JSON.parse('{"a": 1, "a":2}') => duplicate key at line 1 column 1 (JSON::ParserError) +# JSON.parse('{"a": 1, "a":2}', allow_duplicate_key: true) => {"a" => 2} # # --- # @@ -360,18 +354,13 @@ # hashes with duplicate keys should be allowed or produce an error. # defaults to emit a deprecation warning. # -# With the default, (not set): -# Warning[:deprecated] = true +# With the default, false: # JSON.generate({ foo: 1, "foo" => 2 }) -# # warning: detected duplicate key "foo" in {foo: 1, "foo" => 2}. -# # This will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true` -# # => '{"foo":1,"foo":2}' -# -# With false -# JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: false) # # detected duplicate key "foo" in {foo: 1, "foo" => 2} (JSON::GeneratorError) # -# In version 3.0, false will become the default. +# With true +# JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: true) +# # => '{"foo":1,"foo":2}' # # --- # diff --git a/lib/json/common.rb b/lib/json/common.rb index a8a83d9c9..92ee0e4c7 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -193,18 +193,13 @@ def generator=(generator) # :nodoc: private # Called from the extension when a hash has both string and symbol keys - def on_mixed_keys_hash(hash, do_raise) + def on_mixed_keys_hash(hash) set = {} hash.each_key do |key| key_str = key.to_s if set[key_str] - message = "detected duplicate key #{key_str.inspect} in #{hash.inspect}" - if do_raise - raise GeneratorError, message - else - deprecation_warning("#{message}.\nThis will raise an error in json 3.0 unless enabled via `allow_duplicate_key: true`") - end + raise GeneratorError, "detected duplicate key #{key_str.inspect} in #{hash.inspect}" else set[key_str] = true end diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index ee6186fbe..450684a50 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -167,6 +167,7 @@ def initialize(opts = nil) @strict = false @max_nesting = 100 @sort_keys = false + @allow_duplicate_key = false configure(opts) if opts end @@ -318,6 +319,7 @@ def configure(opts) self.sort_keys = opts[:sort_keys] if opts.key?(:sort_keys) @depth = opts[:depth] || 0 @buffer_initial_length ||= opts[:buffer_initial_length] + @allow_duplicate_key = !!opts[:allow_duplicate_key] @script_safe = if opts.key?(:script_safe) !!opts[:script_safe] @@ -327,12 +329,6 @@ def configure(opts) false end - if opts.key?(:allow_duplicate_key) - @allow_duplicate_key = !!opts[:allow_duplicate_key] - else - @allow_duplicate_key = nil # nil is deprecation - end - @strict = !!opts[:strict] if opts.key?(:strict) if !opts.key?(:max_nesting) # defaults to 100 @@ -362,10 +358,6 @@ def to_h result[key.to_sym] = instance_variable_get(iv) end - if result[:allow_duplicate_key].nil? - result.delete(:allow_duplicate_key) - end - result end @@ -416,7 +408,7 @@ def generate(obj, anIO = nil) else if key_type && !@allow_duplicate_key && key_type != k.class key_type = nil # stop checking - JSON.send(:on_mixed_keys_hash, obj, !@allow_duplicate_key.nil?) + JSON.send(:on_mixed_keys_hash, obj) end buf << ',' end @@ -569,7 +561,7 @@ def json_transform(state) else if key_type && !state.allow_duplicate_key? && key_type != key.class key_type = nil # stop checking - JSON.send(:on_mixed_keys_hash, self, state.allow_duplicate_key? == false) + JSON.send(:on_mixed_keys_hash, self) end result << delim end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index ab19704b1..94c48a604 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -329,6 +329,7 @@ def test_falsy_state def test_state_defaults state = JSON::State.new assert_equal({ + :allow_duplicate_key => false, :allow_nan => false, :array_nl => "", :as_json => false, @@ -1090,13 +1091,6 @@ def test_generate_duplicate_keys_allowed assert_equal %({"foo":1,"foo":2}), JSON.generate(hash, allow_duplicate_key: true) end - def test_generate_duplicate_keys_deprecated - hash = { foo: 1, "foo" => 2 } - assert_deprecated_warning(/allow_duplicate_key/) do - assert_equal %({"foo":1,"foo":2}), JSON.generate(hash) - end - end - def test_generate_duplicate_keys_disallowed hash = { foo: 1, "foo" => 2 } error = assert_raise JSON::GeneratorError do diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index f75741fd6..f0df848aa 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -436,35 +436,16 @@ def test_parse_escaped_key def test_parse_duplicate_key expected = {"a" => 2} - expected_sym = {a: 2} assert_equal expected, parse('{"a": 1, "a": 2}', allow_duplicate_key: true) assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false) } assert_raise(ParserError) { parse('{"a": 1, "a": 2}', allow_duplicate_key: false, symbolize_names: true) } - assert_deprecated_warning(/duplicate key "a"/) do - assert_equal expected, parse('{"a": 1, "a": 2}') - end - assert_deprecated_warning(/duplicate key "a"/) do - assert_equal expected_sym, parse('{"a": 1, "a": 2}', symbolize_names: true) - end - - if RUBY_ENGINE == 'ruby' - assert_deprecated_warning(/#{File.basename(__FILE__)}\:#{__LINE__ + 1}/) do - assert_equal expected, parse('{"a": 1, "a": 2}') - end - end - unless RUBY_ENGINE == 'jruby' assert_raise(ParserError) do fake_key = Object.new JSON.load('{"a": 1, "a": 2}', -> (obj) { obj == "a" ? fake_key : obj }, allow_duplicate_key: false) end - - assert_deprecated_warning(/duplicate key # (obj) { obj == "a" ? fake_key : obj }) - end end end diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 0b8e13560..289da7547 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -689,19 +689,6 @@ def test_parse_error_message assert_equal 0, error.column end - def test_duplicate_key_warning_message - parser = new_parser - parser << (%q({"a":1,"a":2,"pad":") + ("x" * (4 * 1024 * 1024))) - refute parser.parse - - parser << %q(",) - refute parser.parse - - assert_deprecated_warning(/duplicate key "a"/) do - parser.partial_value - end - end - private def assert_parse_error(json, expected_error_message = nil) From bc24969dab43d83f80a2dd039374a89e76ab1d07 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:07:20 +0200 Subject: [PATCH 420/506] Make `allow_comments: false` the default --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 35 +++-------------------------------- java/src/json/ext/Parser.java | 20 ++------------------ lib/json.rb | 8 +++----- test/json/json_parser_test.rb | 8 -------- 5 files changed, 9 insertions(+), 63 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ee3674f7b..77b8b428d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. * Remove `Kernel#j` and `Kernel#jj` diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 4a6344cb6..54ecb2c89 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -403,18 +403,12 @@ typedef struct json_frame_stack_struct { json_frame *ptr; } json_frame_stack; -enum deprecatable_action { - JSON_DEPRECATED = 0, - JSON_IGNORE, - JSON_RAISE, -}; - typedef struct JSON_ParserStruct { VALUE on_load_proc; VALUE decimal_class; ID decimal_method_id; - enum deprecatable_action on_comment; int max_nesting; + bool allow_comments; bool allow_duplicate_key; bool allow_nan; bool allow_trailing_comma; @@ -435,7 +429,6 @@ typedef struct JSON_ParserStateStruct { rvalue_cache name_cache; int in_array; int current_nesting; - unsigned int emitted_deprecations; VALUE parser; } JSON_ParserState; @@ -619,21 +612,6 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum *column_out = column; } -static const unsigned int MAX_DEPRECATIONS = 5; - -static void emit_parse_warning(const char *message, JSON_ParserState *state) -{ - VALUE warning; - if (state->parser) { // line and columns can't be accurate in resumable - warning = rb_utf8_str_new_cstr(message); - } else { - long line, column; - cursor_position(state, &line, &column); - warning = rb_sprintf("%s at line %ld column %ld", message, line, column); - } - rb_funcall(mJSON, rb_intern("deprecation_warning"), 1, warning); -} - #define PARSE_ERROR_FRAGMENT_LEN 32 static VALUE build_parse_error_message(const char *format, JSON_ParserState *state) @@ -772,11 +750,10 @@ static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const static const rb_data_type_t JSON_ParserConfig_type; -const char *COMMENT_DEPRECATION_MESSAGE = "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`"; NOINLINE(static) void json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char *resume_pos) { - if (config->on_comment == JSON_RAISE) { + if (!config->allow_comments) { raise_syntax_error("unexpected token %s", state); } @@ -826,11 +803,6 @@ json_eat_comments(JSON_ParserState *state, JSON_ParserConfig *config, const char raise_parse_error_at("unexpected token %s", state, eos(state) ? rewind_pos : start, eos(state)); break; } - - if (config->on_comment == JSON_DEPRECATED && state->emitted_deprecations < MAX_DEPRECATIONS) { - state->emitted_deprecations++; - emit_parse_warning(COMMENT_DEPRECATION_MESSAGE, state); - } } ALWAYS_INLINE(static) void @@ -2007,7 +1979,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) if (key == sym_max_nesting) { config->max_nesting = RTEST(val) ? FIX2INT(val) : 0; } else if (key == sym_allow_nan) { config->allow_nan = RTEST(val); } else if (key == sym_allow_trailing_comma) { config->allow_trailing_comma = RTEST(val); } - else if (key == sym_allow_comments) { config->on_comment = RTEST(val) ? JSON_IGNORE : JSON_RAISE; } + else if (key == sym_allow_comments) { config->allow_comments = RTEST(val); } else if (key == sym_allow_control_characters) { config->allow_control_characters = RTEST(val); } else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } @@ -2641,7 +2613,6 @@ static VALUE cResumableParser_clear(VALUE self) parser->state.name_cache.length = 0; parser->state.current_nesting = 0; parser->state.in_array = 1; - parser->state.emitted_deprecations = 0; parser->state.start = parser->state.cursor = parser->state.end = NULL; return self; } diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 6d5c18e06..b2be38d15 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -46,7 +46,6 @@ public class Parser extends RubyObject { private boolean allowNaN; private boolean allowTrailingComma; private boolean allowComments; - private boolean deprecateComments; private boolean allowControlCharacters; private boolean allowInvalidEscape; private boolean allowDuplicateKey; @@ -96,14 +95,7 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { OptionsReader opts = new OptionsReader(context, options); this.maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); this.allowNaN = opts.getBool("allow_nan", false); - if (opts.hasKey("allow_comments")) { - this.allowComments = opts.getBool("allow_comments", false); - this.deprecateComments = false; - } else { - this.allowComments = true; - this.deprecateComments = true; - } - + this.allowComments = opts.getBool("allow_comments", false); this.allowControlCharacters = opts.getBool("allow_control_characters", false); this.allowInvalidEscape = opts.getBool("allow_invalid_escape", false); this.allowTrailingComma = opts.getBool("allow_trailing_comma", false); @@ -843,15 +835,7 @@ private void eatWhitespace() { private void eatComments() { if (!config.allowComments) { - if (config.deprecateComments) { - if (emittedDeprecations < MAX_DEPRECATIONS) { - emittedDeprecations++; - context.runtime.getWarnings().warning( - "Encountered comment in JSON. This will raise an error in json 3.0 unless enabled via `allow_comments: true`"); - } - } else { - throw unexpectedToken(cursor, end); - } + throw unexpectedToken(cursor, end); } int start = cursor; diff --git a/lib/json.rb b/lib/json.rb index e544fc773..90b4b3c91 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -184,13 +184,11 @@ # JavaScript style comments (either // comment or /* comment */); # defaults to +false+. # -# When not specified, a deprecation warning is emitted if a comment is encountered. +# When set to +false+, the default: +# JSON.parse('/* comment */ {"a": 1, "a":2}') # unexpected character: '/' at line 1 column 1 (JSON::ParserError) # # When set to +true+, comments are ignored: -# JSON.parse('/* comment */ {"a": 1, "a":2}') # => {"a" => 2} -# -# When set to +false+, the future default: -# JSON.parse('/* comment */ {"a": 1, "a":2}') # unexpected character: '/' at line 1 column 1 (JSON::ParserError) +# JSON.parse('/* comment */ {"a": 1, "a":2} // more comment') # => {"a" => 2} # # --- # diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index f0df848aa..1f257e71d 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -543,16 +543,8 @@ def test_parse_comments assert_raise(ParserError) { parse('{} /*/', allow_comments: true) } assert_raise(ParserError) { parse('{} /x wrong comment', allow_comments: true) } assert_raise(ParserError) { parse('{} /', allow_comments: true) } - end - - def test_parse_comments_deprecation assert_equal({}, parse('/**/ {}', allow_comments: true)) assert_raise(ParserError) { parse('/**/ {}', allow_comments: false) } - if RUBY_ENGINE == 'ruby' - assert_deprecated_warning(/Encountered comment in JSON/) do - parse('/**/ {}') - end - end end def test_nesting From e8f34a8f2dd96bc2bce9ba30547b59091c8e31a7 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:28:40 +0200 Subject: [PATCH 421/506] Remove `default_options` singleton accessors --- CHANGES.md | 5 +- lib/json/common.rb | 108 +++++------------------- test/json/json_addition_test.rb | 6 -- test/json/json_common_interface_test.rb | 12 --- 4 files changed, 25 insertions(+), 106 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 77b8b428d..da1ef1ef1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,10 @@ * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. -* Remove `Kernel#j` and `Kernel#jj` +* Removed `Kernel#j` and `Kernel#jj` +* Removed `JSON.load_default_options`. +* Removed `JSON.unsafe_load_default_options`. +* Removed `JSON.dump_default_options`. ### 2026-07-31 (2.21.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index 92ee0e4c7..44be124e0 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -64,7 +64,6 @@ def create_additions_proc(opts) when String opts[:match_string]&.each do |pattern, klass| if match = pattern.match(object) - create_additions_warning if create_additions.nil? object = klass.json_create(object) break end @@ -79,7 +78,6 @@ def create_additions_proc(opts) end if klass.respond_to?(:json_creatable?) ? klass.json_creatable? : klass.respond_to?(:json_create) - create_additions_warning if create_additions.nil? object = klass.json_create(object) end end @@ -91,12 +89,6 @@ def create_additions_proc(opts) opts end - - def create_additions_warning - JSON.deprecation_warning "JSON.load implicit support for `create_additions: true` is deprecated " \ - "and will be removed in 3.0, use JSON.unsafe_load or explicitly " \ - "pass `create_additions: true`" - end end end @@ -205,27 +197,6 @@ def on_mixed_keys_hash(hash) end end end - - def deprecated_singleton_attr_accessor(*attrs) - args = RUBY_VERSION >= "3.0" ? ", category: :deprecated" : "" - attrs.each do |attr| - singleton_class.class_eval <<~RUBY - def #{attr} - warn "JSON.#{attr} is deprecated and will be removed in json 3.0.0", uplevel: 1 #{args} - @#{attr} - end - - def #{attr}=(val) - warn "JSON.#{attr}= is deprecated and will be removed in json 3.0.0", uplevel: 1 #{args} - @#{attr} = val - end - - def _#{attr} - @#{attr} - end - RUBY - end - end end # Sets create identifier, which is used to decide if the _json_create_ @@ -531,30 +502,6 @@ def pretty_generate(obj, opts = nil) State.generate(obj, options, nil) end - # Sets or returns default options for the JSON.unsafe_load method. - # Initially: - # opts = JSON.load_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} - deprecated_singleton_attr_accessor :unsafe_load_default_options - - @unsafe_load_default_options = { - :max_nesting => false, - :allow_nan => true, - :allow_blank => true, - :create_additions => true, - } - - # Sets or returns default options for the JSON.load method. - # Initially: - # opts = JSON.load_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true, :allow_blank=>true, :create_additions=>true} - deprecated_singleton_attr_accessor :load_default_options - - @load_default_options = { - :allow_nan => true, - :allow_blank => true, - :create_additions => nil, - } # :call-seq: # JSON.unsafe_load(source, options = {}) -> object # JSON.unsafe_load(source, proc = nil, options = {}) -> object @@ -581,7 +528,6 @@ def pretty_generate(obj, opts = nil) # See details below. # - Argument +opts+, if given, contains a \Hash of options for the parsing. # See {Parsing Options}[#module-JSON-label-Parsing+Options]. - # The default options can be changed via method JSON.unsafe_load_default_options=. # # --- # @@ -687,17 +633,18 @@ def pretty_generate(obj, opts = nil) # @attributes={"type"=>"Admin", "password"=>"0wn3d"}>} # def unsafe_load(source, proc = nil, options = nil) - opts = if options.nil? - if proc && proc.is_a?(Hash) - options, proc = proc, nil - options - else - _unsafe_load_default_options - end - else - _unsafe_load_default_options.merge(options) + opts = { + max_nesting: false, + allow_nan: true, + allow_blank: true, + create_additions: true, + } + if options.nil? && proc && proc.is_a?(Hash) + options, proc = proc, nil end + opts.merge!(options) unless options.nil? + unless source.is_a?(String) if source.respond_to? :to_str source = source.to_str @@ -752,7 +699,6 @@ def unsafe_load(source, proc = nil, options = nil) # See details below. # - Argument +opts+, if given, contains a \Hash of options for the parsing. # See {Parsing Options}[#module-JSON-label-Parsing+Options]. - # The default options can be changed via method JSON.load_default_options=. # # --- # @@ -863,16 +809,12 @@ def load(source, proc = nil, options = nil) proc = nil end - opts = if options.nil? - if proc && proc.is_a?(Hash) - options, proc = proc, nil - options - else - _load_default_options - end - else - _load_default_options.merge(options) - end + opts = { + allow_nan: true, + allow_blank: true, + } + + opts.merge!(options) unless options.nil? unless source.is_a?(String) if source.respond_to? :to_str @@ -896,16 +838,6 @@ def load(source, proc = nil, options = nil) parse(source, opts) end - # Sets or returns the default options for the JSON.dump method. - # Initially: - # opts = JSON.dump_default_options - # opts # => {:max_nesting=>false, :allow_nan=>true} - deprecated_singleton_attr_accessor :dump_default_options - @dump_default_options = { - :max_nesting => false, - :allow_nan => true, - } - # :call-seq: # JSON.dump(obj, io = nil, limit = nil) # @@ -954,9 +886,11 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end end - opts = JSON._dump_default_options - opts = opts.merge(:max_nesting => limit) if limit - opts = opts.merge(kwargs) if kwargs + opts = { + allow_nan: true, + max_nesting: limit, + } + opts.merge!(kwargs) if kwargs begin State.generate(obj, opts, anIO) diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index 4d8d18687..fedd1f1d9 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -150,12 +150,6 @@ def test_core assert_equal(/foo/i, JSON(JSON(/foo/i), :create_additions => true)) end - def test_deprecated_load_create_additions - assert_deprecated_warning(/use JSON\.unsafe_load/) do - JSON.load(JSON.dump(Time.now)) - end - end - def test_utc_datetime now = Time.now d = DateTime.parse(now.to_s) # usual case diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 37568b556..72b8f390a 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -271,12 +271,6 @@ def test_dump_in_io assert_equal JSON.dump(big_object), io.string end - def test_dump_should_modify_defaults - max_nesting = JSON._dump_default_options[:max_nesting] - dump([], StringIO.new, 10) - assert_equal max_nesting, JSON._dump_default_options[:max_nesting] - end - def test_JSON assert_equal @json, JSON(@hash) assert_equal @json, JSON(@hash_with_method_missing) @@ -315,12 +309,6 @@ def test_load_file_with_bad_default_external_encoding end end - def test_deprecated_dump_default_options - assert_deprecated_warning(/dump_default_options/) do - JSON.dump_default_options - end - end - private def with_external_encoding(encoding) From c1d784113c0e46aa4a81d1cacd6996596853ecca Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:34:31 +0200 Subject: [PATCH 422/506] Remove `JSON::State#[]` and `JSON::State#[]=` --- CHANGES.md | 3 ++- java/src/json/ext/GeneratorState.java | 24 ----------------- lib/json/common.rb | 17 ------------ lib/json/ext/generator/state.rb | 27 ------------------- lib/json/truffle_ruby/generator.rb | 22 ---------------- test/json/json_generator_test.rb | 37 --------------------------- 6 files changed, 2 insertions(+), 128 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index da1ef1ef1..48d96a615 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,10 +4,11 @@ * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. -* Removed `Kernel#j` and `Kernel#jj` +* Removed `Kernel#j` and `Kernel#jj`. * Removed `JSON.load_default_options`. * Removed `JSON.unsafe_load_default_options`. * Removed `JSON.dump_default_options`. +* Removed `JSON::State#[]` and `JSON::State#[]=`. ### 2026-07-31 (2.21.2) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index b5b28d2be..1086bf6d3 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -270,30 +270,6 @@ public IRubyObject generate(ThreadContext context, IRubyObject obj) { return generate(context, obj, context.nil); } - @JRubyMethod(name="[]") - public IRubyObject op_aref(ThreadContext context, IRubyObject vName) { - String name = vName.asJavaString(); - if (getMetaClass().isMethodBound(name, true)) { - return send(context, vName, Block.NULL_BLOCK); - } else { - IRubyObject value = getInstanceVariables().getInstanceVariable("@" + name); - return value == null ? context.nil : value; - } - } - - @JRubyMethod(name="[]=") - public IRubyObject op_aset(ThreadContext context, IRubyObject vName, IRubyObject value) { - checkFrozen(); - String name = vName.asJavaString(); - String nameWriter = name + "="; - if (getMetaClass().isMethodBound(nameWriter, true)) { - return send(context, context.runtime.newString(nameWriter), value, Block.NULL_BLOCK); - } else { - getInstanceVariables().setInstanceVariable("@" + name, value); - } - return context.nil; - } - public ByteList getIndent() { return indent; } diff --git a/lib/json/common.rb b/lib/json/common.rb index 44be124e0..5fea1c532 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -93,23 +93,6 @@ def create_additions_proc(opts) end class << self - def deprecation_warning(message, uplevel = 3) # :nodoc: - gem_root = File.expand_path("..", __dir__) + "/" - caller_locations(uplevel, 10).each do |frame| - if frame.path.nil? || frame.path.start_with?(gem_root) || frame.path.end_with?("/truffle/cext_ruby.rb", ".c") - uplevel += 1 - else - break - end - end - - if RUBY_VERSION >= "3.0" - warn(message, uplevel: uplevel, category: :deprecated) - else - warn(message, uplevel: uplevel) - end - end - # :call-seq: # JSON[object] -> new_array or new_string # diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index 3c1d2fb94..a9578b9bd 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -71,33 +71,6 @@ def to_h end alias_method :to_hash, :to_h - - # call-seq: [](name) - # - # Returns the value returned by method +name+. - def [](name) - ::JSON.deprecation_warning("JSON::State#[] is deprecated and will be removed in json 3.0.0") - - if respond_to?(name) - __send__(name) - else - instance_variable_get("@#{name}") if - instance_variables.include?("@#{name}".to_sym) # avoid warning - end - end - - # call-seq: []=(name, value) - # - # Sets the attribute name to value. - def []=(name, value) - ::JSON.deprecation_warning("JSON::State#[]= is deprecated and will be removed in json 3.0.0") - - if respond_to?(name_writer = "#{name}=") - __send__ name_writer, value - else - instance_variable_set "@#{name}", value - end - end end end end diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 450684a50..495167be2 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -475,28 +475,6 @@ def generate(obj, anIO = nil) end buf << '"' end - - # Return the value returned by method +name+. - def [](name) - ::JSON.deprecation_warning("JSON::State#[] is deprecated and will be removed in json 3.0.0") - - if respond_to?(name) - __send__(name) - else - instance_variable_get("@#{name}") if - instance_variables.include?("@#{name}".to_sym) # avoid warning - end - end - - def []=(name, value) - ::JSON.deprecation_warning("JSON::State#[]= is deprecated and will be removed in json 3.0.0") - - if respond_to?(name_writer = "#{name}=") - __send__ name_writer, value - else - instance_variable_set "@#{name}", value - end - end end module GeneratorMethods diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 94c48a604..937f71651 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -287,10 +287,6 @@ def test_states json = generate({1=>2}, nil) assert_equal('{"1":2}', json) s = JSON.state.new - assert s.check_circular? - assert_deprecated_warning(/JSON::State/) do - assert s[:check_circular?] - end h = { 1=>2 } h[3] = h assert_raise(JSON::NestingError) { generate(h) } @@ -299,10 +295,6 @@ def test_states a = [ 1, 2 ] a << a assert_raise(JSON::NestingError) { generate(a, s) } - assert s.check_circular? - assert_deprecated_warning(/JSON::State/) do - assert s[:check_circular?] - end end def test_falsy_state @@ -578,35 +570,6 @@ def to_s bignum.class.define_method(:to_s, original_to_s) if original_to_s end - def test_hash_likeness_set_symbol - assert_deprecated_warning(/JSON::State/) do - state = JSON.state.new - assert_equal nil, state[:foo] - assert_equal nil.class, state[:foo].class - assert_equal nil, state['foo'] - state[:foo] = :bar - assert_equal :bar, state[:foo] - assert_equal :bar, state['foo'] - state_hash = state.to_hash - assert_kind_of Hash, state_hash - assert_equal :bar, state_hash[:foo] - end - end - - def test_hash_likeness_set_string - assert_deprecated_warning(/JSON::State/) do - state = JSON.state.new - assert_equal nil, state[:foo] - assert_equal nil, state['foo'] - state['foo'] = :bar - assert_equal :bar, state[:foo] - assert_equal :bar, state['foo'] - state_hash = state.to_hash - assert_kind_of Hash, state_hash - assert_equal :bar, state_hash[:foo] - end - end - def test_json_state_to_h_roundtrip state = JSON.state.new assert_equal state.to_h, JSON.state.new(state.to_h).to_h From 545a2ed894002efb7253141e4b5f4dd22e99d6b0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:36:55 +0200 Subject: [PATCH 423/506] Remove deprecated JSON.dump aliases --- CHANGES.md | 4 ++++ lib/json/common.rb | 42 ------------------------------------------ 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 48d96a615..483e8983c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,10 @@ * Removed `JSON.unsafe_load_default_options`. * Removed `JSON.dump_default_options`. * Removed `JSON::State#[]` and `JSON::State#[]=`. +* Removed `JSON.unparse`. +* Removed `JSON.fast_unparse`. +* Removed `JSON.pretty_unparse`. +* Removed `JSON.restore`. ### 2026-07-31 (2.21.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index 5fea1c532..5a1b4be23 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -883,48 +883,6 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end # :stopdoc: - # All these were meant to be deprecated circa 2009, but were just set as undocumented - # so usage still exist in the wild. - def unparse(...) - if RUBY_VERSION >= "3.0" - warn "JSON.unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated - else - warn "JSON.unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 - end - generate(...) - end - module_function :unparse - - def fast_unparse(...) - if RUBY_VERSION >= "3.0" - warn "JSON.fast_unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated - else - warn "JSON.fast_unparse is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 - end - generate(...) - end - module_function :fast_unparse - - def pretty_unparse(...) - if RUBY_VERSION >= "3.0" - warn "JSON.pretty_unparse is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1, category: :deprecated - else - warn "JSON.pretty_unparse is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1 - end - pretty_generate(...) - end - module_function :fast_unparse - - def restore(...) - if RUBY_VERSION >= "3.0" - warn "JSON.restore is deprecated and will be removed in json 3.0.0, just use JSON.load", uplevel: 1, category: :deprecated - else - warn "JSON.restore is deprecated and will be removed in json 3.0.0, just use JSON.load", uplevel: 1 - end - load(...) - end - module_function :restore - class << self private From dfffed72b33c5b376c3b878beb2850dded2f4950 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:38:23 +0200 Subject: [PATCH 424/506] Remove PRETTY_STATE_PROTOTYPE --- CHANGES.md | 1 + lib/json/common.rb | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 483e8983c..a2c5e137c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ * Removed `JSON.fast_unparse`. * Removed `JSON.pretty_unparse`. * Removed `JSON.restore`. +* Removed `JSON::PRETTY_STATE_PROTOTYPE`. ### 2026-07-31 (2.21.2) diff --git a/lib/json/common.rb b/lib/json/common.rb index 5a1b4be23..a030f0464 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -882,26 +882,6 @@ def dump(obj, anIO = nil, limit = nil, kwargs = nil) end end - # :stopdoc: - class << self - private - - def const_missing(const_name) - case const_name - when :PRETTY_STATE_PROTOTYPE - if RUBY_VERSION >= "3.0" - warn "JSON::PRETTY_STATE_PROTOTYPE is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1, category: :deprecated - else - warn "JSON::PRETTY_STATE_PROTOTYPE is deprecated and will be removed in json 3.0.0, just use JSON.pretty_generate", uplevel: 1 - end - state.new(PRETTY_GENERATE_OPTIONS) - else - super - end - end - end - # :startdoc: - # JSON::Coder holds a parser and generator configuration. # # module MyApp From 1f65fe1c2da09e0ca9458323792aa7b24b944628 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:56:36 +0200 Subject: [PATCH 425/506] Remove `create_additions` option --- CHANGES.md | 1 + README.md | 74 +------- benchmark/standalone.rb | 2 +- lib/json.rb | 240 ------------------------ lib/json/add/bigdecimal.rb | 58 ------ lib/json/add/complex.rb | 51 ----- lib/json/add/core.rb | 13 -- lib/json/add/date.rb | 54 ------ lib/json/add/date_time.rb | 67 ------- lib/json/add/exception.rb | 49 ----- lib/json/add/ostruct.rb | 54 ------ lib/json/add/range.rb | 54 ------ lib/json/add/rational.rb | 49 ----- lib/json/add/regexp.rb | 48 ----- lib/json/add/set.rb | 48 ----- lib/json/add/string.rb | 35 ---- lib/json/add/struct.rb | 52 ----- lib/json/add/symbol.rb | 52 ----- lib/json/add/time.rb | 52 ----- lib/json/common.rb | 72 +------ lib/json/generic_object.rb | 7 - test/json/json_addition_test.rb | 186 ------------------ test/json/json_common_interface_test.rb | 8 - test/json/json_generic_object_test.rb | 40 +--- test/json/json_parser_test.rb | 29 --- test/json/json_string_matching_test.rb | 38 ---- 26 files changed, 10 insertions(+), 1423 deletions(-) delete mode 100644 lib/json/add/bigdecimal.rb delete mode 100644 lib/json/add/complex.rb delete mode 100644 lib/json/add/core.rb delete mode 100644 lib/json/add/date.rb delete mode 100644 lib/json/add/date_time.rb delete mode 100644 lib/json/add/exception.rb delete mode 100644 lib/json/add/ostruct.rb delete mode 100644 lib/json/add/range.rb delete mode 100644 lib/json/add/rational.rb delete mode 100644 lib/json/add/regexp.rb delete mode 100644 lib/json/add/set.rb delete mode 100644 lib/json/add/string.rb delete mode 100644 lib/json/add/struct.rb delete mode 100644 lib/json/add/symbol.rb delete mode 100644 lib/json/add/time.rb delete mode 100644 test/json/json_addition_test.rb delete mode 100644 test/json/json_string_matching_test.rb diff --git a/CHANGES.md b/CHANGES.md index a2c5e137c..382428b19 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ * Removed `JSON.pretty_unparse`. * Removed `JSON.restore`. * Removed `JSON::PRETTY_STATE_PROTOTYPE`. +* Removed the insecure `create_additions` option. ### 2026-07-31 (2.21.2) diff --git a/README.md b/README.md index 5e6ffe48f..79a8c3f3e 100644 --- a/README.md +++ b/README.md @@ -143,79 +143,7 @@ posts_json.map! { |post_json| JSON::Fragment.new(post_json) } JSON.generate({ posts: posts_json, count: posts_json.count }) ``` -## Round-tripping arbitrary types - -> [!CAUTION] -> You should never use `JSON.unsafe_load` nor `JSON.parse(str, create_additions: true)` to parse untrusted user input, -> as it can lead to remote code execution vulnerabilities. - -To create a JSON document from a ruby data structure, you can call -`JSON.generate` like that: - -```ruby -json = JSON.generate [1, 2, {"a"=>3.141}, false, true, nil, 4..10] -# => "[1,2,{\"a\":3.141},false,true,null,\"4..10\"]" -``` - -To get back a ruby data structure from a JSON document, you have to call -JSON.parse on it: - -```ruby -JSON.parse json -# => [1, 2, {"a"=>3.141}, false, true, nil, "4..10"] -``` - -Note, that the range from the original data structure is a simple -string now. The reason for this is, that JSON doesn't support ranges -or arbitrary classes. In this case the json library falls back to call -`Object#to_json`, which is the same as `#to_s.to_json`. - -It's possible to add JSON support serialization to arbitrary classes by -simply implementing a more specialized version of the `#to_json method`, that -should return a JSON object (a hash converted to JSON with `#to_json`) like -this (don't forget the `*a` for all the arguments): - -```ruby -class Range - def to_json(*a) - { - 'json_class' => self.class.name, # = 'Range' - 'data' => [ first, last, exclude_end? ] - }.to_json(*a) - end -end -``` - -The hash key `json_class` is the class, that will be asked to deserialise the -JSON representation later. In this case it's `Range`, but any namespace of -the form `A::B` or `::A::B` will do. All other keys are arbitrary and can be -used to store the necessary data to configure the object to be deserialised. - -If the key `json_class` is found in a JSON object, the JSON parser checks -if the given class responds to the `json_create` class method. If so, it is -called with the JSON object converted to a Ruby hash. So a range can -be deserialised by implementing `Range.json_create` like this: - -```ruby -class Range - def self.json_create(o) - new(*o['data']) - end -end -``` - -Now it possible to serialise/deserialise ranges as well: - -```ruby -json = JSON.generate [1, 2, {"a"=>3.141}, false, true, nil, 4..10] -# => "[1,2,{\"a\":3.141},false,true,null,{\"json_class\":\"Range\",\"data\":[4,10,false]}]" -JSON.parse json -# => [1, 2, {"a"=>3.141}, false, true, nil, 4..10] -json = JSON.generate [1, 2, {"a"=>3.141}, false, true, nil, 4..10] -# => "[1,2,{\"a\":3.141},false,true,null,{\"json_class\":\"Range\",\"data\":[4,10,false]}]" -JSON.unsafe_load json -# => [1, 2, {"a"=>3.141}, false, true, nil, 4..10] -``` +## Pretty Printing `JSON.generate` always creates the shortest possible string representation of a ruby data structure in one line. This is good for data storage or network diff --git a/benchmark/standalone.rb b/benchmark/standalone.rb index deeb45dce..ebe759c5f 100644 --- a/benchmark/standalone.rb +++ b/benchmark/standalone.rb @@ -32,7 +32,7 @@ JSON.dump(obj) end else - x.report('JSON.load(str)') do # max_nesting: false, allow_nan: true, allow_blank: true, create_additions: true + x.report('JSON.load(str)') do # max_nesting: false, allow_nan: true, allow_blank: true JSON.load(str) end end diff --git a/lib/json.rb b/lib/json.rb index 90b4b3c91..9c77544f1 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -259,11 +259,6 @@ # ruby = JSON.parse(source, {array_class: Set}) # ruby # => # # -# --- -# -# Option +create_additions+ (boolean) specifies whether to use \JSON additions in parsing. -# See {\JSON Additions}[#module-JSON-label-JSON+Additions]. -# # === Generating \JSON # # To generate a Ruby \String containing \JSON data, @@ -450,241 +445,6 @@ # } # } # -# == \JSON Additions -# -# Note that JSON Additions must only be used with trusted data, and is -# deprecated. -# -# When you "round trip" a non-\String object from Ruby to \JSON and back, -# you have a new \String, instead of the object you began with: -# ruby0 = Range.new(0, 2) -# json = JSON.generate(ruby0) -# json # => '0..2"' -# ruby1 = JSON.parse(json) -# ruby1 # => '0..2' -# ruby1.class # => String -# -# You can use \JSON _additions_ to preserve the original object. -# The addition is an extension of a ruby class, so that: -# - \JSON.generate stores more information in the \JSON string. -# - \JSON.parse, called with option +create_additions+, -# uses that information to create a proper Ruby object. -# -# This example shows a \Range being generated into \JSON -# and parsed back into Ruby, both without and with -# the addition for \Range: -# ruby = Range.new(0, 2) -# # This passage does not use the addition for Range. -# json0 = JSON.generate(ruby) -# ruby0 = JSON.parse(json0) -# # This passage uses the addition for Range. -# require 'json/add/range' -# json1 = JSON.generate(ruby) -# ruby1 = JSON.parse(json1, create_additions: true) -# # Make a nice display. -# display = <<~EOT -# Generated JSON: -# Without addition: #{json0} (#{json0.class}) -# With addition: #{json1} (#{json1.class}) -# Parsed JSON: -# Without addition: #{ruby0.inspect} (#{ruby0.class}) -# With addition: #{ruby1.inspect} (#{ruby1.class}) -# EOT -# puts display -# -# This output shows the different results: -# Generated JSON: -# Without addition: "0..2" (String) -# With addition: {"json_class":"Range","a":[0,2,false]} (String) -# Parsed JSON: -# Without addition: "0..2" (String) -# With addition: 0..2 (Range) -# -# The \JSON module includes additions for certain classes. -# You can also craft custom additions. -# See {Custom \JSON Additions}[#module-JSON-label-Custom+JSON+Additions]. -# -# === Built-in Additions -# -# The \JSON module includes additions for certain classes. -# To use an addition, +require+ its source: -# - BigDecimal: require 'json/add/bigdecimal' -# - Complex: require 'json/add/complex' -# - Date: require 'json/add/date' -# - DateTime: require 'json/add/date_time' -# - Exception: require 'json/add/exception' -# - OpenStruct: require 'json/add/ostruct' -# - Range: require 'json/add/range' -# - Rational: require 'json/add/rational' -# - Regexp: require 'json/add/regexp' -# - Set: require 'json/add/set' -# - Struct: require 'json/add/struct' -# - Symbol: require 'json/add/symbol' -# - Time: require 'json/add/time' -# -# To reduce punctuation clutter, the examples below -# show the generated \JSON via +puts+, rather than the usual +inspect+, -# -# \BigDecimal: -# require 'json/add/bigdecimal' -# ruby0 = BigDecimal(0) # 0.0 -# json = JSON.generate(ruby0) # {"json_class":"BigDecimal","b":"27:0.0"} -# ruby1 = JSON.parse(json, create_additions: true) # 0.0 -# ruby1.class # => BigDecimal -# -# \Complex: -# require 'json/add/complex' -# ruby0 = Complex(1+0i) # 1+0i -# json = JSON.generate(ruby0) # {"json_class":"Complex","r":1,"i":0} -# ruby1 = JSON.parse(json, create_additions: true) # 1+0i -# ruby1.class # Complex -# -# \Date: -# require 'json/add/date' -# ruby0 = Date.today # 2020-05-02 -# json = JSON.generate(ruby0) # {"json_class":"Date","y":2020,"m":5,"d":2,"sg":2299161.0} -# ruby1 = JSON.parse(json, create_additions: true) # 2020-05-02 -# ruby1.class # Date -# -# \DateTime: -# require 'json/add/date_time' -# ruby0 = DateTime.now # 2020-05-02T10:38:13-05:00 -# json = JSON.generate(ruby0) # {"json_class":"DateTime","y":2020,"m":5,"d":2,"H":10,"M":38,"S":13,"of":"-5/24","sg":2299161.0} -# ruby1 = JSON.parse(json, create_additions: true) # 2020-05-02T10:38:13-05:00 -# ruby1.class # DateTime -# -# \Exception (and its subclasses including \RuntimeError): -# require 'json/add/exception' -# ruby0 = Exception.new('A message') # A message -# json = JSON.generate(ruby0) # {"json_class":"Exception","m":"A message","b":null} -# ruby1 = JSON.parse(json, create_additions: true) # A message -# ruby1.class # Exception -# ruby0 = RuntimeError.new('Another message') # Another message -# json = JSON.generate(ruby0) # {"json_class":"RuntimeError","m":"Another message","b":null} -# ruby1 = JSON.parse(json, create_additions: true) # Another message -# ruby1.class # RuntimeError -# -# \OpenStruct: -# require 'json/add/ostruct' -# ruby0 = OpenStruct.new(name: 'Matz', language: 'Ruby') # # -# json = JSON.generate(ruby0) # {"json_class":"OpenStruct","t":{"name":"Matz","language":"Ruby"}} -# ruby1 = JSON.parse(json, create_additions: true) # # -# ruby1.class # OpenStruct -# -# \Range: -# require 'json/add/range' -# ruby0 = Range.new(0, 2) # 0..2 -# json = JSON.generate(ruby0) # {"json_class":"Range","a":[0,2,false]} -# ruby1 = JSON.parse(json, create_additions: true) # 0..2 -# ruby1.class # Range -# -# \Rational: -# require 'json/add/rational' -# ruby0 = Rational(1, 3) # 1/3 -# json = JSON.generate(ruby0) # {"json_class":"Rational","n":1,"d":3} -# ruby1 = JSON.parse(json, create_additions: true) # 1/3 -# ruby1.class # Rational -# -# \Regexp: -# require 'json/add/regexp' -# ruby0 = Regexp.new('foo') # (?-mix:foo) -# json = JSON.generate(ruby0) # {"json_class":"Regexp","o":0,"s":"foo"} -# ruby1 = JSON.parse(json, create_additions: true) # (?-mix:foo) -# ruby1.class # Regexp -# -# \Set: -# require 'json/add/set' -# ruby0 = Set.new([0, 1, 2]) # # -# json = JSON.generate(ruby0) # {"json_class":"Set","a":[0,1,2]} -# ruby1 = JSON.parse(json, create_additions: true) # # -# ruby1.class # Set -# -# \Struct: -# require 'json/add/struct' -# Customer = Struct.new(:name, :address) # Customer -# ruby0 = Customer.new("Dave", "123 Main") # # -# json = JSON.generate(ruby0) # {"json_class":"Customer","v":["Dave","123 Main"]} -# ruby1 = JSON.parse(json, create_additions: true) # # -# ruby1.class # Customer -# -# \Symbol: -# require 'json/add/symbol' -# ruby0 = :foo # foo -# json = JSON.generate(ruby0) # {"json_class":"Symbol","s":"foo"} -# ruby1 = JSON.parse(json, create_additions: true) # foo -# ruby1.class # Symbol -# -# \Time: -# require 'json/add/time' -# ruby0 = Time.now # 2020-05-02 11:28:26 -0500 -# json = JSON.generate(ruby0) # {"json_class":"Time","s":1588436906,"n":840560000} -# ruby1 = JSON.parse(json, create_additions: true) # 2020-05-02 11:28:26 -0500 -# ruby1.class # Time -# -# -# === Custom \JSON Additions -# -# In addition to the \JSON additions provided, -# you can craft \JSON additions of your own, -# either for Ruby built-in classes or for user-defined classes. -# -# Here's a user-defined class +Foo+: -# class Foo -# attr_accessor :bar, :baz -# def initialize(bar, baz) -# self.bar = bar -# self.baz = baz -# end -# end -# -# Here's the \JSON addition for it: -# # Extend class Foo with JSON addition. -# class Foo -# # Serialize Foo object with its class name and arguments -# def to_json(*args) -# { -# JSON.create_id => self.class.name, -# 'a' => [ bar, baz ] -# }.to_json(*args) -# end -# # Deserialize JSON string by constructing new Foo object with arguments. -# def self.json_create(object) -# new(*object['a']) -# end -# end -# -# Demonstration: -# require 'json' -# # This Foo object has no custom addition. -# foo0 = Foo.new(0, 1) -# json0 = JSON.generate(foo0) -# obj0 = JSON.parse(json0) -# # Lood the custom addition. -# require_relative 'foo_addition' -# # This foo has the custom addition. -# foo1 = Foo.new(0, 1) -# json1 = JSON.generate(foo1) -# obj1 = JSON.parse(json1, create_additions: true) -# # Make a nice display. -# display = <<~EOT -# Generated JSON: -# Without custom addition: #{json0} (#{json0.class}) -# With custom addition: #{json1} (#{json1.class}) -# Parsed JSON: -# Without custom addition: #{obj0.inspect} (#{obj0.class}) -# With custom addition: #{obj1.inspect} (#{obj1.class}) -# EOT -# puts display -# -# Output: -# -# Generated JSON: -# Without custom addition: "#" (String) -# With custom addition: {"json_class":"Foo","a":[0,1]} (String) -# Parsed JSON: -# Without custom addition: "#" (String) -# With custom addition: # (Foo) -# module JSON require 'json/version' require 'json/ext' diff --git a/lib/json/add/bigdecimal.rb b/lib/json/add/bigdecimal.rb deleted file mode 100644 index 5dbc12c07..000000000 --- a/lib/json/add/bigdecimal.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end -begin - require 'bigdecimal' -rescue LoadError -end - -class BigDecimal - - # See #as_json. - def self.json_create(object) - BigDecimal._load object['b'] - end - - # Methods BigDecimal#as_json and +BigDecimal.json_create+ may be used - # to serialize and deserialize a \BigDecimal object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method BigDecimal#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/bigdecimal' - # x = BigDecimal(2).as_json # => {"json_class"=>"BigDecimal", "b"=>"27:0.2e1"} - # y = BigDecimal(2.0, 4).as_json # => {"json_class"=>"BigDecimal", "b"=>"36:0.2e1"} - # z = BigDecimal(Complex(2, 0)).as_json # => {"json_class"=>"BigDecimal", "b"=>"27:0.2e1"} - # - # \Method +JSON.create+ deserializes such a hash, returning a \BigDecimal object: - # - # BigDecimal.json_create(x) # => 0.2e1 - # BigDecimal.json_create(y) # => 0.2e1 - # BigDecimal.json_create(z) # => 0.2e1 - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'b' => _dump.force_encoding(Encoding::UTF_8), - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/bigdecimal' - # puts BigDecimal(2).to_json - # puts BigDecimal(2.0, 4).to_json - # puts BigDecimal(Complex(2, 0)).to_json - # - # Output: - # - # {"json_class":"BigDecimal","b":"27:0.2e1"} - # {"json_class":"BigDecimal","b":"36:0.2e1"} - # {"json_class":"BigDecimal","b":"27:0.2e1"} - # - def to_json(*args) - as_json.to_json(*args) - end -end if defined?(::BigDecimal) diff --git a/lib/json/add/complex.rb b/lib/json/add/complex.rb deleted file mode 100644 index a69002eff..000000000 --- a/lib/json/add/complex.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Complex - - # See #as_json. - def self.json_create(object) - Complex(object['r'], object['i']) - end - - # Methods Complex#as_json and +Complex.json_create+ may be used - # to serialize and deserialize a \Complex object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Complex#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/complex' - # x = Complex(2).as_json # => {"json_class"=>"Complex", "r"=>2, "i"=>0} - # y = Complex(2.0, 4).as_json # => {"json_class"=>"Complex", "r"=>2.0, "i"=>4} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Complex object: - # - # Complex.json_create(x) # => (2+0i) - # Complex.json_create(y) # => (2.0+4i) - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'r' => real, - 'i' => imag, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/complex' - # puts Complex(2).to_json - # puts Complex(2.0, 4).to_json - # - # Output: - # - # {"json_class":"Complex","r":2,"i":0} - # {"json_class":"Complex","r":2.0,"i":4} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/core.rb b/lib/json/add/core.rb deleted file mode 100644 index 61ff45421..000000000 --- a/lib/json/add/core.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true -# This file requires the implementations of ruby core's custom objects for -# serialisation/deserialisation. - -require 'json/add/date' -require 'json/add/date_time' -require 'json/add/exception' -require 'json/add/range' -require 'json/add/regexp' -require 'json/add/string' -require 'json/add/struct' -require 'json/add/symbol' -require 'json/add/time' diff --git a/lib/json/add/date.rb b/lib/json/add/date.rb deleted file mode 100644 index 66965d491..000000000 --- a/lib/json/add/date.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end -require 'date' - -class Date - - # See #as_json. - def self.json_create(object) - civil(*object.values_at('y', 'm', 'd', 'sg')) - end - - alias start sg unless method_defined?(:start) - - # Methods Date#as_json and +Date.json_create+ may be used - # to serialize and deserialize a \Date object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Date#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/date' - # x = Date.today.as_json - # # => {"json_class"=>"Date", "y"=>2023, "m"=>11, "d"=>21, "sg"=>2299161.0} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Date object: - # - # Date.json_create(x) - # # => # - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'y' => year, - 'm' => month, - 'd' => day, - 'sg' => start, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/date' - # puts Date.today.to_json - # - # Output: - # - # {"json_class":"Date","y":2023,"m":11,"d":21,"sg":2299161.0} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/date_time.rb b/lib/json/add/date_time.rb deleted file mode 100644 index 569f6ec17..000000000 --- a/lib/json/add/date_time.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end -require 'date' - -class DateTime - - # See #as_json. - def self.json_create(object) - args = object.values_at('y', 'm', 'd', 'H', 'M', 'S') - of_a, of_b = object['of'].split('/') - if of_b and of_b != '0' - args << Rational(of_a.to_i, of_b.to_i) - else - args << of_a - end - args << object['sg'] - civil(*args) - end - - alias start sg unless method_defined?(:start) - - # Methods DateTime#as_json and +DateTime.json_create+ may be used - # to serialize and deserialize a \DateTime object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method DateTime#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/datetime' - # x = DateTime.now.as_json - # # => {"json_class"=>"DateTime", "y"=>2023, "m"=>11, "d"=>21, "sg"=>2299161.0} - # - # \Method +JSON.create+ deserializes such a hash, returning a \DateTime object: - # - # DateTime.json_create(x) # BUG? Raises Date::Error "invalid date" - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'y' => year, - 'm' => month, - 'd' => day, - 'H' => hour, - 'M' => min, - 'S' => sec, - 'of' => offset.to_s, - 'sg' => start, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/datetime' - # puts DateTime.now.to_json - # - # Output: - # - # {"json_class":"DateTime","y":2023,"m":11,"d":21,"sg":2299161.0} - # - def to_json(*args) - as_json.to_json(*args) - end -end - - diff --git a/lib/json/add/exception.rb b/lib/json/add/exception.rb deleted file mode 100644 index 5338ff83d..000000000 --- a/lib/json/add/exception.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Exception - - # See #as_json. - def self.json_create(object) - result = new(object['m']) - result.set_backtrace object['b'] - result - end - - # Methods Exception#as_json and +Exception.json_create+ may be used - # to serialize and deserialize a \Exception object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Exception#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/exception' - # x = Exception.new('Foo').as_json # => {"json_class"=>"Exception", "m"=>"Foo", "b"=>nil} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Exception object: - # - # Exception.json_create(x) # => # - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'm' => message, - 'b' => backtrace, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/exception' - # puts Exception.new('Foo').to_json - # - # Output: - # - # {"json_class":"Exception","m":"Foo","b":null} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/ostruct.rb b/lib/json/add/ostruct.rb deleted file mode 100644 index 534403b78..000000000 --- a/lib/json/add/ostruct.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end -begin - require 'ostruct' -rescue LoadError -end - -class OpenStruct - - # See #as_json. - def self.json_create(object) - new(object['t'] || object[:t]) - end - - # Methods OpenStruct#as_json and +OpenStruct.json_create+ may be used - # to serialize and deserialize a \OpenStruct object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method OpenStruct#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/ostruct' - # x = OpenStruct.new('name' => 'Rowdy', :age => nil).as_json - # # => {"json_class"=>"OpenStruct", "t"=>{:name=>'Rowdy', :age=>nil}} - # - # \Method +JSON.create+ deserializes such a hash, returning a \OpenStruct object: - # - # OpenStruct.json_create(x) - # # => # - # - def as_json(*) - klass = self.class.name - klass.to_s.empty? and raise JSON::JSONError, "Only named structs are supported!" - { - JSON.create_id => klass, - 't' => table, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/ostruct' - # puts OpenStruct.new('name' => 'Rowdy', :age => nil).to_json - # - # Output: - # - # {"json_class":"OpenStruct","t":{'name':'Rowdy',"age":null}} - # - def to_json(*args) - as_json.to_json(*args) - end -end if defined?(::OpenStruct) diff --git a/lib/json/add/range.rb b/lib/json/add/range.rb deleted file mode 100644 index eb4b29a8e..000000000 --- a/lib/json/add/range.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Range - - # See #as_json. - def self.json_create(object) - new(*object['a']) - end - - # Methods Range#as_json and +Range.json_create+ may be used - # to serialize and deserialize a \Range object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Range#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/range' - # x = (1..4).as_json # => {"json_class"=>"Range", "a"=>[1, 4, false]} - # y = (1...4).as_json # => {"json_class"=>"Range", "a"=>[1, 4, true]} - # z = ('a'..'d').as_json # => {"json_class"=>"Range", "a"=>["a", "d", false]} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Range object: - # - # Range.json_create(x) # => 1..4 - # Range.json_create(y) # => 1...4 - # Range.json_create(z) # => "a".."d" - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'a' => [ first, last, exclude_end? ] - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/range' - # puts (1..4).to_json - # puts (1...4).to_json - # puts ('a'..'d').to_json - # - # Output: - # - # {"json_class":"Range","a":[1,4,false]} - # {"json_class":"Range","a":[1,4,true]} - # {"json_class":"Range","a":["a","d",false]} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/rational.rb b/lib/json/add/rational.rb deleted file mode 100644 index 1eb231474..000000000 --- a/lib/json/add/rational.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Rational - - # See #as_json. - def self.json_create(object) - Rational(object['n'], object['d']) - end - - # Methods Rational#as_json and +Rational.json_create+ may be used - # to serialize and deserialize a \Rational object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Rational#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/rational' - # x = Rational(2, 3).as_json - # # => {"json_class"=>"Rational", "n"=>2, "d"=>3} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Rational object: - # - # Rational.json_create(x) - # # => (2/3) - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'n' => numerator, - 'd' => denominator, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/rational' - # puts Rational(2, 3).to_json - # - # Output: - # - # {"json_class":"Rational","n":2,"d":3} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/regexp.rb b/lib/json/add/regexp.rb deleted file mode 100644 index f033dd1de..000000000 --- a/lib/json/add/regexp.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Regexp - - # See #as_json. - def self.json_create(object) - new(object['s'], object['o']) - end - - # Methods Regexp#as_json and +Regexp.json_create+ may be used - # to serialize and deserialize a \Regexp object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Regexp#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/regexp' - # x = /foo/.as_json - # # => {"json_class"=>"Regexp", "o"=>0, "s"=>"foo"} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Regexp object: - # - # Regexp.json_create(x) # => /foo/ - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'o' => options, - 's' => source, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/regexp' - # puts /foo/.to_json - # - # Output: - # - # {"json_class":"Regexp","o":0,"s":"foo"} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/set.rb b/lib/json/add/set.rb deleted file mode 100644 index c521d8b90..000000000 --- a/lib/json/add/set.rb +++ /dev/null @@ -1,48 +0,0 @@ -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end -defined?(::Set) or require 'set' - -class Set - - # See #as_json. - def self.json_create(object) - new object['a'] - end - - # Methods Set#as_json and +Set.json_create+ may be used - # to serialize and deserialize a \Set object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Set#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/set' - # x = Set.new(%w/foo bar baz/).as_json - # # => {"json_class"=>"Set", "a"=>["foo", "bar", "baz"]} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Set object: - # - # Set.json_create(x) # => # - # - def as_json(*) - { - JSON.create_id => self.class.name, - 'a' => to_a, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/set' - # puts Set.new(%w/foo bar baz/).to_json - # - # Output: - # - # {"json_class":"Set","a":["foo","bar","baz"]} - # - def to_json(*args) - as_json.to_json(*args) - end -end - diff --git a/lib/json/add/string.rb b/lib/json/add/string.rb deleted file mode 100644 index 9c3bde27f..000000000 --- a/lib/json/add/string.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class String - # call-seq: json_create(o) - # - # Raw Strings are JSON Objects (the raw bytes are stored in an array for the - # key "raw"). The Ruby String can be created by this class method. - def self.json_create(object) - object["raw"].pack("C*") - end - - # call-seq: to_json_raw_object() - # - # This method creates a raw object hash, that can be nested into - # other data structures and will be generated as a raw string. This - # method should be used, if you want to convert raw strings to JSON - # instead of UTF-8 strings, e. g. binary data. - def to_json_raw_object - { - JSON.create_id => self.class.name, - "raw" => unpack("C*"), - } - end - - # call-seq: to_json_raw(*args) - # - # This method creates a JSON text from the result of a call to - # to_json_raw_object of this String. - def to_json_raw(...) - to_json_raw_object.to_json(...) - end -end diff --git a/lib/json/add/struct.rb b/lib/json/add/struct.rb deleted file mode 100644 index 98c38d326..000000000 --- a/lib/json/add/struct.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Struct - - # See #as_json. - def self.json_create(object) - new(*object['v']) - end - - # Methods Struct#as_json and +Struct.json_create+ may be used - # to serialize and deserialize a \Struct object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Struct#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/struct' - # Customer = Struct.new('Customer', :name, :address, :zip) - # x = Struct::Customer.new.as_json - # # => {"json_class"=>"Struct::Customer", "v"=>[nil, nil, nil]} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Struct object: - # - # Struct::Customer.json_create(x) - # # => # - # - def as_json(*) - klass = self.class.name - klass.to_s.empty? and raise JSON::JSONError, "Only named structs are supported!" - { - JSON.create_id => klass, - 'v' => values, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/struct' - # Customer = Struct.new('Customer', :name, :address, :zip) - # puts Struct::Customer.new.to_json - # - # Output: - # - # {"json_class":"Struct","t":{'name':'Rowdy',"age":null}} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/add/symbol.rb b/lib/json/add/symbol.rb deleted file mode 100644 index 20dd59485..000000000 --- a/lib/json/add/symbol.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Symbol - - # Methods Symbol#as_json and +Symbol.json_create+ may be used - # to serialize and deserialize a \Symbol object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Symbol#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/symbol' - # x = :foo.as_json - # # => {"json_class"=>"Symbol", "s"=>"foo"} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Symbol object: - # - # Symbol.json_create(x) # => :foo - # - def as_json(*) - { - JSON.create_id => self.class.name, - 's' => to_s, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/symbol' - # puts :foo.to_json - # - # Output: - # - # # {"json_class":"Symbol","s":"foo"} - # - def to_json(state = nil, *a) - state = ::JSON::State.from_state(state) - if state.strict? - super - else - as_json.to_json(state, *a) - end - end - - # See #as_json. - def self.json_create(o) - o['s'].to_sym - end -end diff --git a/lib/json/add/time.rb b/lib/json/add/time.rb deleted file mode 100644 index 05a1f242f..000000000 --- a/lib/json/add/time.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true -unless defined?(::JSON::JSON_LOADED) and ::JSON::JSON_LOADED - require 'json' -end - -class Time - - # See #as_json. - def self.json_create(object) - if usec = object.delete('u') # used to be tv_usec -> tv_nsec - object['n'] = usec * 1000 - end - at(object['s'], Rational(object['n'], 1000)) - end - - # Methods Time#as_json and +Time.json_create+ may be used - # to serialize and deserialize a \Time object; - # see Marshal[https://docs.ruby-lang.org/en/master/Marshal.html]. - # - # \Method Time#as_json serializes +self+, - # returning a 2-element hash representing +self+: - # - # require 'json/add/time' - # x = Time.now.as_json - # # => {"json_class"=>"Time", "s"=>1700931656, "n"=>472846644} - # - # \Method +JSON.create+ deserializes such a hash, returning a \Time object: - # - # Time.json_create(x) - # # => 2023-11-25 11:00:56.472846644 -0600 - # - def as_json(*) - { - JSON.create_id => self.class.name, - 's' => tv_sec, - 'n' => tv_nsec, - } - end - - # Returns a JSON string representing +self+: - # - # require 'json/add/time' - # puts Time.now.to_json - # - # Output: - # - # {"json_class":"Time","s":1700931678,"n":980650786} - # - def to_json(*args) - as_json.to_json(*args) - end -end diff --git a/lib/json/common.rb b/lib/json/common.rb index a030f0464..679222a05 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -17,10 +17,6 @@ def prepare(opts) opts[:on_load] = on_load end - if opts.fetch(:create_additions, false) != false - opts = create_additions_proc(opts) - end - opts end @@ -47,48 +43,6 @@ def array_class_proc(array_class, on_load) on_load.nil? ? obj : on_load.call(obj) end end - - # TODO: extract :create_additions support to another gem for version 3.0 - def create_additions_proc(opts) - if opts[:symbolize_names] - raise ArgumentError, "options :symbolize_names and :create_additions cannot be used in conjunction" - end - - opts = opts.dup - create_additions = opts.fetch(:create_additions, false) - on_load = opts[:on_load] - object_class = opts[:object_class] || Hash - - opts[:on_load] = ->(object) do - case object - when String - opts[:match_string]&.each do |pattern, klass| - if match = pattern.match(object) - object = klass.json_create(object) - break - end - end - when object_class - if opts[:create_additions] != false - if class_path = object[JSON.create_id] - klass = begin - Object.const_get(class_path) - rescue NameError => e - raise ArgumentError, "can't get const #{class_path}: #{e}" - end - - if klass.respond_to?(:json_creatable?) ? klass.json_creatable? : klass.respond_to?(:json_create) - object = klass.json_create(object) - end - end - end - end - - on_load.nil? ? object : on_load.call(object) - end - - opts - end end end @@ -182,19 +136,6 @@ def on_mixed_keys_hash(hash) end end - # Sets create identifier, which is used to decide if the _json_create_ - # hook of a class should be called; initial value is +json_class+: - # JSON.create_id # => 'json_class' - def self.create_id=(new_value) - Thread.current[:"JSON.create_id"] = new_value.dup.freeze - end - - # Returns the current create identifier. - # See also JSON.create_id=. - def self.create_id - Thread.current[:"JSON.create_id"] || 'json_class' - end - NaN = Float::NAN Infinity = Float::INFINITY @@ -491,7 +432,7 @@ def pretty_generate(obj, opts = nil) # # Returns the Ruby objects created by parsing the given +source+. # - # BEWARE: This method is meant to serialise data from trusted user input, + # BEWARE: This method is meant to deserialise data from trusted user input, # like from your own database server or clients under your control, it could # be dangerous to allow untrusted users to pass JSON sources into it. # @@ -620,7 +561,6 @@ def unsafe_load(source, proc = nil, options = nil) max_nesting: false, allow_nan: true, allow_blank: true, - create_additions: true, } if options.nil? && proc && proc.is_a?(Hash) options, proc = proc, nil @@ -656,16 +596,6 @@ def unsafe_load(source, proc = nil, options = nil) # # Returns the Ruby objects created by parsing the given +source+. # - # BEWARE: This method is meant to serialise data from trusted user input, - # like from your own database server or clients under your control, it could - # be dangerous to allow untrusted users to pass JSON sources into it. - # If you must use it, use JSON.unsafe_load instead to make it clear. - # - # Since JSON version 2.8.0, `load` emits a deprecation warning when a - # non native type is deserialized, without `create_additions` being explicitly - # enabled, and in JSON version 3.0, `load` will have `create_additions` disabled - # by default. - # # - Argument +source+ must be, or be convertible to, a \String: # - If +source+ responds to instance method +to_str+, # source.to_str becomes the source. diff --git a/lib/json/generic_object.rb b/lib/json/generic_object.rb index 5c8ace354..4d2144f93 100644 --- a/lib/json/generic_object.rb +++ b/lib/json/generic_object.rb @@ -10,12 +10,6 @@ class GenericObject < OpenStruct class << self alias [] new - def json_creatable? - @json_creatable - end - - attr_writer :json_creatable - def json_create(data) data = data.dup data.delete JSON.create_id @@ -46,7 +40,6 @@ def dump(obj, *args) ::JSON.dump(obj, *args) end end - self.json_creatable = false def to_hash table diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb deleted file mode 100644 index fedd1f1d9..000000000 --- a/test/json/json_addition_test.rb +++ /dev/null @@ -1,186 +0,0 @@ -# frozen_string_literal: true -require_relative 'test_helper' -require 'json/add/core' -require 'json/add/complex' -require 'json/add/rational' -require 'json/add/bigdecimal' -require 'json/add/ostruct' -require 'json/add/set' -require 'date' - -class JSONAdditionTest < Test::Unit::TestCase - include JSON - - class A - def initialize(a) - @a = a - end - - attr_reader :a - - def ==(other) - a == other.a - end - - def self.json_create(object) - new(*object['args']) - end - - def to_json(*args) - { - 'json_class' => self.class.name, - 'args' => [ @a ], - }.to_json(*args) - end - end - - class A2 < A - def to_json(*args) - { - 'json_class' => self.class.name, - 'args' => [ @a ], - }.to_json(*args) - end - end - - class B - def to_json(*args) - { - 'json_class' => self.class.name, - }.to_json(*args) - end - end - - class C - def to_json(*args) - { - 'json_class' => 'JSONAdditionTest::Nix', - }.to_json(*args) - end - end - - def test_extended_json - a = A.new(666) - json = generate(a) - a_again = parse(json, :create_additions => true) - assert_kind_of a.class, a_again - assert_equal a, a_again - end - - def test_extended_json_default - a = A.new(666) - assert A.respond_to?(:json_create) - json = generate(a) - a_hash = parse(json) - assert_kind_of Hash, a_hash - end - - def test_extended_json_disabled - a = A.new(666) - json = generate(a) - a_again = parse(json, :create_additions => true) - assert_kind_of a.class, a_again - assert_equal a, a_again - a_hash = parse(json, :create_additions => false) - assert_kind_of Hash, a_hash - assert_equal( - {"args"=>[666], "json_class"=>"JSONAdditionTest::A"}.sort_by { |k,| k }, - a_hash.sort_by { |k,| k } - ) - end - - def test_extended_json_fail1 - b = B.new - json = generate(b) - assert_equal({ "json_class"=>"JSONAdditionTest::B" }, parse(json)) - end - - def test_extended_json_fail2 - c = C.new - json = generate(c) - assert_raise(ArgumentError, NameError) { parse(json, :create_additions => true) } - end - - def test_raw_strings - raw = ''.b - raw_array = [] - for i in 0..255 - raw << i - raw_array << i - end - json = raw.to_json_raw - json_raw_object = raw.to_json_raw_object - hash = { 'json_class' => 'String', 'raw'=> raw_array } - assert_equal hash, json_raw_object - assert_match(/\A\{.*\}\z/, json) - assert_match(/"json_class":"String"/, json) - assert_match(/"raw":\[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255\]/, json) - raw_again = parse(json, :create_additions => true) - assert_equal raw, raw_again - end - - MyJsonStruct = Struct.new 'MyJsonStruct', :foo, :bar - - def test_core - t = Time.now - assert_equal t, JSON(JSON(t), :create_additions => true) - d = Date.today - assert_equal d, JSON(JSON(d), :create_additions => true) - d = DateTime.civil(2007, 6, 14, 14, 57, 10, Rational(1, 12), 2299161) - assert_equal d, JSON(JSON(d), :create_additions => true) - assert_equal 1..10, JSON(JSON(1..10), :create_additions => true) - assert_equal 1...10, JSON(JSON(1...10), :create_additions => true) - assert_equal "a".."c", JSON(JSON("a".."c"), :create_additions => true) - assert_equal "a"..."c", JSON(JSON("a"..."c"), :create_additions => true) - s = MyJsonStruct.new 4711, 'foot' - assert_equal s, JSON(JSON(s), :create_additions => true) - struct = Struct.new :foo, :bar - s = struct.new 4711, 'foot' - assert_raise(JSONError) { JSON(s) } - begin - raise TypeError, "test me" - rescue TypeError => e - e_json = JSON.generate e - e_again = JSON e_json, :create_additions => true - assert_kind_of TypeError, e_again - assert_equal e.message, e_again.message - assert_equal e.backtrace, e_again.backtrace - end - assert_equal(/foo/, JSON(JSON(/foo/), :create_additions => true)) - assert_equal(/foo/i, JSON(JSON(/foo/i), :create_additions => true)) - end - - def test_utc_datetime - now = Time.now - d = DateTime.parse(now.to_s) # usual case - assert_equal d, parse(d.to_json, :create_additions => true) - d = DateTime.parse(now.utc.to_s) # of = 0 - assert_equal d, parse(d.to_json, :create_additions => true) - d = DateTime.civil(2008, 6, 17, 11, 48, 32, Rational(1,24)) - assert_equal d, parse(d.to_json, :create_additions => true) - d = DateTime.civil(2008, 6, 17, 11, 48, 32, Rational(12,24)) - assert_equal d, parse(d.to_json, :create_additions => true) - end - - def test_rational_complex - assert_equal Rational(2, 9), parse(JSON(Rational(2, 9)), :create_additions => true) - assert_equal Complex(2, 9), parse(JSON(Complex(2, 9)), :create_additions => true) - end - - def test_bigdecimal - assert_equal BigDecimal('3.141', 23), JSON(JSON(BigDecimal('3.141', 23)), :create_additions => true) - assert_equal BigDecimal('3.141', 666), JSON(JSON(BigDecimal('3.141', 666)), :create_additions => true) - end if defined?(::BigDecimal) - - def test_ostruct - o = OpenStruct.new - # XXX this won't work; o.foo = { :bar => true } - o.foo = { 'bar' => true } - assert_equal o, parse(JSON(o), :create_additions => true) - end if defined?(::OpenStruct) - - def test_set - s = Set.new([:a, :b, :c, :a]) - assert_equal s, JSON.parse(JSON(s), :create_additions => true) - end -end diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 72b8f390a..34ed27c69 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -54,14 +54,6 @@ def test_state assert_match(/::(TruffleRuby)?Generator::State\z/, JSON.state.name) end - def test_create_id - assert_equal 'json_class', JSON.create_id - JSON.create_id = 'foo_bar' - assert_equal 'foo_bar', JSON.create_id - ensure - JSON.create_id = 'json_class' - end - def test_parse assert_equal [ 1, 2, 3, ], JSON.parse('[ 1, 2, 3 ]') end diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb index 57e3bf3c5..01ae71e56 100644 --- a/test/json/json_generic_object_test.rb +++ b/test/json/json_generic_object_test.rb @@ -26,32 +26,13 @@ def test_attributes assert_nil @go[:c] end - def test_generate_json - switch_json_creatable do - assert_equal @go, JSON(JSON(@go), :create_additions => true) - end - end - def test_parse_json - assert_kind_of Hash, - JSON( - '{ "json_class": "JSON::GenericObject", "a": 1, "b": 2 }', - :create_additions => true - ) - switch_json_creatable do - assert_equal @go, l = - JSON( - '{ "json_class": "JSON::GenericObject", "a": 1, "b": 2 }', - :create_additions => true - ) - assert_equal 1, l.a - assert_equal @go, - l = JSON('{ "a": 1, "b": 2 }', :object_class => JSON::GenericObject) - assert_equal 1, l.a - assert_equal JSON::GenericObject[:a => JSON::GenericObject[:b => 2]], - l = JSON('{ "a": { "b": 2 } }', :object_class => JSON::GenericObject) - assert_equal 2, l.a.b - end + assert_equal @go, + l = JSON('{ "a": 1, "b": 2 }', :object_class => JSON::GenericObject) + assert_equal 1, l.a + assert_equal JSON::GenericObject[:a => JSON::GenericObject[:b => 2]], + l = JSON('{ "a": { "b": 2 } }', :object_class => JSON::GenericObject) + assert_equal 2, l.a.b end def test_from_hash @@ -79,13 +60,4 @@ def test_json_generic_object_load json = JSON::GenericObject.dump(JSON::GenericObject[:hello => 'world']) assert_equal JSON(json), JSON('{"json_class":"JSON::GenericObject","hello":"world"}') end - - private - - def switch_json_creatable - JSON::GenericObject.json_creatable = true - yield - ensure - JSON::GenericObject.json_creatable = false - end end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 1f257e71d..03a29d0e8 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -477,9 +477,6 @@ def test_symbolize_names parse('{"foo":"bar", "baz":"quux"}')) assert_equal({ :foo => "bar", :baz => "quux" }, parse('{"foo":"bar", "baz":"quux"}', :symbolize_names => true)) - assert_raise(ArgumentError) do - parse('{}', :symbolize_names => true, :create_additions => true) - end end def test_freeze @@ -687,20 +684,6 @@ def shifted? end end - class SubArray2 < Array - def to_json(*a) - { - JSON.create_id => self.class.name, - 'ary' => to_a, - }.to_json(*a) - end - - def self.json_create(o) - o.delete JSON.create_id - o['ary'] - end - end - class SubArrayWrapper def initialize @data = [] @@ -813,18 +796,6 @@ def test_parse_generic_object end end - def test_generate_core_subclasses_with_new_to_json - obj = SubHash2["foo" => SubHash2["bar" => true]] - obj_json = JSON(obj) - obj_again = parse(obj_json, :create_additions => true) - assert_kind_of SubHash2, obj_again - assert_kind_of SubHash2, obj_again['foo'] - assert obj_again['foo']['bar'] - assert_equal obj, obj_again - assert_equal ["foo"], - JSON(JSON(SubArray2["foo"]), :create_additions => true) - end - def test_generate_core_subclasses_with_default_to_json assert_equal '{"foo":"bar"}', JSON(SubHash["foo" => "bar"]) assert_equal '["foo"]', JSON(SubArray["foo"]) diff --git a/test/json/json_string_matching_test.rb b/test/json/json_string_matching_test.rb deleted file mode 100644 index 21cd64902..000000000 --- a/test/json/json_string_matching_test.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true -require_relative 'test_helper' -require 'time' - -class JSONStringMatchingTest < Test::Unit::TestCase - include JSON - - class TestTime < ::Time - def self.json_create(string) - Time.parse(string) - end - - def to_json(*) - %{"#{strftime('%FT%T%z')}"} - end - - def ==(other) - to_i == other.to_i - end - end - - def test_match_date - t = TestTime.new - t_json = [ t ].to_json - time_regexp = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[+-]\d{4}\z/ - assert_equal [ t ], - parse( - t_json, - :create_additions => true, - :match_string => { time_regexp => TestTime } - ) - assert_equal [ t.strftime('%FT%T%z') ], - parse( - t_json, - :match_string => { time_regexp => TestTime } - ) - end -end From e93e5d645ed20fb6644536abf272f6291ea71097 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 21:04:13 +0200 Subject: [PATCH 426/506] Remove JSON.fast_generate --- CHANGES.md | 1 + lib/json/common.rb | 22 -------------- test/json/json_generator_test.rb | 52 +++++++++++--------------------- 3 files changed, 19 insertions(+), 56 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 382428b19..c0d4462cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ * Removed `JSON.dump_default_options`. * Removed `JSON::State#[]` and `JSON::State#[]=`. * Removed `JSON.unparse`. +* Removed `JSON.fast_generate`. * Removed `JSON.fast_unparse`. * Removed `JSON.pretty_unparse`. * Removed `JSON.restore`. diff --git a/lib/json/common.rb b/lib/json/common.rb index 679222a05..27a489c11 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -345,28 +345,6 @@ def generate(obj, opts = nil) end end - # :call-seq: - # JSON.fast_generate(obj, opts) -> new_string - # - # Arguments +obj+ and +opts+ here are the same as - # arguments +obj+ and +opts+ in JSON.generate. - # - # By default, generates \JSON data without checking - # for circular references in +obj+ (option +max_nesting+ set to +false+, disabled). - # - # Raises an exception if +obj+ contains circular references: - # a = []; b = []; a.push(b); b.push(a) - # # Raises SystemStackError (stack level too deep): - # JSON.fast_generate(a) - def fast_generate(obj, opts = nil) - if RUBY_VERSION >= "3.0" - warn "JSON.fast_generate is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1, category: :deprecated - else - warn "JSON.fast_generate is deprecated and will be removed in json 3.0.0, just use JSON.generate", uplevel: 1 - end - generate(obj, opts) - end - PRETTY_GENERATE_OPTIONS = { indent: ' ', space: ' ', diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 937f71651..f124e22b6 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -256,20 +256,6 @@ def test_generate_custom JSON end - def test_fast_generate - assert_deprecated_warning(/fast_generate/) do - json = fast_generate(@hash) - assert_equal(parse(@json2), parse(json)) - parsed_json = parse(json) - assert_equal(@hash, parsed_json) - json = fast_generate({1=>2}) - assert_equal('{"1":2}', json) - parsed_json = parse(json) - assert_equal({"1"=>2}, parsed_json) - assert_equal '666', fast_generate(666) - end - end - def test_own_state state = State.new json = generate(@hash, state) @@ -359,26 +345,24 @@ def test_state_defaults end def test_allow_nan - assert_deprecated_warning(/fast_generate/) do - error = assert_raise(GeneratorError) { generate([JSON::NaN]) } - assert_same JSON::NaN, error.invalid_object - assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::NaN]) } - assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) } - assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true) - error = assert_raise(GeneratorError) { generate([JSON::Infinity]) } - assert_same JSON::Infinity, error.invalid_object - assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::Infinity]) } - assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) } - assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true) - error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } - assert_same JSON::MinusInfinity, error.invalid_object - assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true) - assert_raise(GeneratorError) { fast_generate([JSON::MinusInfinity]) } - assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) } - assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true) - end + error = assert_raise(GeneratorError) { generate([JSON::NaN]) } + assert_same JSON::NaN, error.invalid_object + assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true) + assert_raise(GeneratorError) { generate([JSON::NaN]) } + assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) } + assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true) + error = assert_raise(GeneratorError) { generate([JSON::Infinity]) } + assert_same JSON::Infinity, error.invalid_object + assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true) + assert_raise(GeneratorError) { generate([JSON::Infinity]) } + assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) } + assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true) + error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } + assert_same JSON::MinusInfinity, error.invalid_object + assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true) + assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } + assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) } + assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true) end # An object that changes state.depth when it receives to_json(state) From 273b38d8f44d1af87b5048df9a9ccedd440f8c11 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 27 Jul 2026 20:57:45 +0200 Subject: [PATCH 427/506] Remove JSON::GenericObject --- CHANGES.md | 1 + Gemfile | 1 - lib/json/common.rb | 2 - lib/json/generic_object.rb | 60 ------------------------- test/json/json_generic_object_test.rb | 63 --------------------------- test/json/json_parser_test.rb | 53 ++++++++-------------- 6 files changed, 20 insertions(+), 160 deletions(-) delete mode 100644 lib/json/generic_object.rb delete mode 100644 test/json/json_generic_object_test.rb diff --git a/CHANGES.md b/CHANGES.md index c0d4462cc..5479d7965 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ * Removed `JSON.restore`. * Removed `JSON::PRETTY_STATE_PROTOTYPE`. * Removed the insecure `create_additions` option. +* Removed `JSON::GenericObject`. ### 2026-07-31 (2.21.2) diff --git a/Gemfile b/Gemfile index 7956c2dc2..407a3f9b0 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,6 @@ gemspec group :development do gem "ruby_memcheck" if RUBY_PLATFORM =~ /linux/i gem "bigdecimal" - gem "ostruct" gem "rake" gem "rake-compiler" gem "test-unit" diff --git a/lib/json/common.rb b/lib/json/common.rb index 27a489c11..78dc867dd 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -3,8 +3,6 @@ require 'json/version' module JSON - autoload :GenericObject, 'json/generic_object' - module ParserOptions # :nodoc: class << self def prepare(opts) diff --git a/lib/json/generic_object.rb b/lib/json/generic_object.rb deleted file mode 100644 index 4d2144f93..000000000 --- a/lib/json/generic_object.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true -begin - require 'ostruct' -rescue LoadError - warn "JSON::GenericObject requires 'ostruct'. Please install it with `gem install ostruct`." -end - -module JSON - class GenericObject < OpenStruct - class << self - alias [] new - - def json_create(data) - data = data.dup - data.delete JSON.create_id - self[data] - end - - def from_hash(object) - case - when object.respond_to?(:to_hash) - result = new - object.to_hash.each do |key, value| - result[key] = from_hash(value) - end - result - when object.respond_to?(:to_ary) - object.to_ary.map { |a| from_hash(a) } - else - object - end - end - - def load(source, proc = nil, opts = {}) - result = ::JSON.load(source, proc, opts.merge(:object_class => self)) - result.nil? ? new : result - end - - def dump(obj, *args) - ::JSON.dump(obj, *args) - end - end - - def to_hash - table - end - - def |(other) - self.class[other.to_hash.merge(to_hash)] - end - - def as_json(*) - { JSON.create_id => self.class.name }.merge to_hash - end - - def to_json(*a) - as_json.to_json(*a) - end - end if defined?(::OpenStruct) -end diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb deleted file mode 100644 index 01ae71e56..000000000 --- a/test/json/json_generic_object_test.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true -require_relative 'test_helper' - -# ostruct is required to test JSON::GenericObject -begin - require "ostruct" -rescue LoadError - return -end - -class JSONGenericObjectTest < Test::Unit::TestCase - def setup - if defined?(JSON::GenericObject) - @go = JSON::GenericObject[ :a => 1, :b => 2 ] - else - omit("JSON::GenericObject is not available") - end - end - - def test_attributes - assert_equal 1, @go.a - assert_equal 1, @go[:a] - assert_equal 2, @go.b - assert_equal 2, @go[:b] - assert_nil @go.c - assert_nil @go[:c] - end - - def test_parse_json - assert_equal @go, - l = JSON('{ "a": 1, "b": 2 }', :object_class => JSON::GenericObject) - assert_equal 1, l.a - assert_equal JSON::GenericObject[:a => JSON::GenericObject[:b => 2]], - l = JSON('{ "a": { "b": 2 } }', :object_class => JSON::GenericObject) - assert_equal 2, l.a.b - end - - def test_from_hash - result = JSON::GenericObject.from_hash( - :foo => { :bar => { :baz => true }, :quux => [ { :foobar => true } ] }) - assert_kind_of JSON::GenericObject, result.foo - assert_kind_of JSON::GenericObject, result.foo.bar - assert_equal true, result.foo.bar.baz - assert_kind_of JSON::GenericObject, result.foo.quux.first - assert_equal true, result.foo.quux.first.foobar - assert_equal true, JSON::GenericObject.from_hash(true) - end - - def test_json_generic_object_load - empty = JSON::GenericObject.load(nil) - assert_kind_of JSON::GenericObject, empty - simple_json = '{"json_class":"JSON::GenericObject","hello":"world"}' - simple = JSON::GenericObject.load(simple_json) - assert_kind_of JSON::GenericObject, simple - assert_equal "world", simple.hello - converting = JSON::GenericObject.load('{ "hello": "world" }') - assert_kind_of JSON::GenericObject, converting - assert_equal "world", converting.hello - - json = JSON::GenericObject.dump(JSON::GenericObject[:hello => 'world']) - assert_equal JSON(json), JSON('{"json_class":"JSON::GenericObject","hello":"world"}') - end -end diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 03a29d0e8..1819d0abf 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -2,10 +2,6 @@ require_relative 'test_helper' require 'stringio' require 'tempfile' -begin - require 'ostruct' -rescue LoadError -end begin require 'bigdecimal' rescue LoadError @@ -758,44 +754,33 @@ def test_parse_object_custom_hash_derived_class assert res.item_set? end - if defined?(::OpenStruct) - class SubOpenStruct < OpenStruct - def [](k) - __send__(k) - end - - def []=(k, v) - @item_set = true - __send__("#{k}=", v) - end + class OpenStructLike + def initialize + @attrs = {} + end - def item_set? - @item_set - end + def [](k) + @attrs[k.to_sym] end - def test_parse_object_custom_non_hash_derived_class - res = parse('{"foo":"bar"}', :object_class => SubOpenStruct) - assert_equal "bar", res.foo - assert_equal "bar", res[:foo] - assert_equal(SubOpenStruct, res.class) - assert res.item_set? + def []=(k, v) + @attrs[k.to_sym] = v end - def test_parse_generic_object - res = parse( - '{"foo":"bar", "baz":{}}', - :object_class => JSON::GenericObject - ) - assert_equal(JSON::GenericObject, res.class) - assert_equal "bar", res.foo - assert_equal "bar", res["foo"] - assert_equal "bar", res[:foo] - assert_equal "bar", res.to_hash[:foo] - assert_equal(JSON::GenericObject, res.baz.class) + def method_missing(name, ...) + @attrs.fetch(name) do + super + end end end + def test_parse_object_custom_non_hash_derived_class + res = parse('{"foo":"bar"}', :object_class => OpenStructLike) + assert_equal "bar", res.foo + assert_equal "bar", res[:foo] + assert_equal(OpenStructLike, res.class) + end + def test_generate_core_subclasses_with_default_to_json assert_equal '{"foo":"bar"}', JSON(SubHash["foo" => "bar"]) assert_equal '["foo"]', JSON(SubArray["foo"]) From 2aa6ecb44f56817cd5a207e955a0894b230fb205 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 16:10:02 +0200 Subject: [PATCH 428/506] Remove `escape_slash` --- CHANGES.md | 1 + ext/json/ext/generator/generator.c | 7 +------ java/src/json/ext/GeneratorState.java | 9 +++------ lib/json/truffle_ruby/generator.rb | 2 -- test/json/json_generator_test.rb | 2 +- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5479d7965..1f0f9111e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. +* Removed the `escape_slash` alias of `script_safe`. * Removed `Kernel#j` and `Kernel#jj`. * Removed `JSON.load_default_options`. * Removed `JSON.unsafe_load_default_options`. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index a45fd3928..753a5656e 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -34,7 +34,7 @@ static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_ static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, - sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_escape_slash, sym_strict, sym_as_json, sym_sort_keys; + sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys; #define GET_STATE_TO(self, state) \ @@ -1868,7 +1868,6 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_depth) { state->depth = depth_config(val); } else if (key == sym_buffer_initial_length) { buffer_initial_length_set(state, val); } else if (key == sym_script_safe) { state->script_safe = RTEST(val); } - else if (key == sym_escape_slash) { state->script_safe = RTEST(val); } else if (key == sym_strict) { state->strict = RTEST(val); } else if (key == sym_allow_duplicate_key) { state->allow_duplicate_key = RTEST(val); } else if (key == sym_as_json) { @@ -1991,9 +1990,6 @@ void Init_generator(void) rb_define_method(cState, "script_safe", cState_script_safe, 0); rb_define_method(cState, "script_safe?", cState_script_safe, 0); rb_define_method(cState, "script_safe=", cState_script_safe_set, 1); - rb_define_alias(cState, "escape_slash", "script_safe"); - rb_define_alias(cState, "escape_slash?", "script_safe?"); - rb_define_alias(cState, "escape_slash=", "script_safe="); rb_define_method(cState, "strict", cState_strict, 0); rb_define_method(cState, "strict?", cState_strict, 0); rb_define_method(cState, "strict=", cState_strict_set, 1); @@ -2035,7 +2031,6 @@ void Init_generator(void) sym_depth = ID2SYM(rb_intern("depth")); sym_buffer_initial_length = ID2SYM(rb_intern("buffer_initial_length")); sym_script_safe = ID2SYM(rb_intern("script_safe")); - sym_escape_slash = ID2SYM(rb_intern("escape_slash")); sym_strict = ID2SYM(rb_intern("strict")); sym_as_json = ID2SYM(rb_intern("as_json")); sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 1086bf6d3..143cf2d7c 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -397,19 +397,19 @@ public boolean scriptSafe() { return scriptSafe; } - @JRubyMethod(name="script_safe", alias="escape_slash") + @JRubyMethod(name="script_safe") public RubyBoolean script_safe_get(ThreadContext context) { return RubyBoolean.newBoolean(context, scriptSafe); } - @JRubyMethod(name="script_safe=", alias="escape_slash=") + @JRubyMethod(name="script_safe=") public IRubyObject script_safe_set(IRubyObject script_safe) { checkFrozen(); scriptSafe = script_safe.isTrue(); return script_safe.getRuntime().newBoolean(scriptSafe); } - @JRubyMethod(name="script_safe?", alias="escape_slash?") + @JRubyMethod(name="script_safe?") public RubyBoolean script_safe_p(ThreadContext context) { return RubyBoolean.newBoolean(context, scriptSafe); } @@ -566,9 +566,6 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { allowNaN = opts.getBool("allow_nan", DEFAULT_ALLOW_NAN); asciiOnly = opts.getBool("ascii_only", DEFAULT_ASCII_ONLY); scriptSafe = opts.getBool("script_safe", DEFAULT_SCRIPT_SAFE); - if (!scriptSafe) { - scriptSafe = opts.getBool("escape_slash", DEFAULT_SCRIPT_SAFE); - } strict = opts.getBool("strict", DEFAULT_STRICT); bufferInitialLength = opts.getInt("buffer_initial_length", DEFAULT_BUFFER_INITIAL_LENGTH); diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 495167be2..4f033ac58 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -323,8 +323,6 @@ def configure(opts) @script_safe = if opts.key?(:script_safe) !!opts[:script_safe] - elsif opts.key?(:escape_slash) - !!opts[:escape_slash] else false end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f124e22b6..7ae415cf5 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -651,7 +651,7 @@ def test_backslash # data = [ "/\u2028\u2029" ] json = '["\/\u2028\u2029"]' - assert_equal json, generate(data, :escape_slash => true) + assert_equal json, generate(data, :script_safe => true) # data = ['"'] json = '["\""]' From 6e668d2cfe98373851d862677968f2f9b5fd52c0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 16:14:34 +0200 Subject: [PATCH 429/506] Remove the `limit` positional argument of `JSON.dump`. It's entirely redundant with `max_nesting`, makes little sense except to match Marshal signature exactly, but I don't see the point. --- CHANGES.md | 1 + lib/json/common.rb | 31 ++++++--------------- test/json/json_common_interface_test.rb | 36 +++++++++++-------------- 3 files changed, 25 insertions(+), 43 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1f0f9111e..98893bc68 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. +* Removed the `limit` positional argument of `JSON.dump`. * Removed the `escape_slash` alias of `script_safe`. * Removed `Kernel#j` and `Kernel#jj`. * Removed `JSON.load_default_options`. diff --git a/lib/json/common.rb b/lib/json/common.rb index 78dc867dd..f796a0c81 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -728,7 +728,7 @@ def load(source, proc = nil, options = nil) end # :call-seq: - # JSON.dump(obj, io = nil, limit = nil) + # JSON.dump(obj, io = nil, options = nil) # # Dumps +obj+ as a \JSON string, i.e. calls generate on the object and returns the result. # @@ -737,7 +737,6 @@ def load(source, proc = nil, options = nil) # - Argument +io+, if given, should respond to method +write+; # the \JSON \String is written to +io+, and +io+ is returned. # If +io+ is not given, the \JSON \String is returned. - # - Argument +limit+, if given, is passed to JSON.generate as option +max_nesting+. # # --- # @@ -754,38 +753,24 @@ def load(source, proc = nil, options = nil) # puts File.read(path) # Output: # {"foo":[0,1],"bar":{"baz":2,"bat":3},"bam":"bad"} - def dump(obj, anIO = nil, limit = nil, kwargs = nil) + def dump(obj, anIO = nil, kwargs = nil) if kwargs.nil? - if limit.nil? - if anIO.is_a?(Hash) - kwargs = anIO - anIO = nil - end - elsif limit.is_a?(Hash) - kwargs = limit - limit = nil + if anIO.is_a?(Hash) + kwargs = anIO + anIO = nil end end - unless anIO.nil? - if anIO.respond_to?(:to_io) - anIO = anIO.to_io - elsif limit.nil? && !anIO.respond_to?(:write) - anIO, limit = nil, anIO - end + if anIO&.respond_to?(:to_io) + anIO = anIO.to_io end opts = { allow_nan: true, - max_nesting: limit, } opts.merge!(kwargs) if kwargs - begin - State.generate(obj, opts, anIO) - rescue JSON::NestingError - raise ArgumentError, "exceed depth limit" - end + State.generate(obj, opts, anIO) end # JSON::Coder holds a parser and generator configuration. diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 34ed27c69..9b0802de5 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -229,27 +229,23 @@ def test_unsafe_load_null end def test_dump - too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' + too_deep = '[' * 101 + ']' * 101 obj = eval(too_deep) - assert_equal too_deep, dump(obj) - assert_kind_of String, Marshal.dump(obj) - assert_raise(ArgumentError) { dump(obj, 100) } - assert_raise(ArgumentError) { Marshal.dump(obj, 100) } - assert_equal too_deep, dump(obj, 101) - assert_kind_of String, Marshal.dump(obj, 101) - - assert_equal too_deep, JSON.dump(obj, StringIO.new, 101, strict: false).string - assert_equal too_deep, dump(obj, StringIO.new, 101, strict: false).string - assert_raise(JSON::GeneratorError) { JSON.dump(Object.new, StringIO.new, 101, strict: true).string } - assert_raise(JSON::GeneratorError) { dump(Object.new, StringIO.new, 101, strict: true).string } - - assert_equal too_deep, dump(obj, nil, nil, strict: false) - assert_equal too_deep, dump(obj, nil, 101, strict: false) - assert_equal too_deep, dump(obj, StringIO.new, nil, strict: false).string - assert_equal too_deep, dump(obj, nil, strict: false) - assert_equal too_deep, dump(obj, 101, strict: false) - assert_equal too_deep, dump(obj, StringIO.new, strict: false).string - assert_equal too_deep, dump(obj, strict: false) + assert_raise(JSON::NestingError) { dump(obj) } + assert_equal too_deep, dump(obj, max_nesting: 101) + + assert_equal too_deep, JSON.dump(obj, StringIO.new, max_nesting: 101, strict: false).string + assert_equal too_deep, dump(obj, StringIO.new, max_nesting: 101, strict: false).string + assert_raise(JSON::GeneratorError) { JSON.dump(Object.new, StringIO.new, max_nesting: 101, strict: true).string } + assert_raise(JSON::GeneratorError) { dump(Object.new, StringIO.new, max_nesting: 101, strict: true).string } + + assert_raise(JSON::NestingError) { dump(obj, nil, strict: false) } + assert_equal too_deep, dump(obj, nil, max_nesting: 101, strict: false) + assert_raise(JSON::NestingError) { dump(obj, StringIO.new, strict: false) } + assert_raise(JSON::NestingError) { dump(obj, strict: false) } + assert_equal too_deep, dump(obj, max_nesting: 101, strict: false) + assert_raise(JSON::NestingError) { dump(obj, StringIO.new, strict: false) } + assert_raise(JSON::NestingError) { dump(obj, strict: false) } end def test_dump_in_io From ebf0f2901c162eb0b3008001a50ba378199a0a2f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 12:00:31 +0200 Subject: [PATCH 430/506] Make parsing options keyword arguments Or at least behave like them. --- ext/json/ext/parser/parser.c | 2 +- lib/json/common.rb | 156 +++++++++--------------- test/json/json_common_interface_test.rb | 7 ++ 3 files changed, 68 insertions(+), 97 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 54ecb2c89..8b6c04efe 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2015,7 +2015,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) } } } - else if (args->strict) { + else { if (!args->unknown_keywords) { args->unknown_keywords = rb_obj_hide(rb_ary_new()); } diff --git a/lib/json/common.rb b/lib/json/common.rb index f796a0c81..6196aef6c 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -5,17 +5,10 @@ module JSON module ParserOptions # :nodoc: class << self - def prepare(opts) - if opts[:object_class] || opts[:array_class] - opts = opts.dup - on_load = opts[:on_load] - - on_load = object_class_proc(opts[:object_class], on_load) if opts[:object_class] - on_load = array_class_proc(opts[:array_class], on_load) if opts[:array_class] - opts[:on_load] = on_load - end - - opts + def on_load(on_load, object_class, array_class) + on_load = object_class_proc(object_class, on_load) if object_class + on_load = array_class_proc(array_class, on_load) if array_class + on_load end private @@ -44,6 +37,8 @@ def array_class_proc(array_class, on_load) end end + private_constant :ParserOptions + class << self # :call-seq: # JSON[object] -> new_array or new_string @@ -57,12 +52,14 @@ class << self # ruby = [0, 1, nil] # JSON[ruby] # => '[0,1,null]' def [](object, opts = nil) + opts ||= {}.freeze + if object.is_a?(String) - return JSON.parse(object, opts) + return JSON.parse(object, **opts) elsif object.respond_to?(:to_str) str = object.to_str if str.is_a?(String) - return JSON.parse(str, opts) + return JSON.parse(str, **opts) end end @@ -247,16 +244,14 @@ def to_json(state = nil, *) # # Raises JSON::ParserError (783: unexpected token at ''): # JSON.parse('') # - def parse(source, opts = nil) - opts = ParserOptions.prepare(opts) unless opts.nil? - Parser.parse(source, opts) - end + def parse(source, on_load: nil, object_class: nil, array_class: nil, **options) + if object_class || array_class + on_load = ParserOptions.on_load(on_load, object_class, array_class) + end - PARSE_L_OPTIONS = { - max_nesting: false, - allow_nan: true, - }.freeze - private_constant :PARSE_L_OPTIONS + options[:on_load] = on_load if on_load + Parser.parse(source, options) + end # :call-seq: # JSON.parse!(source, opts) -> object @@ -269,34 +264,30 @@ def parse(source, opts = nil) # - Option +max_nesting+, if not provided, defaults to +false+, # which disables checking for nesting depth. # - Option +allow_nan+, if not provided, defaults to +true+. - def parse!(source, opts = nil) - if opts.nil? - parse(source, PARSE_L_OPTIONS) - else - parse(source, PARSE_L_OPTIONS.merge(opts)) - end + def parse!(source, **options) + parse(source, max_nesting: false, allow_nan: true, **options) end # :call-seq: - # JSON.load_file(path, opts={}) -> object + # JSON.load_file(path, **) -> object # # Calls: - # parse(File.read(path), opts) + # parse(File.read(path), **) # # See method #parse. - def load_file(filespec, opts = nil) - parse(File.read(filespec, encoding: Encoding::UTF_8), opts) + def load_file(filespec, ...) + parse(File.read(filespec, encoding: Encoding::UTF_8), ...) end # :call-seq: - # JSON.load_file!(path, opts = {}) + # JSON.load_file!(path, **) # # Calls: - # JSON.parse!(File.read(path, opts)) + # JSON.parse!(File.read(path), **) # # See method #parse! - def load_file!(filespec, opts = nil) - parse!(File.read(filespec, encoding: Encoding::UTF_8), opts) + def load_file!(filespec, ...) + parse!(File.read(filespec, encoding: Encoding::UTF_8), ...) end # :call-seq: @@ -532,38 +523,8 @@ def pretty_generate(obj, opts = nil) # #"Admin", "password"=>"0wn3d"}>} # - def unsafe_load(source, proc = nil, options = nil) - opts = { - max_nesting: false, - allow_nan: true, - allow_blank: true, - } - if options.nil? && proc && proc.is_a?(Hash) - options, proc = proc, nil - end - - opts.merge!(options) unless options.nil? - - unless source.is_a?(String) - if source.respond_to? :to_str - source = source.to_str - elsif source.respond_to? :to_io - source = source.to_io.read - elsif source.respond_to?(:read) - source = source.read - end - end - - if opts[:allow_blank] && (source.nil? || source.empty?) - source = 'null' - end - - if proc - opts = opts.dup - opts[:on_load] = proc.to_proc - end - - parse(source, opts) + def unsafe_load(source, proc = nil, **options) + load(source, proc, max_nesting: false, **options) end # :call-seq: @@ -692,19 +653,7 @@ def unsafe_load(source, proc = nil, options = nil) # #"Admin", "password"=>"0wn3d"}>} # - def load(source, proc = nil, options = nil) - if proc && options.nil? && proc.is_a?(Hash) - options = proc - proc = nil - end - - opts = { - allow_nan: true, - allow_blank: true, - } - - opts.merge!(options) unless options.nil? - + def load(source, proc = nil, allow_blank: true, **options) unless source.is_a?(String) if source.respond_to? :to_str source = source.to_str @@ -715,16 +664,15 @@ def load(source, proc = nil, options = nil) end end - if opts[:allow_blank] && (source.nil? || (String === source && source.empty?)) + if allow_blank && (source.nil? || (String === source && source.empty?)) source = 'null' end if proc - opts = opts.dup - opts[:on_load] = proc.to_proc + parse(source, allow_nan: true, on_load: proc.to_proc, **options) + else + parse(source, allow_nan: true, **options) end - - parse(source, opts) end # :call-seq: @@ -784,6 +732,20 @@ def dump(obj, anIO = nil, kwargs = nil) # MyApp::JSONC_CODER.load(document) # class Coder + PARSER_OPTIONS = %i( + max_nesting + allow_nan + allow_trailing_comma + allow_comments + allow_control_characters + allow_invalid_escape + symbolize_names + freeze + allow_duplicate_key + decimal_class + ).freeze + private_constant :PARSER_OPTIONS + # :call-seq: # JSON.new(options = nil, &block) # @@ -808,17 +770,19 @@ class Coder # # puts MyApp::API_JSON_CODER.dump(Time.now.utc) # => "2025-01-21T08:41:44.286Z" # - def initialize(options = nil, &as_json) - if options.nil? - options = { strict: true } - else - options = options.dup - options[:strict] = true + def initialize(object_class: nil, array_class: nil, on_load: nil, **options, &as_json) + if object_class || array_class + on_load = ParserOptions.on_load(on_load, object_class, array_class) end - options[:as_json] = as_json if as_json - - @state = State.new(options).freeze - @parser_config = Ext::Parser::Config.new(ParserOptions.prepare(options)).freeze + parser_options = options.slice(*PARSER_OPTIONS) + parser_options[:on_load] = on_load if on_load + @parser_config = Ext::Parser::Config.new(parser_options).freeze + + @state = State.new( + **options, + strict: true, + as_json: as_json, + ).freeze end # call-seq: diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 9b0802de5..9abe7f731 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -58,6 +58,13 @@ def test_parse assert_equal [ 1, 2, 3, ], JSON.parse('[ 1, 2, 3 ]') end + def test_parse_unknown_option + error = assert_raise(ArgumentError) do + JSON.parse('[]', quirks_mode: true) + end + assert_match "quirks_mode", error.message + end + def test_parse_bang assert_equal [ 1, Infinity, 3, ], JSON.parse!('[ 1, Infinity, 3 ]') end From a8d7f6da6b67c6b378eab179753a6272cc47651c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 14:24:34 +0200 Subject: [PATCH 431/506] Make generator options strict --- CHANGES.md | 1 + ext/json/ext/generator/generator.c | 12 +++- ext/json/ext/json.h | 13 ++++ ext/json/ext/parser/parser.c | 10 +--- java/src/json/ext/GeneratorState.java | 2 + java/src/json/ext/OptionsReader.java | 30 ++++++++-- java/src/json/ext/Parser.java | 2 + lib/json/common.rb | 13 +++- lib/json/truffle_ruby/generator.rb | 80 +++++++++++-------------- test/json/json_common_interface_test.rb | 17 ++++++ test/json/json_generator_test.rb | 2 +- 11 files changed, 119 insertions(+), 63 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 98893bc68..aed16f405 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ### Unreleased +* All all unknown options are now cause an `ArgumentError` rather than to be ignored. * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. * Removed the `limit` positional argument of `JSON.dump`. diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 753a5656e..38607cf4e 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1841,6 +1841,7 @@ static VALUE cState_buffer_initial_length_set(VALUE self, VALUE buffer_initial_l struct configure_state_data { JSON_Generator_State *state; VALUE vstate; // Ruby object that owns the state, or Qfalse if stack-allocated + VALUE unknown_keywords; }; static inline void state_write_value(struct configure_state_data *data, VALUE *field, VALUE value) @@ -1878,6 +1879,12 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_sort_keys) { state_write_value(data, &state->sort_keys, normalize_sort_keys(val)); } + else { + if (!data->unknown_keywords) { + data->unknown_keywords = rb_obj_hide(rb_ary_new()); + } + rb_ary_push(data->unknown_keywords, key); + } return ST_CONTINUE; } @@ -1891,12 +1898,15 @@ static void configure_state(JSON_Generator_State *state, VALUE vstate, VALUE con struct configure_state_data data = { .state = state, - .vstate = vstate + .vstate = vstate, + .unknown_keywords = Qfalse, }; // We assume in most cases few keys are set so it's faster to go over // the provided keys than to check all possible keys. rb_hash_foreach(config, configure_state_i, (VALUE)&data); + + raise_argument_error_on_unknown_keywords(data.unknown_keywords); } static VALUE cState_configure(VALUE self, VALUE opts) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index afddbe2a2..be69bf589 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -180,4 +180,17 @@ static inline VALUE json_rb_catch_obj(VALUE tag, VALUE (*func)(VALUE args), VALU #endif // JSON_TRUFFLERUBY_RB_CATCH_BUG +static inline void raise_argument_error_on_unknown_keywords(VALUE unknown_keywords) +{ + if (RB_UNLIKELY(unknown_keywords)) { + if (RARRAY_LEN(unknown_keywords) == 1) { + rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(unknown_keywords, 0)); + } + else { + VALUE keywords = rb_ary_join(unknown_keywords, rb_utf8_str_new_cstr(", ")); + rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords); + } + } +} + #endif // _JSON_H_ diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 8b6c04efe..f10e2260a 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2043,15 +2043,7 @@ static void parser_config_init(JSON_ParserConfig *config, VALUE opts, VALUE self // the provided keys than to check all possible keys. rb_hash_foreach(opts, parser_config_init_i, (VALUE)&args); - if (RB_UNLIKELY(args.unknown_keywords)) { - if (RARRAY_LEN(args.unknown_keywords) == 1) { - rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, RARRAY_AREF(args.unknown_keywords, 0)); - } - else { - VALUE keywords = rb_ary_join(args.unknown_keywords, rb_utf8_str_new_cstr(", ")); - rb_raise(rb_eArgError, "unknown keywords: %" PRIsVALUE, keywords); - } - } + raise_argument_error_on_unknown_keywords(args.unknown_keywords); } /* diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 143cf2d7c..dabc9e255 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -577,6 +577,8 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); + opts.ensureEmpty(); + return this; } diff --git a/java/src/json/ext/OptionsReader.java b/java/src/json/ext/OptionsReader.java index 829e36b31..d93d3cd79 100644 --- a/java/src/json/ext/OptionsReader.java +++ b/java/src/json/ext/OptionsReader.java @@ -26,15 +26,19 @@ final class OptionsReader { OptionsReader(ThreadContext context, IRubyObject vOpts) { this.context = context; this.runtime = context.runtime; + + RubyHash options; if (vOpts == null || vOpts.isNil()) { - opts = null; + options = null; } else if (vOpts.respondsTo("to_hash")) { - opts = vOpts.convertToHash(); + options = vOpts.convertToHash(); } else if (vOpts.respondsTo("to_h")) { - opts = vOpts.callMethod(context, "to_h").convertToHash(); + options = vOpts.callMethod(context, "to_h").convertToHash(); } else { - opts = vOpts.convertToHash(); /* Should just raise the correct TypeError */ + options = vOpts.convertToHash(); /* Should just raise the correct TypeError */ } + + this.opts = options == null ? null : (RubyHash)options.dup(); } private RuntimeInfo getRuntimeInfo() { @@ -43,6 +47,19 @@ private RuntimeInfo getRuntimeInfo() { return info; } + + void ensureEmpty() { + if (opts == null) return; + + if (opts.size() == 1) { + throw context.runtime.newArgumentError("unknown keyword: " + opts.keys().first()); + } + + if (opts.size() > 1) { + throw context.runtime.newArgumentError("unknown keywords: " + opts.keys().join(context, runtime.newString(", "))); + } + } + /** * Efficiently looks up items with a {@link org.jruby.RubySymbol Symbol} key * @param key The Symbol name to look up for @@ -50,7 +67,10 @@ private RuntimeInfo getRuntimeInfo() { * if not found */ IRubyObject get(String key) { - return opts == null ? null : opts.fastARef(runtime.newSymbol(key)); + if (opts == null) { + return null; + } + return opts.delete(runtime.newSymbol(key)); } boolean hasKey(String key) { diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index b2be38d15..59ebda179 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -115,6 +115,8 @@ public IRubyObject initialize(ThreadContext context, IRubyObject options) { this.decimalFactory = this::createCustomDecimal; } + opts.ensureEmpty(); + return this; } diff --git a/lib/json/common.rb b/lib/json/common.rb index 6196aef6c..29ac7aa0d 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -330,7 +330,7 @@ def generate(obj, opts = nil) if State === opts opts.generate(obj) else - State.generate(obj, opts, nil) + State.generate(obj, opts.frozen? ? opts : opts.dup, nil) end end @@ -387,6 +387,7 @@ def pretty_generate(obj, opts = nil) raise TypeError, "can't convert #{opts.class} into Hash" end end + options = options.merge(opts) end @@ -746,6 +747,13 @@ class Coder ).freeze private_constant :PARSER_OPTIONS + EXCLUDED_GENERATOR_OPTIONS = (PARSER_OPTIONS - %i( + max_nesting + allow_nan + allow_duplicate_key + )).freeze + private_constant :EXCLUDED_GENERATOR_OPTIONS + # :call-seq: # JSON.new(options = nil, &block) # @@ -778,8 +786,9 @@ def initialize(object_class: nil, array_class: nil, on_load: nil, **options, &as parser_options[:on_load] = on_load if on_load @parser_config = Ext::Parser::Config.new(parser_options).freeze + generator_options = options.reject { |k, _| EXCLUDED_GENERATOR_OPTIONS.include?(k) } @state = State.new( - **options, + **generator_options, strict: true, as_json: as_json, ).freeze diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 4f033ac58..6dad5e7f4 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -168,7 +168,7 @@ def initialize(opts = nil) @max_nesting = 100 @sort_keys = false @allow_duplicate_key = false - configure(opts) if opts + _configure(**opts) if opts end # This string is used to indent levels in the JSON text. @@ -290,58 +290,48 @@ def strict? # Configure this State instance with the Hash _opts_, and return # itself. - def configure(opts) - if opts.respond_to?(:to_hash) - opts = opts.to_hash - elsif opts.respond_to?(:to_h) - opts = opts.to_h - else - raise TypeError, "can't convert #{opts.class} into Hash" - end - - if opts[:depth]&.negative? - raise ArgumentError, "depth must be >= 0 (got #{opts[:depth]})" + def configure(options) + unless Hash == options + if options.respond_to?(:to_hash) + options = options.to_hash + elsif options.respond_to?(:to_h) + options = options.to_h + end end + _configure(**options) + end + alias merge configure - opts.each do |key, value| - instance_variable_set "@#{key}", value + private def _configure( + indent: '', space: '', space_before: '', object_nl: '', array_nl: '', allow_nan: false, + as_json: false, ascii_only: false, sort_keys: false, depth: 0, buffer_initial_length: 1024, + allow_duplicate_key: false, script_safe: false, strict: false, max_nesting: 100 + ) + if depth.negative? + raise ArgumentError, "depth must be >= 0 (got #{depth})" end - # NOTE: If adding new instance variables here, check whether #generate should check them for #generate_json - @indent = opts[:indent] || '' if opts.key?(:indent) - @space = opts[:space] || '' if opts.key?(:space) - @space_before = opts[:space_before] || '' if opts.key?(:space_before) - @object_nl = opts[:object_nl] || '' if opts.key?(:object_nl) - @array_nl = opts[:array_nl] || '' if opts.key?(:array_nl) - @allow_nan = !!opts[:allow_nan] if opts.key?(:allow_nan) - @as_json = opts[:as_json].to_proc if opts[:as_json] - @ascii_only = opts[:ascii_only] if opts.key?(:ascii_only) - self.sort_keys = opts[:sort_keys] if opts.key?(:sort_keys) - @depth = opts[:depth] || 0 - @buffer_initial_length ||= opts[:buffer_initial_length] - @allow_duplicate_key = !!opts[:allow_duplicate_key] - - @script_safe = if opts.key?(:script_safe) - !!opts[:script_safe] - else - false + if max_nesting && !(Integer === max_nesting) + raise TypeError, ":max_nesting must be `false` or an Integer, got: #{max_nesting.class}" end - @strict = !!opts[:strict] if opts.key?(:strict) - - if !opts.key?(:max_nesting) # defaults to 100 - @max_nesting = 100 - elsif opts[:max_nesting] - unless opts[:max_nesting].is_a?(Integer) - raise TypeError, ":max_nesting must be an Integer, got: #{opts[:max_nesting].class}" - end - @max_nesting = opts[:max_nesting] - else - @max_nesting = 0 - end + @indent = indent || '' + @space = space || '' + @space_before = space_before || '' + @object_nl = object_nl || '' + @array_nl = array_nl || '' + @allow_nan = allow_nan || false + @as_json = as_json || false + @ascii_only = ascii_only || false + self.sort_keys = sort_keys + @depth = depth + @buffer_initial_length = buffer_initial_length + @allow_duplicate_key = allow_duplicate_key + @script_safe = script_safe + @strict = strict + @max_nesting = max_nesting || 0 self end - alias merge configure def allow_duplicate_key? # :nodoc: @allow_duplicate_key diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 9abe7f731..1acfc9ba5 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -63,6 +63,11 @@ def test_parse_unknown_option JSON.parse('[]', quirks_mode: true) end assert_match "quirks_mode", error.message + + error = assert_raise(ArgumentError) do + JSON.parse('[]', a: 1, b: 2) + end + assert_match "a, b", error.message end def test_parse_bang @@ -73,6 +78,18 @@ def test_generate assert_equal '[1,2,3]', JSON.generate([ 1, 2, 3 ]) end + def test_generate_unknown_option + error = assert_raise(ArgumentError) do + JSON.generate([], quirks_mode: true) + end + assert_match "quirks_mode", error.message + + error = assert_raise(ArgumentError) do + JSON.generate([], a: 1, b: 2) + end + assert_match(/unknown keywords: :?a, :?b/, error.message) + end + def test_fast_generate assert_equal '[1,2,3]', JSON.generate([ 1, 2, 3 ]) end diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 7ae415cf5..27882458a 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -322,7 +322,7 @@ def test_state_defaults :space => "", :space_before => "", :sort_keys => false, - }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) + }.sort_by { |n,| n.to_s }.to_h, state.to_h.sort_by { |n,| n.to_s }.to_h) state = JSON::State.new(allow_duplicate_key: true) assert_equal({ From 4ddc092392dfd24b5c7a49e6b69fd4ce2a78a13d Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 10 Aug 2026 16:39:28 +0200 Subject: [PATCH 432/506] Add Ractor compatibility tests for JSON.load and JSON.dump --- test/json/ractor_test.rb | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index e53c405a7..9e28f13c1 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -19,10 +19,18 @@ module RactorBackport end def test_generate + roundtrip_test(:generate, :parse) + end + + def test_dump + roundtrip_test(:dump, :load) + end + + def roundtrip_test(generator_method, parser_method) pid = fork do Warning[:experimental] = false - r = Ractor.new do - json = JSON.generate({ + generator = Ractor.new(generator_method) do |generator_method| + json = JSON.public_send(generator_method, { 'a' => 2, 'b' => 3.141, 'c' => 'c', @@ -34,10 +42,14 @@ def test_generate }) JSON.parse(json) end - expected_json = JSON.parse('{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},' + + + parser = Ractor.new(parser_method) do |parser_method| + JSON.public_send(parser_method, '{"a":2,"b":3.141,"c":"c","d":[1,"b",3.14],"e":{"foo":"bar"},' + '"g":"\\"\\u0000\\u001f","h":1000.0,"i":0.001}') - actual_json = r.value + end + actual_json = generator.value + expected_json = parser.value if expected_json == actual_json exit 0 else From 4816cab1096de26c5e136ab8c5016b5a90004032 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 11 Aug 2026 09:52:45 +0200 Subject: [PATCH 433/506] Get rid of JSON::JSON_LOADED --- lib/json/ext.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 28500f149..da17df1e8 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -66,6 +66,4 @@ def empty? end end end - - JSON_LOADED = true unless defined?(JSON::JSON_LOADED) end From 7924498395ac40eaf42a029ca43996b9e32edbcc Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 11 Aug 2026 09:36:47 +0200 Subject: [PATCH 434/506] Prepare 3.x release notes --- CHANGES.md | 22 +++++++++++++++++++++- README.md | 14 +++----------- lib/json.rb | 5 +++-- lib/json/common.rb | 4 ++-- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index aed16f405..a2a1f5777 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,27 @@ ### Unreleased -* All all unknown options are now cause an `ArgumentError` rather than to be ignored. +With the removal of the insecure `create_additions` option, `JSON.load` and `JSON.dump` are +now safe to use. Them being unsafe by default caused multiple security vulnerabilites in the past. + +If you did depend on `create_additions`, the recommended migration is to [implement a custom serializer using +`JSON::Coder`](https://byroot.github.io/ruby/json/2025/08/02/whats-wrong-with-the-json-gem-api.html#the-create_additions-option). + +All the mutable default options, such as `JSON.load_default_options` have been removed. +They were preventing Ractor compatiblity, and causing bug in libraries using JSON expecting the default behavior. +`JSON` methods now always behave the same unless monkey patched. + +All methods options are now either keyword arguments or checked like keyword arguments, meaning +unknown options such as typos raise `ArgumentError`. + +Duplicated keys are now rejected by default. + +JavaScript comments in documents are no longer supported by default. + +Numerous rarely used aliases have been removed. + +* `JSON.load` defaults are now safe to use. +* All unknown options are now cause an `ArgumentError` rather than to be ignored. * The `allow_comments` parsing option now default to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. * Removed the `limit` positional argument of `JSON.dump`. diff --git a/README.md b/README.md index 79a8c3f3e..ab550ad7c 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,7 @@ UTF-16 surrogate pairs in order to be able to generate the whole range of unicode code points. All strings, that are to be encoded as JSON strings, should be UTF-8 byte -sequences on the Ruby side. To encode raw binary strings, that aren't UTF-8 -encoded, please use the to\_json\_raw\_object method of String (which produces -an object, that contains a byte array) and decode the result on the receiving -endpoint. +sequences on the Ruby side. ## Installation @@ -49,8 +46,7 @@ JSON.generate(data) ``` You can also use the `pretty_generate` method (which formats the output more -verbosely and nicely) or `fast_generate` (which doesn't do any of the security -checks generate performs, e. g. nesting deepness checks). +verbosely and nicely). ## Casting non native types @@ -148,7 +144,7 @@ JSON.generate({ posts: posts_json, count: posts_json.count }) `JSON.generate` always creates the shortest possible string representation of a ruby data structure in one line. This is good for data storage or network protocols, but not so good for humans to read. Fortunately there's also -`JSON.pretty_generate` (or `JSON.pretty_generate`) that creates a more readable +`JSON.pretty_generate` that creates a more readable output: ```ruby @@ -173,10 +169,6 @@ output: ] ``` -There are also the methods `Kernel#j` for generate, and `Kernel#jj` for -`pretty_generate` output to the console, that work analogous to Core Ruby's `p` and -the `pp` library's `pp` methods. - ## Security When parsing or serializing untrusted input, parser and generator options should never be user controlled. diff --git a/lib/json.rb b/lib/json.rb index 9c77544f1..1e7573df9 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -374,8 +374,9 @@ # # Raises SystemStackError: stack level too deep # JSON.generate(obj, max_nesting: false) # -# Setting +max_nesting+ to +false+ can lead to a stackoverflow and may leave the program -# in an unrecoverable state. It is discouraged. +# Setting +max_nesting+ to +false+ or a very large number can lead to a stack overflow +# which may leave the process in an unrecoverable state. +# It is highly discouraged. # # ====== Escaping Options # diff --git a/lib/json/common.rb b/lib/json/common.rb index 29ac7aa0d..0512d4aef 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -241,8 +241,8 @@ def to_json(state = nil, *) # --- # # Raises an exception if +source+ is not valid JSON: - # # Raises JSON::ParserError (783: unexpected token at ''): - # JSON.parse('') + # # Raises JSON::ParserError unexpected character: 'invalid' at line 1 column 1 : + # JSON.parse('invalid') # def parse(source, on_load: nil, object_class: nil, array_class: nil, **options) if object_class || array_class From 4f89162e8bc11ec7d6088ac6b7e8f6f694d84ca1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 11 Aug 2026 10:00:30 +0200 Subject: [PATCH 435/506] Relase 3.0.0.rc1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a2a1f5777..b842cb86d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-08-11 (3.0.0.rc1) + With the removal of the insecure `create_additions` option, `JSON.load` and `JSON.dump` are now safe to use. Them being unsafe by default caused multiple security vulnerabilites in the past. diff --git a/lib/json/version.rb b/lib/json/version.rb index 5c66a9d70..8e7e31755 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '2.21.2' + VERSION = '3.0.0.rc1' end From 38c9ab062acc2ed86f8ff33189d65b15dfc79d77 Mon Sep 17 00:00:00 2001 From: Brandon Hicks Date: Tue, 11 Aug 2026 09:00:27 -0700 Subject: [PATCH 436/506] Update CHANGES.md for clarity and consistency --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b842cb86d..e1cd34a86 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,8 +24,8 @@ JavaScript comments in documents are no longer supported by default. Numerous rarely used aliases have been removed. * `JSON.load` defaults are now safe to use. -* All unknown options are now cause an `ArgumentError` rather than to be ignored. -* The `allow_comments` parsing option now default to `false`. +* All unknown options will now cause an `ArgumentError` rather than to be ignored. +* The `allow_comments` parsing option now defaults to `false`. * The `allow_duplicate_key` option now defaults to `false`, for both parsing and generating JSON. * Removed the `limit` positional argument of `JSON.dump`. * Removed the `escape_slash` alias of `script_safe`. From 4a7a53b2125bf66604764aec7f7965a6cf9f8b5a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 17 Aug 2026 08:59:10 +0200 Subject: [PATCH 437/506] Get rid of tools/fuzz.rb It could easily be fixed by changing Fixnum/Bignum to Integer, but it's also a very basic fuzzer that isn't very liekly to find bugs. I'd welcome a proper fuzzer, but I don't think this one is worth keeping. --- tools/fuzz.rb | 131 -------------------------------------------------- 1 file changed, 131 deletions(-) delete mode 100755 tools/fuzz.rb diff --git a/tools/fuzz.rb b/tools/fuzz.rb deleted file mode 100755 index 826193019..000000000 --- a/tools/fuzz.rb +++ /dev/null @@ -1,131 +0,0 @@ -require 'json' - -class Fuzzer - def initialize(n, freqs = {}) - sum = freqs.inject(0.0) { |s, x| s + x.last } - freqs.each_key { |x| freqs[x] /= sum } - s = 0.0 - freqs.each_key do |x| - freqs[x] = s .. (s + t = freqs[x]) - s += t - end - @freqs = freqs - @n = n - @alpha = (0..0xff).to_a - end - - def random_string - s = '' - 30.times { s << @alpha[rand(@alpha.size)] } - s - end - - def pick - r = rand - found = @freqs.find { |k, f| f.include? rand } - found && found.first - end - - def make_pick - k = pick - case - when k == Hash, k == Array - k.new - when k == true, k == false, k == nil - k - when k == String - random_string - when k == Fixnum - rand(2 ** 30) - 2 ** 29 - when k == Bignum - rand(2 ** 70) - 2 ** 69 - end - end - - def fuzz(current = nil) - if @n > 0 - case current - when nil - @n -= 1 - current = fuzz [ Hash, Array ][rand(2)].new - when Array - while @n > 0 - @n -= 1 - current << case p = make_pick - when Array, Hash - fuzz(p) - else - p - end - end - when Hash - while @n > 0 - @n -= 1 - current[random_string] = case p = make_pick - when Array, Hash - fuzz(p) - else - p - end - end - end - end - current - end -end - -class MyState < JSON.state - WS = " \r\t\n" - - def initialize - super( - :indent => make_spaces, - :space => make_spaces, - :space_before => make_spaces, - :object_nl => make_spaces, - :array_nl => make_spaces, - :max_nesting => false - ) - end - - def make_spaces - s = '' - rand(1).times { s << WS[rand(WS.size)] } - s - end -end - -n = (ARGV.shift || 500).to_i -loop do - fuzzer = Fuzzer.new(n, - Hash => 25, - Array => 25, - String => 10, - Fixnum => 10, - Bignum => 10, - nil => 5, - true => 5, - false => 5 - ) - o1 = fuzzer.fuzz - json = JSON.generate o1, MyState.new - if $DEBUG - puts "-" * 80 - puts json, json.size - else - puts json.size - end - begin - o2 = JSON.parse(json, :max_nesting => false) - rescue JSON::ParserError => e - puts "Caught #{e.class}: #{e.message}\n#{e.backtrace * "\n"}" - puts "o1 = #{o1.inspect}", "json = #{json}", "json_str = #{json.inspect}" - puts "locals = #{local_variables.inspect}" - exit - end - if o1 != o2 - puts "mismatch", "o1 = #{o1.inspect}", "o2 = #{o2.inspect}", - "json = #{json}", "json_str = #{json.inspect}" - puts "locals = #{local_variables.inspect}" - end -end From 0479b66aec3fd3625f82bbe4d25db0a0421b71e4 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 17 Aug 2026 09:00:51 +0200 Subject: [PATCH 438/506] Update Hash syntax in tests --- Rakefile | 24 +++--- test/json/json_common_interface_test.rb | 26 +++--- test/json/json_generator_test.rb | 110 ++++++++++++------------ test/json/json_parser_test.rb | 44 +++++----- 4 files changed, 102 insertions(+), 102 deletions(-) diff --git a/Rakefile b/Rakefile index c11d7b8d0..0416411a6 100644 --- a/Rakefile +++ b/Rakefile @@ -143,10 +143,10 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' end desc "Compiling jruby extension" - task :compile => JAVA_CLASSES + task compile: JAVA_CLASSES desc "Package the jruby gem" - task :jruby_gem => :create_jar do + task jruby_gem: :create_jar do mkdir_p 'pkg' sh "gem build -o pkg/json-#{PKG_VERSION}-java.gem json.gemspec" end @@ -158,7 +158,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' t.options = '-v' end desc "Testing library (jruby)" - task :test => [:create_jar ] + task test: [:create_jar ] file JRUBY_PARSER_JAR => :compile do cd 'java/src' do @@ -178,7 +178,7 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' end desc "Create parser jar" - task :create_parser_jar => JRUBY_PARSER_JAR + task create_parser_jar: JRUBY_PARSER_JAR file JRUBY_GENERATOR_JAR => :compile do cd 'java/src' do @@ -198,15 +198,15 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby' end desc "Create generator jar" - task :create_generator_jar => JRUBY_GENERATOR_JAR + task create_generator_jar: JRUBY_GENERATOR_JAR desc "Create parser and generator jars" - task :create_jar => [ :create_parser_jar, :create_generator_jar ] + task create_jar: [ :create_parser_jar, :create_generator_jar ] desc "Build all gems and archives for a new release of the jruby extension." - task :build => [ :clean, :jruby_gem ] + task build: [ :clean, :jruby_gem ] - task :release => :build + task release: :build else require 'rake/extensiontask' @@ -224,7 +224,7 @@ else end desc "Testing library (extension)" - task :test => [ :compile ] + task test: [ :compile ] begin require "ruby_memcheck" @@ -249,10 +249,10 @@ else end desc "Build all gems and archives for a new release of json" - task :build => [ :clean, :package ] + task build: [ :clean, :package ] - task :release => :build + task release: :build end desc "Compile in the the source directory" -task :default => [ :clean, :test ] +task default: [ :clean, :test ] diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 1acfc9ba5..2c4c9a213 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -158,16 +158,16 @@ def test_load_with_proc def test_load_with_options json = '{ "foo": NaN }' - assert JSON.load(json, nil, :allow_nan => true)['foo'].nan? - assert JSON.load(json, :allow_nan => true)['foo'].nan? + assert JSON.load(json, nil, allow_nan: true)['foo'].nan? + assert JSON.load(json, allow_nan: true)['foo'].nan? end def test_load_null - assert_equal nil, JSON.load(nil, nil, :allow_blank => true) - assert_raise(TypeError) { JSON.load(nil, nil, :allow_blank => false) } - assert_raise(JSON::ParserError) { JSON.load('', nil, :allow_blank => false) } - assert_raise(TypeError) { JSON.load([], nil, :allow_blank => true) } - assert_raise(TypeError) { JSON.load({}, nil, :allow_blank => true) } + assert_equal nil, JSON.load(nil, nil, allow_blank: true) + assert_raise(TypeError) { JSON.load(nil, nil, allow_blank: false) } + assert_raise(JSON::ParserError) { JSON.load('', nil, allow_blank: false) } + assert_raise(TypeError) { JSON.load([], nil, allow_blank: true) } + assert_raise(TypeError) { JSON.load({}, nil, allow_blank: true) } end def test_unsafe_load @@ -240,16 +240,16 @@ def test_unsafe_load_default_options def test_unsafe_load_with_options nan_json = '{ "foo": NaN }' - assert_raise(JSON::ParserError) { JSON.unsafe_load(nan_json, nil, :allow_nan => false)['foo'].nan? } + assert_raise(JSON::ParserError) { JSON.unsafe_load(nan_json, nil, allow_nan: false)['foo'].nan? } # make sure it still uses the defaults when something is provided - assert JSON.unsafe_load(nan_json, nil, :allow_blank => true)['foo'].nan? - assert JSON.unsafe_load(nan_json, :allow_nan => true)['foo'].nan? + assert JSON.unsafe_load(nan_json, nil, allow_blank: true)['foo'].nan? + assert JSON.unsafe_load(nan_json, allow_nan: true)['foo'].nan? end def test_unsafe_load_null - assert_equal nil, JSON.unsafe_load(nil, nil, :allow_blank => true) - assert_raise(TypeError) { JSON.unsafe_load(nil, nil, :allow_blank => false) } - assert_raise(JSON::ParserError) { JSON.unsafe_load('', nil, :allow_blank => false) } + assert_equal nil, JSON.unsafe_load(nil, nil, allow_blank: true) + assert_raise(TypeError) { JSON.unsafe_load(nil, nil, allow_blank: false) } + assert_raise(JSON::ParserError) { JSON.unsafe_load('', nil, allow_blank: false) } end def test_dump diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 27882458a..2a64a5e75 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -175,7 +175,7 @@ def test_generate_pretty end def test_generate_pretty_custom - state = State.new(:space_before => "", :space => "", :indent => "", :object_nl => "\n\n", :array_nl => "") + state = State.new(space_before: "", space: "", indent: "", object_nl: "\n\n", array_nl: "") json = pretty_generate({1=>{}, 2=>['a','b'], 3=>4}, state) assert_equal(<<~'JSON'.chomp, json) { @@ -244,7 +244,7 @@ def test_generate_sort_keys_with_proc end def test_generate_custom - state = State.new(:space_before => " ", :space => " ", :indent => "", :object_nl => "\n", :array_nl => "") + state = State.new(space_before: " ", space: " ", indent: "", object_nl: "\n", array_nl: "") json = generate({1=>{2=>3,4=>[5,6]}}, state) assert_equal(<<~'JSON'.chomp, json) { @@ -307,62 +307,62 @@ def test_falsy_state def test_state_defaults state = JSON::State.new assert_equal({ - :allow_duplicate_key => false, - :allow_nan => false, - :array_nl => "", - :as_json => false, - :ascii_only => false, - :buffer_initial_length => 1024, - :depth => 0, - :script_safe => false, - :strict => false, - :indent => "", - :max_nesting => 100, - :object_nl => "", - :space => "", - :space_before => "", - :sort_keys => false, + allow_duplicate_key: false, + allow_nan: false, + array_nl: "", + as_json: false, + ascii_only: false, + buffer_initial_length: 1024, + depth: 0, + script_safe: false, + strict: false, + indent: "", + max_nesting: 100, + object_nl: "", + space: "", + space_before: "", + sort_keys: false, }.sort_by { |n,| n.to_s }.to_h, state.to_h.sort_by { |n,| n.to_s }.to_h) state = JSON::State.new(allow_duplicate_key: true) assert_equal({ - :allow_duplicate_key => true, - :allow_nan => false, - :array_nl => "", - :as_json => false, - :ascii_only => false, - :buffer_initial_length => 1024, - :depth => 0, - :script_safe => false, - :strict => false, - :indent => "", - :max_nesting => 100, - :object_nl => "", - :space => "", - :space_before => "", - :sort_keys => false, + allow_duplicate_key: true, + allow_nan: false, + array_nl: "", + as_json: false, + ascii_only: false, + buffer_initial_length: 1024, + depth: 0, + script_safe: false, + strict: false, + indent: "", + max_nesting: 100, + object_nl: "", + space: "", + space_before: "", + sort_keys: false, }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end def test_allow_nan error = assert_raise(GeneratorError) { generate([JSON::NaN]) } assert_same JSON::NaN, error.invalid_object - assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true) + assert_equal '[NaN]', generate([JSON::NaN], allow_nan: true) assert_raise(GeneratorError) { generate([JSON::NaN]) } assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) } - assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true) + assert_equal "[\n NaN\n]", pretty_generate([JSON::NaN], allow_nan: true) error = assert_raise(GeneratorError) { generate([JSON::Infinity]) } assert_same JSON::Infinity, error.invalid_object - assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true) + assert_equal '[Infinity]', generate([JSON::Infinity], allow_nan: true) assert_raise(GeneratorError) { generate([JSON::Infinity]) } assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) } - assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true) + assert_equal "[\n Infinity\n]", pretty_generate([JSON::Infinity], allow_nan: true) error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } assert_same JSON::MinusInfinity, error.invalid_object - assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true) + assert_equal '[-Infinity]', generate([JSON::MinusInfinity], allow_nan: true) assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) } assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) } - assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], :allow_nan => true) + assert_equal "[\n -Infinity\n]", pretty_generate([JSON::MinusInfinity], allow_nan: true) end # An object that changes state.depth when it receives to_json(state) @@ -489,11 +489,11 @@ def test_gc def test_configure_using_configure_and_merge numbered_state = { - :indent => "1", - :space => '2', - :space_before => '3', - :object_nl => '4', - :array_nl => '5' + indent: "1", + space: '2', + space_before: '3', + object_nl: '4', + array_nl: '5' } state1 = JSON.state.new state1.merge(numbered_state) @@ -513,7 +513,7 @@ def test_configure_using_configure_and_merge def test_configure_hash_conversion state = JSON.state.new - state.configure(:indent => '1') + state.configure(indent: '1') assert_equal '1', state.indent state = JSON.state.new foo = 'foo'.dup @@ -595,14 +595,14 @@ def test_nesting too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' too_deep_ary = eval too_deep assert_raise(JSON::NestingError) { generate too_deep_ary } - assert_raise(JSON::NestingError) { generate too_deep_ary, :max_nesting => 100 } - ok = generate too_deep_ary, :max_nesting => 101 + assert_raise(JSON::NestingError) { generate too_deep_ary, max_nesting: 100 } + ok = generate too_deep_ary, max_nesting: 101 assert_equal too_deep, ok - ok = generate too_deep_ary, :max_nesting => nil + ok = generate too_deep_ary, max_nesting: nil assert_equal too_deep, ok - ok = generate too_deep_ary, :max_nesting => false + ok = generate too_deep_ary, max_nesting: false assert_equal too_deep, ok - ok = generate too_deep_ary, :max_nesting => 0 + ok = generate too_deep_ary, max_nesting: 0 assert_equal too_deep, ok assert_raise(TypeError) { generate too_deep_ary, max_nesting: "garbage" } @@ -631,27 +631,27 @@ def test_backslash # data = [ '/' ] json = '["\/"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = [ '///////////' ] json = '["\/\/\/\/\/\/\/\/\/\/\/"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = [ '///////////////////////////////////////////////////////' ] json = '["\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = [ "\u2028\u2029" ] json = '["\u2028\u2029"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = [ "ABC \u2028 DEF \u2029 GHI" ] json = '["ABC \u2028 DEF \u2029 GHI"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = [ "/\u2028\u2029" ] json = '["\/\u2028\u2029"]' - assert_equal json, generate(data, :script_safe => true) + assert_equal json, generate(data, script_safe: true) # data = ['"'] json = '["\""]' diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 1819d0abf..e7940faa4 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -120,11 +120,11 @@ def test_parse_numbers assert_equal_float(-3.141, parse('-3141.0e-3')) assert_equal_float(-3.141, parse('-3141e-3')) assert_raise(ParserError) { parse('NaN') } - assert parse('NaN', :allow_nan => true).nan? + assert parse('NaN', allow_nan: true).nan? assert_raise(ParserError) { parse('Infinity') } - assert_equal(1.0/0, parse('Infinity', :allow_nan => true)) + assert_equal(1.0/0, parse('Infinity', allow_nan: true)) assert_raise(ParserError) { parse('-Infinity') } - assert_equal(-1.0/0, parse('-Infinity', :allow_nan => true)) + assert_equal(-1.0/0, parse('-Infinity', allow_nan: true)) capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) } end @@ -258,9 +258,9 @@ def test_parse_json_primitive_values assert_equal_float 3.141, parse('3.141'), 1E-3 assert_equal 2 ** 64, parse('18446744073709551616') assert_equal 'foo', parse('"foo"') - assert parse('NaN', :allow_nan => true).nan? - assert parse('Infinity', :allow_nan => true).infinite? - assert parse('-Infinity', :allow_nan => true).infinite? + assert parse('NaN', allow_nan: true).nan? + assert parse('Infinity', allow_nan: true).infinite? + assert parse('-Infinity', allow_nan: true).infinite? end def test_parse_arrays_with_allow_trailing_comma @@ -471,17 +471,17 @@ def test_some_wrong_inputs def test_symbolize_names assert_equal({ "foo" => "bar", "baz" => "quux" }, parse('{"foo":"bar", "baz":"quux"}')) - assert_equal({ :foo => "bar", :baz => "quux" }, - parse('{"foo":"bar", "baz":"quux"}', :symbolize_names => true)) + assert_equal({ foo: "bar", baz: "quux" }, + parse('{"foo":"bar", "baz":"quux"}', symbolize_names: true)) end def test_freeze - assert_predicate parse('{}', :freeze => true), :frozen? - assert_predicate parse('[]', :freeze => true), :frozen? - assert_predicate parse('"foo"', :freeze => true), :frozen? + assert_predicate parse('{}', freeze: true), :frozen? + assert_predicate parse('[]', freeze: true), :frozen? + assert_predicate parse('"foo"', freeze: true), :frozen? - assert_same(-'foo', parse('"foo"', :freeze => true)) - assert_same(-'foo', parse('{"foo": 1}', :freeze => true).keys.first) + assert_same(-'foo', parse('"foo"', freeze: true)) + assert_same(-'foo', parse('{"foo": 1}', freeze: true).keys.first) end def test_parse_comments @@ -544,14 +544,14 @@ def test_nesting too_deep = '[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]' too_deep_ary = eval too_deep assert_raise(JSON::NestingError) { parse too_deep } - assert_raise(JSON::NestingError) { parse too_deep, :max_nesting => 100 } - ok = parse too_deep, :max_nesting => 101 + assert_raise(JSON::NestingError) { parse too_deep, max_nesting: 100 } + ok = parse too_deep, max_nesting: 101 assert_equal too_deep_ary, ok - ok = parse too_deep, :max_nesting => nil + ok = parse too_deep, max_nesting: nil assert_equal too_deep_ary, ok - ok = parse too_deep, :max_nesting => false + ok = parse too_deep, max_nesting: false assert_equal too_deep_ary, ok - ok = parse too_deep, :max_nesting => 0 + ok = parse too_deep, max_nesting: 0 assert_equal too_deep_ary, ok end @@ -702,14 +702,14 @@ def shifted? end def test_parse_array_custom_array_derived_class - res = parse('[1,2]', :array_class => SubArray) + res = parse('[1,2]', array_class: SubArray) assert_equal([1,2], res) assert_equal(SubArray, res.class) assert res.shifted? end def test_parse_array_custom_non_array_derived_class - res = parse('[1,2]', :array_class => SubArrayWrapper) + res = parse('[1,2]', array_class: SubArrayWrapper) assert_equal([1,2], res.data) assert_equal(1, res[0]) assert_equal(SubArrayWrapper, res.class) @@ -748,7 +748,7 @@ def self.json_create(o) end def test_parse_object_custom_hash_derived_class - res = parse('{"foo":"bar"}', :object_class => SubHash) + res = parse('{"foo":"bar"}', object_class: SubHash) assert_equal({"foo" => "bar"}, res) assert_equal(SubHash, res.class) assert res.item_set? @@ -775,7 +775,7 @@ def method_missing(name, ...) end def test_parse_object_custom_non_hash_derived_class - res = parse('{"foo":"bar"}', :object_class => OpenStructLike) + res = parse('{"foo":"bar"}', object_class: OpenStructLike) assert_equal "bar", res.foo assert_equal "bar", res[:foo] assert_equal(OpenStructLike, res.class) From 9ae76bdd3dd4759892265086e77a319810ff74db Mon Sep 17 00:00:00 2001 From: Ojus Chugh <79078267+ojuschugh1@users.noreply.github.com> Date: Mon, 24 Aug 2026 20:33:36 +0530 Subject: [PATCH 439/506] Add JSON::ParserError#json_path (#1062) Add JSON::ParserError#json_path Co-authored-by: Jean Boussier --- CHANGES.md | 2 + ext/json/ext/parser/parser.c | 53 ++++++++++++++++++---- lib/json/common.rb | 51 ++++++++++++++++++++- test/json/json_parser_test.rb | 85 +++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1cd34a86..5b4bb0199 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Add `JSON::ParserError#json_path` to locate parse errors in the document as a JSONPath-style string (e.g. `$.foo[0].bar`). For duplicate key errors it points at the duplicated key itself. + ### 2026-08-11 (3.0.0.rc1) With the removal of the insecure `create_additions` option, `JSON.load` and `JSON.dump` are diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index f10e2260a..fc6a7bbf6 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -5,7 +5,7 @@ static VALUE mJSON, eNestingError, eParserError, Encoding_UTF_8; static VALUE CNaN, CInfinity, CMinusInfinity, JSON_empty_string; -static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column; +static ID i_new, i_try_convert, i_encode, i_at_line, i_at_column, i_at_json_path; #ifndef HAVE_RB_STR_TO_INTERNED_STR static ID i_uminus; #endif @@ -651,11 +651,42 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta return rb_enc_sprintf(enc_utf8, format, ptr); } +static VALUE json_path_new(JSON_ParserState *state, VALUE duplicate_key) +{ + VALUE path = rb_ary_new_capa(state->current_nesting); + + json_frame_stack *frames = state->frames; + rvalue_stack *values = state->value_stack; + + for (long depth = 1; depth < frames->head; depth++) { + json_frame *frame = &frames->ptr[depth]; + + bool innermost = depth == frames->head - 1; + long child_head = innermost ? values->head : frames->ptr[depth + 1].value_stack_head; + long count = child_head - frame->value_stack_head; + + if (frame->type == JSON_FRAME_ARRAY) { + rb_ary_push(path, LONG2NUM(frame->phase == JSON_PHASE_ARRAY_COMMA ? count - 1 : count)); + } else if (innermost && !UNDEF_P(duplicate_key)) { + rb_ary_push(path, duplicate_key); + } else if (count & 1) { + rb_ary_push(path, values->ptr[child_head - 1]); + } else if (frame->phase == JSON_PHASE_OBJECT_COMMA && count >= 2) { + rb_ary_push(path, values->ptr[child_head - 2]); + } else { + break; + } + } + + return path; +} + static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos) { VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); rb_ivar_set(exc, i_at_column, LONG2NUM(column)); + rb_ivar_set(exc, i_at_json_path, json_path_new(state, Qundef)); return exc; } @@ -1199,14 +1230,17 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d ); rb_str_concat(message, build_parse_error_message("", state)); + VALUE exc; if (state->parser) { // line and columns can't be accurate in resumable - rb_exc_raise(parse_error_new(state, message, 0, 0, false)); + exc = parse_error_new(state, message, 0, 0, false); } else { long line, column; cursor_position(state, &line, &column); rb_str_catf(message, " at line %ld column %ld", line, column); - rb_exc_raise(parse_error_new(state, message, line, column, false)); + exc = parse_error_new(state, message, line, column, false); } + rb_ivar_set(exc, i_at_json_path, json_path_new(state, duplicate_key)); + rb_exc_raise(exc); } NOINLINE(static) void json_on_duplicate_key(JSON_ParserState *state, JSON_ParserConfig *config, size_t count, const VALUE *pairs) @@ -2122,6 +2156,12 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) // the rvalue stack. VALUE result = complete ? *rvalue_stack_peek(state->value_stack, 1) : Qundef; + if (complete) { + json_ensure_eof(state, config); + } else { + raise_eos_error("unexpected end of input", state); + } + // This may be skipped in case of exception, but // it won't cause a leak. rvalue_stack_eagerly_release(value_stack_handle); @@ -2130,12 +2170,6 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) RB_GC_GUARD(frame_stack_handle); RB_GC_GUARD(Vsource); - if (complete) { - json_ensure_eof(state, config); - } else { - raise_eos_error("unexpected end of input", state); - } - return result; } @@ -2871,6 +2905,7 @@ void Init_parser(void) i_encode = rb_intern("encode"); i_at_line = rb_intern("@line"); i_at_column = rb_intern("@column"); + i_at_json_path = rb_intern("@json_path"); binary_encindex = rb_ascii8bit_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/lib/json/common.rb b/lib/json/common.rb index 0512d4aef..34dc123b5 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -142,7 +142,56 @@ class JSONError < StandardError; end # This exception is raised if a parser error occurs. class ParserError < JSONError - attr_reader :line, :column + # Line number where the parser encountered an error. + # Is nil when raised by JSON::ResumableParser. + attr_reader :line + + # Column number where the parser encountered an error. + # Is nil when raised by JSON::ResumableParser. + attr_reader :column + + # Returns a best effort JSONPath string representing where in the document + # the parser encountered an error: + # + # begin + # JSON.parse('{"articles": [ { "title": invalid } ]}') + # rescue JSON::ParserError => error + # error.json_path # => "$.articles[0].title" + # end + def json_path + return @json_path if String === @json_path + + if Array === @json_path + path = build_json_path(@json_path) + @json_path = path unless frozen? + return path + end + end + + private + + def build_json_path(segments) + error = false + path = segments.filter_map do |segment| + next if error + + case segment + when Integer + "[#{segment}]" + when String, Symbol + if segment.match?(/\A[a-zA-Z\$\_][a-zA-Z\$\_0-9]*\z/) + ".#{segment}" + else + segment = segment.to_s.gsub(/["\\]/, { '"' => '\\"', '\\' => '\\\\' }) + %{["#{segment}"]} + end + else + error = true + nil + end + end.join + "$#{path}".freeze + end end # This exception is raised if the nesting of parsed data structures is too diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index e7940faa4..84839585f 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -847,6 +847,80 @@ def test_parse_error_snippet assert_equal "unexpected character: '@' at line 1 column 1", error.message end + def test_parse_error_json_path + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + assert_parse_error_at "$", "xyz" + assert_parse_error_at "$.a", '{"a": xyz}' + assert_parse_error_at "$[3]", '[1, 2, "hi", xyz]' + assert_parse_error_at "$.a[1].b", '{"a": [1, {"b": xyz}]}' + assert_parse_error_at "$.a", '{"a": 1 xyz}' + assert_parse_error_at "$", '{"a": 1, xyz}' + + assert_parse_error_at "$.a.b.c", '{"a": {"b": {"c"' + assert_parse_error_at "$.a.b.c", '{"a": {"b": {"c":' + assert_parse_error_at "$.a.b", '{"a": {"b": {"c": 1, "d' + + assert_parse_error_at "$[4]", '[1,2,3,4,5' + assert_parse_error_at "$[5]", '[1,2,3,4,5,' + assert_parse_error_at "$[5]", '[1,2,3,4,5,]' + end + + def test_parse_error_json_path_on_load + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + assert_parse_error_at "$" do + JSON.load('{"a": {"b": {"c":', -> (obj) { + if String === obj + BasicObject.new + else + obj + end + }) + end + + assert_parse_error_at "$.a" do + JSON.load('{"a": {"b": {"c":', -> (obj) { + if obj == "b" + BasicObject.new + else + obj + end + }) + end + end + + def test_parse_error_json_path_key_escaping + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + assert_parse_error_at '$["hello world"]', '{"hello world": xyz}' + assert_parse_error_at '$["a\"b"]', '{"a\"b": xyz}' + assert_parse_error_at '$[""]', '{"": xyz}' + assert_parse_error_at '$["あ"]', '{"あ": xyz}' + assert_parse_error_at '$.foo["1x"]', '{"foo": {"1x": xyz}}' + end + + def test_parse_error_json_path_duplicate_key + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + assert_parse_error_at "$.a", '{"a": 1, "a": 2}' + assert_parse_error_at "$.x.a", '{"x": {"a": 1, "b": 2, "a": 3}}' + assert_parse_error_at "$.arr[0].a", '{"arr": [{"a": 1, "a": 2}]}' + assert_parse_error_at "$.x.a", '{"x": {"a": 1, "a": 2}}' + end + + def test_parse_error_json_path_resumable + omit "JSON::ResumableParser not available" unless defined?(JSON::ResumableParser) + + parser = JSON::ResumableParser.new + parser << '{"a": [1, {"b": ' + parser.parse + assert_parse_error_at "$.a[1].b" do + parser << 'xyz' + parser.parse + end + end + def test_parse_leading_slash # ref: https://github.com/ruby/ruby/pull/12598 assert_raise(JSON::ParserError) do @@ -888,4 +962,15 @@ def assert_equal_float(expected, actual, delta = 1e-2) Array === actual and actual = actual.first assert_in_delta(expected, actual, delta) end + + def assert_parse_error_at(path, json = nil) + error = assert_raise(JSON::ParserError) do + if block_given? + yield + else + JSON.parse(json) + end + end + assert_equal path, error.json_path + end end From d3c71136ca569ddc92fac5fae5dac144f21a3a13 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 27 Aug 2026 15:49:51 +0900 Subject: [PATCH 440/506] Update parser dependencies Remove the stale fbuffer dependency left after parser float conversion stopped using it. --- ext/json/ext/parser/depend | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/json/ext/parser/depend b/ext/json/ext/parser/depend index 53aa89b70..c46071561 100644 --- a/ext/json/ext/parser/depend +++ b/ext/json/ext/parser/depend @@ -1,5 +1,4 @@ # AUTOGENERATED DEPENDENCIES START -parser.o: $(srcdir)/../fbuffer/fbuffer.h parser.o: $(srcdir)/../json.h parser.o: $(srcdir)/../simd/simd.h parser.o: $(srcdir)/../vendor/fast_float_parser.h From 2d73e6bdfe13503653307568708020e7027b2ff6 Mon Sep 17 00:00:00 2001 From: Shizuo Fujita Date: Thu, 3 Sep 2026 14:53:08 +0900 Subject: [PATCH 441/506] Fix SEGV when as_json returns a non-String for an invalid symbol In strict mode, a Symbol whose name is not UTF-8 compatible was passed to generate_json_string, which handed the as_json result to raw_generate_json_string without checking its type. A non-String result was then read as a String, segfaulting the process. Route strict symbols through the T_STRING case of generate_json_general so they reuse its as_json handling. That re-dispatches the result by type, and its as_json_called flag stops a symbol returned by as_json from recursing. Co-Authored-By: Claude Opus 5 (1M context) --- ext/json/ext/generator/generator.c | 26 +++++++++++++++----------- test/json/json_coder_test.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 38607cf4e..d0d048751 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1124,15 +1124,6 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d } } -static inline void generate_json_symbol(FBuffer *buffer, struct generate_json_data *data, VALUE obj) -{ - if (data->state->strict) { - generate_json_string(buffer, data, rb_sym2str(obj)); - } else { - generate_json_fallback(buffer, data, obj); - } -} - static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { fbuffer_append(buffer, "null", 4); @@ -1218,7 +1209,13 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d } else if (RB_FLONUM_P(obj)) { generate_json_float(buffer, data, obj); } else if (RB_STATIC_SYM_P(obj)) { - generate_json_symbol(buffer, data, obj); + if (data->state->strict) { + obj = rb_sym2str(obj); + JSON_ASSERT(RBASIC_CLASS(obj) == rb_cString); + goto generate_string; + } + + generate_json_fallback(buffer, data, obj); } else { goto general; } @@ -1239,6 +1236,7 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d case T_STRING: if (fallback && klass != rb_cString) goto general; + generate_string: if (RB_LIKELY(valid_json_string_p(obj))) { raw_generate_json_string(buffer, data, obj); } else if (as_json_called) { @@ -1250,7 +1248,13 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d } break; case T_SYMBOL: - generate_json_symbol(buffer, data, obj); + if (data->state->strict) { + obj = rb_sym2str(obj); + JSON_ASSERT(RBASIC_CLASS(obj) == rb_cString); + goto generate_string; + } + + generate_json_fallback(buffer, data, obj); break; case T_FLOAT: if (fallback && klass != rb_cFloat) goto general; diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index a8477dd7b..7be1dfb3f 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -137,6 +137,33 @@ def test_json_coder_string_invalid_encoding assert_equal 2, calls end + def test_json_coder_symbol_invalid_encoding + symbol = "\xFF".b.to_sym + + calls = 0 + coder = JSON::Coder.new do |object, is_key| + calls += 1 + object.bytes + end + + assert_equal "[[255]]", coder.dump([symbol]) + assert_equal 1, calls + + coder = JSON::Coder.new { |object, is_key| nil } + assert_equal "[null]", coder.dump([symbol]) + + coder = JSON::Coder.new { |object, is_key| Object.new } + assert_raise(JSON::GeneratorError) { coder.dump([symbol]) } + + if RUBY_ENGINE == "ruby" + coder = JSON::Coder.new { |object, is_key| symbol } + error = assert_raise JSON::GeneratorError do + coder.dump([symbol]) + end + assert_equal "source sequence is illegal/malformed utf-8", error.message + end + end + def test_depth coder = JSON::Coder.new(object_nl: "\n", array_nl: "\n", space: " ", indent: " ", depth: 1) assert_equal %({\n "foo": 42\n }), coder.dump(foo: 42) From 1d37403f4be5b62e2a112aad3226e53ea83677ee Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 5 Sep 2026 09:34:05 +0200 Subject: [PATCH 442/506] Stop installing ragel on CI The JRuby extension no longer needs it. --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a82e21dd0..d98aa5715 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,6 @@ jobs: with: bundler-cache: true ruby-version: ${{ matrix.ruby }} - apt-get: "${{ startsWith(matrix.ruby, 'jruby') && 'ragel' || '' }}" - brew: "${{ startsWith(matrix.ruby, 'jruby') && 'ragel' || '' }}" - run: bundle exec rake compile ${{ matrix.env }} From 82bf6bedcf3d359adbcb7a82ed99521aa89d695a Mon Sep 17 00:00:00 2001 From: Elankumaran Srinivasan <5340827+elang2@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:43:07 -0700 Subject: [PATCH 443/506] Fix parser silently accepting a lone trailing surrogate in string values The C parser guards leading-first surrogate pairs but silently accepts a lone trailing surrogate, encoding U+DC00..U+DFFF as three UTF-8 bytes into the output buffer. The returned Ruby String is tagged UTF-8 but fails valid_encoding? and raises misleading errors from downstream String operations (upcase, split, regex, encode, JSON.generate). Add a symmetric branch alongside the leading-surrogate check to raise JSON::ParserError at the parse boundary. Extend test_invalid_surogates with the new cases and move the three JSONTestSuite fixtures that this fix newly rejects from INVALID_ENCODING_TESTS into UNDEFINED_FAILING, closing the CRuby/JRuby parser parity gap documented in the file. --- CHANGES.md | 1 + ext/json/ext/parser/parser.c | 3 +++ test/json/json_minefield_parser_test.rb | 6 +++--- test/json/json_parser_test.rb | 9 +++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5b4bb0199..ea9765ef4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ### Unreleased * Add `JSON::ParserError#json_path` to locate parse errors in the document as a JSONPath-style string (e.g. `$.foo[0].bar`). For duplicate key errors it points at the duplicated key itself. +* Fix the parser to also reject lone trailing UTF-16 surrogates (`\uDCxx` with no leading partner), symmetric to the leading-surrogate case. The Java parser already rejected these; this closes the CRuby/JRuby parity gap. ### 2026-08-11 (3.0.0.rc1) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index fc6a7bbf6..5cba2feb6 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1041,6 +1041,9 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser raise_syntax_error_at("incomplete surrogate pair at %s", state, p); break; } + } else if ((ch & 0xFC00) == 0xDC00) { + raise_syntax_error_at("unpaired trailing surrogate at %s", state, p); + break; } int unescape_len = convert_UTF32_to_UTF8(buffer, ch); diff --git a/test/json/json_minefield_parser_test.rb b/test/json/json_minefield_parser_test.rb index e6dcb54b8..555801409 100644 --- a/test/json/json_minefield_parser_test.rb +++ b/test/json/json_minefield_parser_test.rb @@ -22,12 +22,9 @@ def define_test(name, &block) i_string_overlong_sequence_2_bytes i_string_not_in_unicode_range i_string_lone_utf8_continuation_byte - i_string_lone_second_surrogate i_string_iso_latin_1 i_string_invalid_utf-8 - i_string_incomplete_surrogate_pair i_string_UTF-8_invalid_sequence - i_object_key_lone_2nd_surrogate ) COMMENT_TESTS = %w( @@ -56,6 +53,9 @@ def define_test(name, &block) i_string_utf16BE_no_BOM i_string_utf16LE_no_BOM i_structure_UTF-8_BOM_empty_object + i_string_lone_second_surrogate + i_string_incomplete_surrogate_pair + i_object_key_lone_2nd_surrogate ) if RUBY_ENGINE == 'jruby' diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 84839585f..3e8844299 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -403,6 +403,15 @@ def test_invalid_surogates assert_raise(JSON::ParserError) { parse('"\\uD800_________________"') } assert_raise(JSON::ParserError) { parse('"\\uD800\\u0041"') } assert_raise(JSON::ParserError) { parse('"\\uD800\\u004') } + # Lone trailing surrogate (issue #1069): parser previously returned an + # invalid-UTF-8 String instead of raising. Symmetric to the leading cases + # above. + assert_raise(JSON::ParserError) { parse('"\\uDC00"') } + assert_raise(JSON::ParserError) { parse('"\\uDC00_________________"') } + assert_raise(JSON::ParserError) { parse('"\\uDC00\\uD800"') } + # Valid pair still parses to the astral codepoint U+10000. + assert_predicate JSON.parse('"\\uD800\\uDC00"'), :valid_encoding? + assert_equal "\u{10000}", JSON.parse('"\\uD800\\uDC00"') end def test_parse_big_integers From 9427d36671bdeccfba05f51cc6191b02e1d7ff82 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 7 Sep 2026 07:54:05 +0100 Subject: [PATCH 444/506] Release 3.0.0 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ea9765ef4..19d7d51e9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-09-07 (3.0.0) + * Add `JSON::ParserError#json_path` to locate parse errors in the document as a JSONPath-style string (e.g. `$.foo[0].bar`). For duplicate key errors it points at the duplicated key itself. * Fix the parser to also reject lone trailing UTF-16 surrogates (`\uDCxx` with no leading partner), symmetric to the leading-surrogate case. The Java parser already rejected these; this closes the CRuby/JRuby parity gap. diff --git a/lib/json/version.rb b/lib/json/version.rb index 8e7e31755..eb53003aa 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '3.0.0.rc1' + VERSION = '3.0.0' end From 566bd7c108e9ec353201420d96085e092e12bc6b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 8 Sep 2026 07:47:06 +0100 Subject: [PATCH 445/506] Restore the `limit` positional argument of `JSON.dump` I really wanted to get rid of this because the argument parsing is awful and costly. But `rdoc` has been using this positional argument for decades, and while the fix has been merged, having to upgrade to the latest rdoc is a bit too disruptive. --- CHANGES.md | 2 ++ lib/json/common.rb | 28 ++++++++++++++++++---------- test/json/json_generator_test.rb | 6 ++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 19d7d51e9..2e3f7ec3f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Restore the `limit` positional argument of `JSON.dump`. + ### 2026-09-07 (3.0.0) * Add `JSON::ParserError#json_path` to locate parse errors in the document as a JSONPath-style string (e.g. `$.foo[0].bar`). For duplicate key errors it points at the duplicated key itself. diff --git a/lib/json/common.rb b/lib/json/common.rb index 34dc123b5..f8b68e5f9 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -726,16 +726,14 @@ def load(source, proc = nil, allow_blank: true, **options) end # :call-seq: - # JSON.dump(obj, io = nil, options = nil) + # JSON.dump(obj, io = nil, _deprecated_limit = nil, options = nil) # # Dumps +obj+ as a \JSON string, i.e. calls generate on the object and returns the result. # - # The default options can be changed via method JSON.dump_default_options. - # # - Argument +io+, if given, should respond to method +write+; # the \JSON \String is written to +io+, and +io+ is returned. # If +io+ is not given, the \JSON \String is returned. - # + # - Argument +_deprecated_limit+ is deprecated, pass the +:max_nesting+ option instead. # --- # # When argument +io+ is not given, returns the \JSON \String generated from +obj+: @@ -751,21 +749,31 @@ def load(source, proc = nil, allow_blank: true, **options) # puts File.read(path) # Output: # {"foo":[0,1],"bar":{"baz":2,"bat":3},"bam":"bad"} - def dump(obj, anIO = nil, kwargs = nil) + def dump(obj, anIO = nil, _deprecated_limit = nil, kwargs = nil) if kwargs.nil? - if anIO.is_a?(Hash) - kwargs = anIO - anIO = nil + if _deprecated_limit.nil? + if anIO.is_a?(Hash) + kwargs = anIO + anIO = nil + end + elsif _deprecated_limit.is_a?(Hash) + kwargs = _deprecated_limit + _deprecated_limit = nil end end - if anIO&.respond_to?(:to_io) - anIO = anIO.to_io + unless anIO.nil? + if anIO.respond_to?(:to_io) + anIO = anIO.to_io + elsif _deprecated_limit.nil? && !anIO.respond_to?(:write) + anIO, _deprecated_limit = nil, anIO + end end opts = { allow_nan: true, } + opts[:max_nesting] = _deprecated_limit if _deprecated_limit opts.merge!(kwargs) if kwargs State.generate(obj, opts, anIO) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 2a64a5e75..b0df70d5d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -86,6 +86,12 @@ def test_dump_strict assert_equal '{"hello":"world"}', dump({ hello: :world }, strict: true) end + def test_dump_deprecated_limit + io = StringIO.new + JSON.dump([1], io, 0) + assert_equal '[1]', io.string + end + def test_not_frozen [ [[], '[]'], From c557a62719948aa96a0b94746d24e3bb0adc3f02 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 8 Sep 2026 08:07:00 +0100 Subject: [PATCH 446/506] Release 3.0.1 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2e3f7ec3f..0d1a58331 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-09-08 (3.0.1) + * Restore the `limit` positional argument of `JSON.dump`. ### 2026-09-07 (3.0.0) diff --git a/lib/json/version.rb b/lib/json/version.rb index eb53003aa..1cf83a039 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '3.0.0' + VERSION = '3.0.1' end From 73edd5364486f142a44ac30a26ce5d21ae9c5f2a Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 8 Sep 2026 09:22:26 +0100 Subject: [PATCH 447/506] Centralize requires in the test suite --- test/json/json_coder_test.rb | 1 - test/json/json_common_interface_test.rb | 2 -- test/json/json_ext_parser_test.rb | 1 + test/json/json_fixtures_test.rb | 1 + test/json/json_float_parser_fallback_test.rb | 5 +---- test/json/json_generator_test.rb | 1 - test/json/json_minefield_parser_test.rb | 1 + test/json/json_parser_test.rb | 7 +------ test/json/resumable_parser_test.rb | 1 + test/json/test_helper.rb | 6 ++++++ 10 files changed, 12 insertions(+), 14 deletions(-) diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 7be1dfb3f..0e5e5e2f2 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -1,4 +1,3 @@ -#!/usr/bin/env ruby # frozen_string_literal: true require_relative 'test_helper' diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 2c4c9a213..e4bf958d4 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true require_relative 'test_helper' -require 'stringio' -require 'tempfile' class JSONCommonInterfaceTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index d585b8d0d..8b10146ad 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative 'test_helper' class JSONExtParserTest < Test::Unit::TestCase diff --git a/test/json/json_fixtures_test.rb b/test/json/json_fixtures_test.rb index c0d103793..a3f114c41 100644 --- a/test/json/json_fixtures_test.rb +++ b/test/json/json_fixtures_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative 'test_helper' class JSONFixturesTest < Test::Unit::TestCase diff --git a/test/json/json_float_parser_fallback_test.rb b/test/json/json_float_parser_fallback_test.rb index 39801d90d..97423837b 100644 --- a/test/json/json_float_parser_fallback_test.rb +++ b/test/json/json_float_parser_fallback_test.rb @@ -1,9 +1,6 @@ # frozen_string_literal: true + require_relative 'test_helper' -begin - require 'bigdecimal' -rescue LoadError -end class JSONFloatParserFallbackTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index b0df70d5d..fcfc968f2 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1,4 +1,3 @@ -#!/usr/bin/env ruby # frozen_string_literal: true require_relative 'test_helper' diff --git a/test/json/json_minefield_parser_test.rb b/test/json/json_minefield_parser_test.rb index 555801409..a447b8d31 100644 --- a/test/json/json_minefield_parser_test.rb +++ b/test/json/json_minefield_parser_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative 'test_helper' class JSONMinefieldParserTest < Test::Unit::TestCase diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3e8844299..fae2f4587 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1,11 +1,6 @@ # frozen_string_literal: true + require_relative 'test_helper' -require 'stringio' -require 'tempfile' -begin - require 'bigdecimal' -rescue LoadError -end class JSONParserTest < Test::Unit::TestCase include JSON diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 289da7547..2777d50c5 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require_relative 'test_helper' class JSONResumageParserTest < Test::Unit::TestCase diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index ddc75d3ac..a30885c40 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -29,6 +29,12 @@ require 'json' require 'test/unit' +require 'stringio' +require 'tempfile' +begin + require 'bigdecimal' +rescue LoadError +end if ENV["JSON_COMPACT"] if GC.respond_to?(:verify_compaction_references) From 29dad14674d75331457095fa14faf4f6b75aeb84 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Tue, 8 Sep 2026 17:25:32 +0900 Subject: [PATCH 448/506] Fix `JSON.load_file` and `JSON.load_file!` to load on Ruby 2.7.0 through 2.7.2 Since https://github.com/ruby/json/commit/ebf0f29, `load_file` and `load_file!` forward their arguments to `parse` and `parse!` with `...` after a leading parameter. Ruby 2.7 accepts `...` only on its own; a leading parameter before it was added in Ruby 3.0 and reached the 2.7 series in 2.7.3, so on Ruby 2.7.0 through 2.7.2 requiring json 3.0 raises a `SyntaxError` at `lib/json/common.rb`, while the gemspec still allows Ruby 2.7 as a whole. `parse` and `parse!` take keyword arguments only, so forwarding `**options` passes exactly what `...` passed, on every Ruby the gem supports. A `method_missing` helper in the parser tests used the same form; it takes an anonymous splat now, so the test suite itself loads on those versions. Checked with `ruby -c` on Ruby 2.7.0, 2.7.2, 2.7.3, and 3.0.7, and with the test suite on Ruby 2.7.2 and 3.0.7. --- CHANGES.md | 2 ++ lib/json/common.rb | 8 ++++---- test/json/json_parser_test.rb | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0d1a58331..30c51cdd3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix `JSON.load_file` and `JSON.load_file!` to load on Ruby 2.7.0 through 2.7.2, which cannot parse a leading parameter before `...`. + ### 2026-09-08 (3.0.1) * Restore the `limit` positional argument of `JSON.dump`. diff --git a/lib/json/common.rb b/lib/json/common.rb index f8b68e5f9..edcd93d4d 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -324,8 +324,8 @@ def parse!(source, **options) # parse(File.read(path), **) # # See method #parse. - def load_file(filespec, ...) - parse(File.read(filespec, encoding: Encoding::UTF_8), ...) + def load_file(filespec, **options) + parse(File.read(filespec, encoding: Encoding::UTF_8), **options) end # :call-seq: @@ -335,8 +335,8 @@ def load_file(filespec, ...) # JSON.parse!(File.read(path), **) # # See method #parse! - def load_file!(filespec, ...) - parse!(File.read(filespec, encoding: Encoding::UTF_8), ...) + def load_file!(filespec, **options) + parse!(File.read(filespec, encoding: Encoding::UTF_8), **options) end # :call-seq: diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index fae2f4587..bd167d9c8 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -771,7 +771,7 @@ def []=(k, v) @attrs[k.to_sym] = v end - def method_missing(name, ...) + def method_missing(name, *) @attrs.fetch(name) do super end From 8bdd5179df346a606c329dc16c119ff2ee20d1d6 Mon Sep 17 00:00:00 2001 From: Benjamin Quorning Date: Tue, 8 Sep 2026 17:15:06 +0200 Subject: [PATCH 449/506] Update documentation to match json 3.0 API json 3.0 changed `JSON.parse` to take keyword arguments instead of a positional opts Hash, so some of the documented usage would now raise `ArgumentError`. I have updated all such examples to use kwargs instead. Also align documented output with what is actually produced with the current parser/generator on Ruby 4.0, plus a number of other minor changes. --- lib/json.rb | 74 +++++++++++++++++++++++----------------------- lib/json/common.rb | 8 ++--- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/lib/json.rb b/lib/json.rb index 1e7573df9..f5f5ade89 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -44,13 +44,13 @@ # # You can parse a \String containing \JSON data using # either of two methods: -# - JSON.parse(source, opts) -# - JSON.parse!(source, opts) +# - JSON.parse(source, **opts) +# - JSON.parse!(source, **opts) # # where # - +source+ is a Ruby object. -# - +opts+ is a \Hash object containing options -# that control both input allowed and output formatting. +# - +opts+ are keyword arguments that control both input +# allowed and output formatting. # # The difference between the two methods # is that JSON.parse! omits some checks @@ -102,7 +102,7 @@ # ruby # => 1.0 # ruby.class # => Float # ruby = JSON.parse('2.0e2') -# ruby # => 200 +# ruby # => 200.0 # ruby.class # => Float # Boolean: # ruby = JSON.parse('true') @@ -131,10 +131,10 @@ # ruby # => [0, [1, [2, [3]]]] # Too deep: # # Raises JSON::NestingError (nesting of 2 is too deep): -# JSON.parse(source, {max_nesting: 1}) +# JSON.parse(source, max_nesting: 1) # Bad value: -# # Raises TypeError (wrong argument type Symbol (expected Fixnum)): -# JSON.parse(source, {max_nesting: :foo}) +# # Raises TypeError (no implicit conversion of Symbol into Integer): +# JSON.parse(source, max_nesting: :foo) # # --- # @@ -142,11 +142,11 @@ # should be ignored or cause an error to be raised: # # When set to +false+, the default: -# JSON.parse('{"a": 1, "a":2}') => duplicate key at line 1 column 1 (JSON::ParserError) +# JSON.parse('{"a": 1, "a": 2}') # duplicate key "a" at line 1 column 1 (JSON::ParserError) # # When set to +true+: # # The last value is used. -# JSON.parse('{"a": 1, "a":2}', allow_duplicate_key: true) => {"a" => 2} +# JSON.parse('{"a": 1, "a": 2}', allow_duplicate_key: true) # => {"a" => 2} # # --- # @@ -155,15 +155,15 @@ # defaults to +false+. # # With the default, +false+: -# # Raises JSON::ParserError (225: unexpected token at '[NaN]'): +# # Raises JSON::ParserError (unexpected token 'NaN]' at line 1 column 2): # JSON.parse('[NaN]') -# # Raises JSON::ParserError (232: unexpected token at '[Infinity]'): +# # Raises JSON::ParserError (unexpected token 'Infinity]' at line 1 column 2): # JSON.parse('[Infinity]') -# # Raises JSON::ParserError (248: unexpected token at '[-Infinity]'): +# # Raises JSON::ParserError (invalid number: '-Infinity]' at line 1 column 2): # JSON.parse('[-Infinity]') # Allow: # source = '[NaN, Infinity, -Infinity]' -# ruby = JSON.parse(source, {allow_nan: true}) +# ruby = JSON.parse(source, allow_nan: true) # ruby # => [NaN, Infinity, -Infinity] # # --- @@ -185,10 +185,10 @@ # defaults to +false+. # # When set to +false+, the default: -# JSON.parse('/* comment */ {"a": 1, "a":2}') # unexpected character: '/' at line 1 column 1 (JSON::ParserError) +# JSON.parse('/* comment */ {"a": 1, "a": 2}') # unexpected token '/*' at line 1 column 1 (JSON::ParserError) # # When set to +true+, comments are ignored: -# JSON.parse('/* comment */ {"a": 1, "a":2} // more comment') # => {"a" => 2} +# JSON.parse('/* comment */ {"a": 1} // more comment', allow_comments: true) # => {"a" => 1} # # --- # @@ -197,7 +197,7 @@ # defaults to +false+. # # With the default, +false+: -# JSON.parse(%{"Hello\nWorld"}) # invalid ASCII control character in string (JSON::ParserError) +# JSON.parse(%{"Hello\nWorld"}) # invalid ASCII control character in string: \nWorld" at line 2 column 0 (JSON::ParserError) # # When enabled: # JSON.parse(%{"Hello\nWorld"}, allow_control_characters: true) # => "Hello\nWorld" @@ -209,7 +209,7 @@ # defaults to +false+. # # With the default, +false+: -# JSON.parse('"Hell\o"') # invalid escape character in string (JSON::ParserError) +# JSON.parse('"Hell\o"') # invalid escape character in string: '\o"' at line 1 column 6 (JSON::ParserError) # # When enabled: # JSON.parse('"Hell\o"', allow_invalid_escape: true) # => "Hello" @@ -228,8 +228,8 @@ # ruby = JSON.parse(source) # ruby # => {"a"=>"foo", "b"=>1.0, "c"=>true, "d"=>false, "e"=>nil} # Use Symbols: -# ruby = JSON.parse(source, {symbolize_names: true}) -# ruby # => {:a=>"foo", :b=>1.0, :c=>true, :d=>false, :e=>nil} +# ruby = JSON.parse(source, symbolize_names: true) +# ruby # => {a: "foo", b: 1.0, c: true, d: false, e: nil} # # --- # @@ -242,7 +242,7 @@ # ruby = JSON.parse(source) # ruby.class # => Hash # Use class \OpenStruct: -# ruby = JSON.parse(source, {object_class: OpenStruct}) +# ruby = JSON.parse(source, object_class: OpenStruct) # ruby # => # # # --- @@ -256,8 +256,8 @@ # ruby = JSON.parse(source) # ruby.class # => Array # Use class \Set: -# ruby = JSON.parse(source, {array_class: Set}) -# ruby # => # +# ruby = JSON.parse(source, array_class: Set) +# ruby # => Set["foo", 1.0, true, false, nil] # # === Generating \JSON # @@ -319,22 +319,22 @@ # a \String containing a \JSON string representation of the source: # JSON.generate(:foo) # => '"foo"' # JSON.generate(Complex(0, 0)) # => '"0+0i"' -# JSON.generate(Dir.new('.')) # => '"#

"' +# JSON.generate(Dir.new('.')) # => '"#"' # # ==== Generating Options # # ====== Input Options # # Option +allow_nan+ (boolean) specifies whether -# +NaN+, +Infinity+, and -Infinity may be generated; +# +NaN+, +Infinity+, and +-Infinity+ may be generated; # defaults to +false+. # # With the default, +false+: -# # Raises JSON::GeneratorError (920: NaN not allowed in JSON): +# # Raises JSON::GeneratorError (NaN not allowed in JSON): # JSON.generate(JSON::NaN) -# # Raises JSON::GeneratorError (917: Infinity not allowed in JSON): +# # Raises JSON::GeneratorError (Infinity not allowed in JSON): # JSON.generate(JSON::Infinity) -# # Raises JSON::GeneratorError (917: -Infinity not allowed in JSON): +# # Raises JSON::GeneratorError (-Infinity not allowed in JSON): # JSON.generate(JSON::MinusInfinity) # # Allow: @@ -345,14 +345,14 @@ # # Option +allow_duplicate_key+ (boolean) specifies whether # hashes with duplicate keys should be allowed or produce an error. -# defaults to emit a deprecation warning. +# Defaults to +false+, which raises an error. # -# With the default, false: -# JSON.generate({ foo: 1, "foo" => 2 }) +# With the default, +false+: +# JSON.generate({foo: 1, "foo" => 2}) # # detected duplicate key "foo" in {foo: 1, "foo" => 2} (JSON::GeneratorError) # -# With true -# JSON.generate({ foo: 1, "foo" => 2 }, allow_duplicate_key: true) +# With +true+: +# JSON.generate({foo: 1, "foo" => 2}, allow_duplicate_key: true) # # => '{"foo":1,"foo":2}' # # --- @@ -365,13 +365,13 @@ # JSON.generate(obj) # => '[[[[[[0]]]]]]' # # Too deep: -# # Raises JSON::NestingError (nesting of 2 is too deep): +# # Raises JSON::NestingError (nesting of 2 is too deep. Did you try to serialize objects with circular references?): # JSON.generate(obj, max_nesting: 2) # # With +false+: # obj = [] # obj[0] = obj -# # Raises SystemStackError: stack level too deep +# # Raises SystemStackError (stack level too deep): # JSON.generate(obj, max_nesting: false) # # Setting +max_nesting+ to +false+ or a very large number can lead to a stack overflow @@ -410,7 +410,7 @@ # inserted before the colon in each \JSON object's pair; # defaults to the empty \String, ''. # - Option +sort_keys+ (boolean or \Proc) controls whether and how the keys of a -# hash are sorted when generating the output; defaults to false. +# hash are sorted when generating the output; defaults to +false+. # When +true+, keys are sorted lexicographically. When a \Proc, it receives # the entire \Hash and must return a \Hash with its pairs in the desired # order, allowing for arbitrary sort orders. @@ -439,7 +439,7 @@ # "foo" : [ # "bar", # "baz" -# ], +# ], # "bat" : { # "bam" : 0, # "bad" : 1 diff --git a/lib/json/common.rb b/lib/json/common.rb index edcd93d4d..ebec553a2 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -143,11 +143,11 @@ class JSONError < StandardError; end # This exception is raised if a parser error occurs. class ParserError < JSONError # Line number where the parser encountered an error. - # Is nil when raised by JSON::ResumableParser. + # Is +nil+ when raised by JSON::ResumableParser. attr_reader :line # Column number where the parser encountered an error. - # Is nil when raised by JSON::ResumableParser. + # Is +nil+ when raised by JSON::ResumableParser. attr_reader :column # Returns a best effort JSONPath string representing where in the document @@ -290,7 +290,7 @@ def to_json(state = nil, *) # --- # # Raises an exception if +source+ is not valid JSON: - # # Raises JSON::ParserError unexpected character: 'invalid' at line 1 column 1 : + # # Raises JSON::ParserError (unexpected character: 'invalid' at line 1 column 1): # JSON.parse('invalid') # def parse(source, on_load: nil, object_class: nil, array_class: nil, **options) @@ -372,7 +372,7 @@ def load_file!(filespec, **options) # # Raises an exception if +obj+ contains circular references: # a = []; b = []; a.push(b); b.push(a) - # # Raises JSON::NestingError (nesting of 100 is too deep): + # # Raises JSON::NestingError (nesting of 100 is too deep. Did you try to serialize objects with circular references?): # JSON.generate(a) # def generate(obj, opts = nil) From effca2796425f7a94d48497edb8264f44247ca71 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 8 Sep 2026 22:15:04 -0700 Subject: [PATCH 450/506] Fix the JSON::Coder.new call-seq `JSON` has no `.new`, and options have been keyword arguments since 3.0, so passing a positional Hash raises ArgumentError. --- lib/json/common.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index ebec553a2..82d334061 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -812,9 +812,9 @@ class Coder private_constant :EXCLUDED_GENERATOR_OPTIONS # :call-seq: - # JSON.new(options = nil, &block) + # JSON::Coder.new(**options, &block) # - # Argument +options+, if given, contains a \Hash of options for both parsing and generating. + # Keyword arguments +options+, if given, are options for both parsing and generating. # See {Parsing Options}[rdoc-ref:JSON@Parsing+Options], # and {Generating Options}[rdoc-ref:JSON@Generating+Options]. # From 7b2a23deea65d82a4afe6caa20f2ac89c94f7e41 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 9 Sep 2026 07:22:26 +0100 Subject: [PATCH 451/506] Release 3.0.2 --- CHANGES.md | 2 ++ lib/json/version.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 30c51cdd3..8deebca79 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +### 2026-09-09 (3.0.2) + * Fix `JSON.load_file` and `JSON.load_file!` to load on Ruby 2.7.0 through 2.7.2, which cannot parse a leading parameter before `...`. ### 2026-09-08 (3.0.1) diff --git a/lib/json/version.rb b/lib/json/version.rb index 1cf83a039..318272def 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '3.0.1' + VERSION = '3.0.2' end From 46fbe24b0488aac9870220455f6466801e32a5ec Mon Sep 17 00:00:00 2001 From: youdie006 Date: Thu, 10 Sep 2026 04:01:05 +0900 Subject: [PATCH 452/506] Make State#configure and #merge only write the options they are given The pure-Ruby generator's private _configure gives every keyword a literal default, so a call that passes one option silently resets the other fifteen. The method is also aliased as merge, so a state built with indent/object_nl starts emitting compact JSON after any later configure call. The C extension does not behave this way. configure_state_i walks only the keys actually present in the hash, so State#merge really merges there. Default the keywords to the current ivars so the pure generator agrees. initialize already assigns every ivar to its literal default before calling _configure(**opts), so construction is unchanged. --- lib/json/truffle_ruby/generator.rb | 9 ++++--- test/json/json_generator_test.rb | 41 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 6dad5e7f4..fe5daafff 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -302,10 +302,13 @@ def configure(options) end alias merge configure + # Defaults are the current values so that #configure only writes what it was given. private def _configure( - indent: '', space: '', space_before: '', object_nl: '', array_nl: '', allow_nan: false, - as_json: false, ascii_only: false, sort_keys: false, depth: 0, buffer_initial_length: 1024, - allow_duplicate_key: false, script_safe: false, strict: false, max_nesting: 100 + indent: @indent, space: @space, space_before: @space_before, object_nl: @object_nl, + array_nl: @array_nl, allow_nan: @allow_nan, as_json: @as_json, ascii_only: @ascii_only, + sort_keys: @sort_keys, depth: @depth, buffer_initial_length: @buffer_initial_length, + allow_duplicate_key: @allow_duplicate_key, script_safe: @script_safe, strict: @strict, + max_nesting: @max_nesting ) if depth.negative? raise ArgumentError, "depth must be >= 0 (got #{depth})" diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index fcfc968f2..1f4b5d949 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -516,6 +516,47 @@ def test_configure_using_configure_and_merge assert_equal '5', state2.array_nl end + def test_configure_only_writes_the_string_options_it_is_given + state = JSON.state.new(indent: '1', space: '2', space_before: '3', object_nl: '4', array_nl: '5') + state.configure(space: '9') + assert_equal '1', state.indent + assert_equal '9', state.space + assert_equal '3', state.space_before + assert_equal '4', state.object_nl + assert_equal '5', state.array_nl + state.merge(array_nl: '8') + assert_equal '1', state.indent + assert_equal '9', state.space + assert_equal '3', state.space_before + assert_equal '4', state.object_nl + assert_equal '8', state.array_nl + end + + def test_configure_keeps_the_layout_of_a_pretty_state + state = JSON.state.new(indent: ' ', object_nl: "\n", array_nl: "\n") + state.configure(depth: 0) + assert_equal %({\n "foo":[\n 1\n ]\n}), state.generate({ 'foo' => [1] }) + end + + def test_configure_only_writes_the_other_options_it_is_given + omit 'JRuby resets the non-string options' if RUBY_ENGINE == 'jruby' + state = JSON.state.new(max_nesting: 3, allow_nan: true, ascii_only: true, script_safe: true) + state.configure(indent: '1') + assert_equal '1', state.indent + assert_equal 3, state.max_nesting + assert_equal true, state.allow_nan? + assert_equal true, state.ascii_only? + assert_equal true, state.script_safe? + end + + def test_configure_writes_a_string_option_given_as_nil + omit 'JRuby keeps the previous value for an explicit nil' if RUBY_ENGINE == 'jruby' + state = JSON.state.new(indent: '1', space: '2') + state.configure(indent: nil) + assert_equal '', state.indent + assert_equal '2', state.space + end + def test_configure_hash_conversion state = JSON.state.new state.configure(indent: '1') From bbc73fb69df1670065e6a66e219c3af0a6752ba4 Mon Sep 17 00:00:00 2001 From: youdie006 Date: Fri, 11 Sep 2026 15:54:50 +0900 Subject: [PATCH 453/506] Make GeneratorState#_configure only write the options it is given The C extension walks the given hash and writes only the keys present (generator.c:1860), and since #1076 the pure generator does the same. The Java one re-wrote every option with its class default, so any configure or merge silently reset max_nesting, allow_nan, ascii_only, script_safe, strict, buffer_initial_length, allow_duplicate_key, as_json and sort_keys. Re-enables the two tests #1076 marked pending on JRuby. --- java/src/json/ext/GeneratorState.java | 52 +++++++++++++-------------- test/json/json_generator_test.rb | 20 +++++++++-- 2 files changed, 42 insertions(+), 30 deletions(-) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index dabc9e255..3fdf4f276 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -545,43 +545,41 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { checkFrozen(); OptionsReader opts = new OptionsReader(context, vOpts); - ByteList indent = opts.getString("indent"); - if (indent != null) this.indent = indent; - - ByteList space = opts.getString("space"); - if (space != null) this.space = space; - - ByteList spaceBefore = opts.getString("space_before"); - if (spaceBefore != null) this.spaceBefore = spaceBefore; - - ByteList arrayNl = opts.getString("array_nl"); - if (arrayNl != null) this.arrayNl = arrayNl; - - this.asJSON = opts.getProc("as_json"); - - ByteList objectNl = opts.getString("object_nl"); - if (objectNl != null) this.objectNl = objectNl; - - maxNesting = opts.getInt("max_nesting", DEFAULT_MAX_NESTING); - allowNaN = opts.getBool("allow_nan", DEFAULT_ALLOW_NAN); - asciiOnly = opts.getBool("ascii_only", DEFAULT_ASCII_ONLY); - scriptSafe = opts.getBool("script_safe", DEFAULT_SCRIPT_SAFE); - strict = opts.getBool("strict", DEFAULT_STRICT); - bufferInitialLength = opts.getInt("buffer_initial_length", DEFAULT_BUFFER_INITIAL_LENGTH); - - depth = opts.getInt("depth", 0); + this.indent = stringConfig(opts, "indent", this.indent); + this.space = stringConfig(opts, "space", this.space); + this.spaceBefore = stringConfig(opts, "space_before", this.spaceBefore); + this.arrayNl = stringConfig(opts, "array_nl", this.arrayNl); + this.objectNl = stringConfig(opts, "object_nl", this.objectNl); + + if (opts.hasKey("as_json")) this.asJSON = opts.getProc("as_json"); + + maxNesting = opts.getInt("max_nesting", maxNesting); + allowNaN = opts.getBool("allow_nan", allowNaN); + asciiOnly = opts.getBool("ascii_only", asciiOnly); + scriptSafe = opts.getBool("script_safe", scriptSafe); + strict = opts.getBool("strict", strict); + bufferInitialLength = opts.getInt("buffer_initial_length", bufferInitialLength); + + depth = opts.getInt("depth", depth); if (depth < 0) { throw context.runtime.newArgumentError("depth must be >= 0 (got: " + depth + ")"); } - this.allowDuplicateKey = opts.getBool("allow_duplicate_key", false); + this.allowDuplicateKey = opts.getBool("allow_duplicate_key", allowDuplicateKey); - sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); + if (opts.hasKey("sort_keys")) sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); opts.ensureEmpty(); return this; } + // A falsy value writes the empty string, as string_config() does in the C extension. + private static ByteList stringConfig(OptionsReader opts, String key, ByteList current) { + if (!opts.hasKey(key)) return current; + ByteList value = opts.getString(key); + return value == null ? ByteList.EMPTY_BYTELIST : value; + } + /** * State#to_h() * diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1f4b5d949..1b2197735 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -539,18 +539,32 @@ def test_configure_keeps_the_layout_of_a_pretty_state end def test_configure_only_writes_the_other_options_it_is_given - omit 'JRuby resets the non-string options' if RUBY_ENGINE == 'jruby' - state = JSON.state.new(max_nesting: 3, allow_nan: true, ascii_only: true, script_safe: true) + state = JSON.state.new(max_nesting: 3, allow_nan: true, ascii_only: true, script_safe: true, + strict: true, buffer_initial_length: 32) state.configure(indent: '1') assert_equal '1', state.indent assert_equal 3, state.max_nesting assert_equal true, state.allow_nan? assert_equal true, state.ascii_only? assert_equal true, state.script_safe? + assert_equal true, state.strict? + assert_equal 32, state.buffer_initial_length + end + + def test_configure_keeps_sort_keys + state = JSON.state.new(sort_keys: true) + state.configure(depth: 0) + assert_equal '{"a":2,"b":1}', state.generate({ 'b' => 1, 'a' => 2 }) + end + + def test_configure_keeps_as_json + as_json = ->(object, _is_key) { object.to_s } + state = JSON.state.new(strict: true, as_json: as_json) + state.configure(depth: 0) + assert_equal as_json, state.as_json end def test_configure_writes_a_string_option_given_as_nil - omit 'JRuby keeps the previous value for an explicit nil' if RUBY_ENGINE == 'jruby' state = JSON.state.new(indent: '1', space: '2') state.configure(indent: nil) assert_equal '', state.indent From 522b01506352873692860b3314f8272012876059 Mon Sep 17 00:00:00 2001 From: Ojus Chugh Date: Mon, 31 Aug 2026 21:34:51 +0530 Subject: [PATCH 454/506] Add error position and json_path to the JRuby extension --- CHANGES.md | 2 + java/src/json/ext/Parser.java | 83 ++++++++++++++++++++++++++++++----- test/json/json_parser_test.rb | 18 ++++---- 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8deebca79..9ef0ce94a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* JRuby: parser errors now include the position (`line`, `column` and the message suffix) as well as `json_path`, matching the C extension. + ### 2026-09-09 (3.0.2) * Fix `JSON.load_file` and `JSON.load_file!` to load on Ruby 2.7.0 through 2.7.2, which cannot parse a leading parameter before `...`. diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 59ebda179..92712ba27 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -6,6 +6,7 @@ import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyClass; +import org.jruby.RubyException; import org.jruby.RubyFloat; import org.jruby.RubyHash; import org.jruby.RubyObject; @@ -376,7 +377,7 @@ private IRubyObject run() { continue; } case 0: - throw newException(Utils.M_PARSER_ERROR, "unexpected end of input"); + throw parseError("unexpected end of input"); default: throw unexpectedToken(cursor, end); } @@ -572,7 +573,7 @@ private void onDuplicateKey(IRubyObject key) { // match the C parser's message. String keyInspect = key.callMethod(context, "to_s") .callMethod(context, "inspect").asJavaString(); - throw newException(Utils.M_PARSER_ERROR, "duplicate key " + keyInspect); + throw parseError(context.runtime.newString("duplicate key " + keyInspect), key); } private int parseDigits(long value) { @@ -667,8 +668,7 @@ private IRubyObject parseString(boolean isName) { long scanned = scanner.scan(data, chunks, contentStart, end); final int q = (int) scanned; if (q < 0) { - throw newException(Utils.M_PARSER_ERROR, - "unexpected end of input, expected closing \""); + throw parseError("unexpected end of input, expected closing \""); } boolean plain = (scanned & StringScanner.PLAIN_BIT) != 0; @@ -853,8 +853,7 @@ private void eatComments() { while (true) { while (cursor < end && data[cursor] != '*') cursor++; if (cursor >= end) { - throw newException(Utils.M_PARSER_ERROR, - "unterminated comment, expected closing '*/'"); + throw parseError("unterminated comment, expected closing '*/'"); } cursor++; // past '*' if (peek() == '/') { @@ -883,11 +882,77 @@ private IRubyObject getConstant(String name) { return info.jsonModule.get().getConstant(name); } + private long cursorPosition(int position) { + long column = 0; + int i = position; + if (i >= end) { + column = i - end + 1; + i = end - 1; + } + int line = 1; + while (i >= begin) { + if (data[i--] == '\n') { + line++; + break; + } + column++; + } + while (i >= begin) { + if (data[i--] == '\n') { + line++; + } + } + return ((long) line << 32) | column; + } + + private RubyArray jsonPathSegments(IRubyObject duplicateKey) { + Ruby runtime = context.runtime; + RubyArray segments = RubyArray.newArray(runtime); + for (int depth = 1; depth < frameDepth; depth++) { + Frame frame = frameStack[depth]; + boolean innermost = depth == frameDepth - 1; + int childHead = innermost ? valueTop : frameStack[depth + 1].valueStackHead; + int count = childHead - frame.valueStackHead; + + if (frame.type == FrameType.ARRAY) { + segments.append(runtime.newFixnum(frame.phase == FramePhase.ARRAY_COMMA ? count - 1 : count)); + } else if (innermost && duplicateKey != null) { + segments.append(duplicateKey); + } else if ((count & 1) == 1) { + segments.append(valueStack[childHead - 1]); + } else if (frame.phase == FramePhase.OBJECT_COMMA && count >= 2) { + segments.append(valueStack[childHead - 2]); + } else { + break; + } + } + return segments; + } + + private RaiseException parseError(String message) { + return parseError(context.runtime.newString(message), null); + } + + private RaiseException parseError(RubyString message, IRubyObject duplicateKey) { + Ruby runtime = context.runtime; + long position = cursorPosition(cursor); + int line = (int) (position >>> 32); + int column = (int) position; + message.catString(" at line " + line + " column " + column); + RaiseException error = Utils.newException(context, Utils.M_PARSER_ERROR, message); + RubyException exception = error.getException(); + exception.setInstanceVariable("@line", runtime.newFixnum(line)); + exception.setInstanceVariable("@column", runtime.newFixnum(column)); + exception.setInstanceVariable("@json_path", jsonPathSegments(duplicateKey)); + return error; + } + private RaiseException parsingError(int absStart, int absEnd) { + cursor = absStart; RubyString msg = context.runtime.newString("unexpected token at '") .cat(data, absStart, Math.min(absEnd - absStart, 32)) .cat((byte)'\''); - return newException(Utils.M_PARSER_ERROR, msg); + return parseError(msg, null); } private RaiseException unexpectedToken(int absStart, int absEnd) { @@ -897,9 +962,5 @@ private RaiseException unexpectedToken(int absStart, int absEnd) { private RaiseException newException(String className, String message) { return Utils.newException(context, className, message); } - - private RaiseException newException(String className, RubyString message) { - return Utils.newException(context, className, message); - } } } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index bd167d9c8..2fd7e2745 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -852,8 +852,6 @@ def test_parse_error_snippet end def test_parse_error_json_path - omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" - assert_parse_error_at "$", "xyz" assert_parse_error_at "$.a", '{"a": xyz}' assert_parse_error_at "$[3]", '[1, 2, "hi", xyz]' @@ -870,9 +868,17 @@ def test_parse_error_json_path assert_parse_error_at "$[5]", '[1,2,3,4,5,]' end - def test_parse_error_json_path_on_load - omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + def test_parse_error_position + error = assert_raise(JSON::ParserError) { JSON.parse("[1,\n@") } + assert_equal 2, error.line + assert_equal 1, error.column + + error = assert_raise(JSON::ParserError) { JSON.parse('{"a": {"b": [1, {"c": 1, "c": 2}]}}') } + assert_equal 1, error.line + assert_equal 17, error.column + end + def test_parse_error_json_path_on_load assert_parse_error_at "$" do JSON.load('{"a": {"b": {"c":', -> (obj) { if String === obj @@ -895,8 +901,6 @@ def test_parse_error_json_path_on_load end def test_parse_error_json_path_key_escaping - omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" - assert_parse_error_at '$["hello world"]', '{"hello world": xyz}' assert_parse_error_at '$["a\"b"]', '{"a\"b": xyz}' assert_parse_error_at '$[""]', '{"": xyz}' @@ -905,8 +909,6 @@ def test_parse_error_json_path_key_escaping end def test_parse_error_json_path_duplicate_key - omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" - assert_parse_error_at "$.a", '{"a": 1, "a": 2}' assert_parse_error_at "$.x.a", '{"x": {"a": 1, "b": 2, "a": 3}}' assert_parse_error_at "$.arr[0].a", '{"arr": [{"a": 1, "a": 2}]}' From 729af7a243311c2291da865c1390732f2c1fe377 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Wed, 16 Sep 2026 14:49:10 +0200 Subject: [PATCH 455/506] Disable coverage on jruby --- Rakefile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Rakefile b/Rakefile index 0416411a6..db8ea3ac4 100644 --- a/Rakefile +++ b/Rakefile @@ -7,12 +7,14 @@ require 'rake/testtask' class UndocumentedTestTask < Rake::TestTask def desc(*) end - def ruby(...) - env_was = ENV["JSON_COVERAGE"] - ENV["JSON_COVERAGE"] = "1" - ret = super - ENV["JSON_COVERAGE"] = env_was - ret + if RUBY_ENGINE != "jruby" + def ruby(...) + env_was = ENV["JSON_COVERAGE"] + ENV["JSON_COVERAGE"] = "1" + ret = super + ENV["JSON_COVERAGE"] = env_was + ret + end end end From f9698cbb1d1397d25cdead04dca6837573ac35c5 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 19 Jun 2026 14:19:48 -0500 Subject: [PATCH 456/506] Add a non-validating StringEncoder#decodeNoValidate fastpath used when decoding ASCII-only strings with escape or control characters. --- java/src/json/ext/Parser.java | 32 +++-- java/src/json/ext/StringDecoder.java | 46 ++++++++ java/src/json/ext/StringScanner.java | 111 +++++++++++++++--- .../src/json/ext/VectorizedStringScanner.java | 46 +++++++- 4 files changed, 202 insertions(+), 33 deletions(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 92712ba27..fbe243632 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -67,6 +67,15 @@ public class Parser extends RubyObject { private static final String CONST_INFINITY = "Infinity"; private static final String CONST_MINUS_INFINITY = "MinusInfinity"; + private static final String VALIDATE_UTF8_STRINGS = "jruby.json.validateUTF8Strings"; + private static final String VALIDATE_UTF8_STRINGS_DEFAULT = "true"; + private static final boolean validateUTF8Strings; + + static { + String enable = System.getProperty(VALIDATE_UTF8_STRINGS, VALIDATE_UTF8_STRINGS_DEFAULT); + validateUTF8Strings = "true".equalsIgnoreCase(enable) || "1".equals(enable); + } + static final ObjectAllocator ALLOCATOR = Parser::new; public Parser(Ruby runtime, RubyClass metaClass) { @@ -660,11 +669,6 @@ private IRubyObject parseString(boolean isName) { final byte[] data = this.data; final int contentStart = cursor + 1; // skip opening quote - // The scanner finds the closing quote and reports whether the body - // is plain printable ASCII (no escape, no ASCII control character, - // no non-ASCII byte). Anything non-plain is handed to StringDecoder, - // which performs the UTF-8/control validation, escape expansion, and - // error reporting. long scanned = scanner.scan(data, chunks, contentStart, end); final int q = (int) scanned; if (q < 0) { @@ -672,6 +676,8 @@ private IRubyObject parseString(boolean isName) { } boolean plain = (scanned & StringScanner.PLAIN_BIT) != 0; + boolean isAscii = (scanned & StringScanner.ASCII_BIT) != 0; + cursor = q + 1; // past closing quote // Note: When running multiple read-world benchmarks in the same JVM, @@ -691,8 +697,7 @@ private IRubyObject parseString(boolean isName) { off = contentStart; len = q - contentStart; } else { - ByteList decoded = decoder.decode(context, byteList, - contentStart - begin, q - begin); + ByteList decoded = decodeString(context, byteList, contentStart - begin, q - begin, isAscii); buf = decoded.getUnsafeBytes(); off = decoded.begin(); len = decoded.realSize(); @@ -709,8 +714,7 @@ private IRubyObject parseString(boolean isName) { string = RubyString.newString(context.runtime, data, contentStart, q - contentStart, UTF8Encoding.INSTANCE); } else { - ByteList content = decoder.decode(context, byteList, - contentStart - begin, q - begin); + ByteList content = decodeString(context, byteList, contentStart - begin, q - begin, isAscii); string = context.runtime.newString(content); string.setEncoding(UTF8Encoding.INSTANCE); string.clearCodeRange(); @@ -723,6 +727,16 @@ private IRubyObject parseString(boolean isName) { return string; } + private ByteList decodeString(ThreadContext context, ByteList byteList, int start, int end, boolean isAscii) { + if (isAscii) { + return decoder.decodeNoValidate(context, byteList, start, end, chunks); + } + if (!validateUTF8Strings) { + return decoder.decodeNoValidate(context, byteList, start, end, chunks); + } + return decoder.decode(context, byteList, start, end); + } + private static boolean isLetter(byte b) { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'); } diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index 28874acdc..7ae96c5de 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -10,6 +10,7 @@ import org.jruby.util.ByteList; import java.io.IOException; +import java.nio.ByteBuffer; /** * A decoder that reads a JSON-encoded string from the given sources and @@ -25,6 +26,8 @@ final class StringDecoder extends ByteListTranscoder { private boolean allowControlCharacters = false; private boolean allowInvalidEscape = false; + private final StringScanner scanner = StringScanner.getInstance(); + private ByteList out; // Array used for writing multibyte characters into the buffer at once @@ -49,6 +52,49 @@ ByteList decode(ThreadContext context, ByteList src, int start, int end) { } } + // Decodes ASCII-only strings with no UTF-8 validation. It is assumed this is only called when + // the StringScanner#scan returned a result with the ASCII_BIT bit set. + ByteList decodeNoValidate(ThreadContext context, ByteList src, int start, int end, ByteBuffer chunks) { + try { + init(src, start, end); + this.out = new ByteList(end - start); + final byte[] data = src.unsafeBytes(); + final int base = src.begin(); + final int absEnd = base + srcEnd; + int cursor = base + pos; + int runStart = cursor; + while (cursor < absEnd) { + cursor = scanner.scanEscape(data, chunks, cursor, absEnd); + if (cursor >= absEnd) { + break; + } + int b = Byte.toUnsignedInt(data[cursor]); + if (b == '\\') { + if (cursor > runStart) { + append(data, runStart, cursor - runStart); + } + charStart = cursor - base; + pos = charStart + 1; + handleEscapeSequence(context); + cursor = base + pos; + runStart = cursor; + } else { + if (!allowControlCharacters) { + charStart = cursor - base; + throw invalidControlChar(context); + } + cursor++; + } + } + if (absEnd > runStart) { + append(data, runStart, absEnd - runStart); + } + return out; + } catch (IOException e) { + throw context.runtime.newIOErrorFromException(e); + } + } + @Override protected int readUtf8Char(ThreadContext context) { int c = super.readUtf8Char(context); diff --git a/java/src/json/ext/StringScanner.java b/java/src/json/ext/StringScanner.java index 5ca59d177..729c81e05 100644 --- a/java/src/json/ext/StringScanner.java +++ b/java/src/json/ext/StringScanner.java @@ -16,6 +16,8 @@ class StringScanner { /** Set in the returned bits when the whole body is plain printable ASCII. */ static final long PLAIN_BIT = 1L << 32; + static final long ASCII_BIT = 1L << 33; + /** Returned when no closing quote is found before {@code end}. */ static final long NOT_FOUND = -1L; @@ -41,8 +43,6 @@ class StringScanner { .loadClass(VECTORIZED_SCANNER_CLASS); scanner = (StringScanner) vectorized.getDeclaredConstructor().newInstance(); } catch (Throwable t) { - // jdk.incubator.vector unavailable (or any load failure): - // keep the SWAR implementation. scanner = new StringScanner(); } } @@ -54,31 +54,41 @@ static StringScanner getInstance() { } /** - * Scans {@code data[start..end)} for the closing quote, honouring backslash - * escapes. + * Scans {@code data[start..end)} for a closing quote. This method will + * also report if there are any interesting bytes within the range + * {@link data[start..end)}. An interestin byte is defined as control + * characters, backslashes or bytes with the high bit set. + * + * @param chunks a little-endian {@link ByteBuffer} over {@code data} * - * @param chunks a little-endian {@link ByteBuffer} over {@code data}, used - * for the 8-byte SWAR reads (the vectorized subclass reads - * {@code data} directly and ignores it). * @return packed result: the low 32 bits hold the index of the closing * quote, or {@code -1} ({@link #NOT_FOUND}) when none is found * before {@code end}; {@link #PLAIN_BIT} is set when the entire body * is plain printable ASCII (no escape, no ASCII control character, - * and no non-ASCII byte) and can be copied verbatim. + * and no non-ASCII byte) and can be copied verbatim; + * {@link #ASCII_BIT} is set when the body contains no non-ASCII byte + * (it may still hold escapes or control characters) */ long scan(byte[] data, ByteBuffer chunks, int start, int end) { int p = start; boolean plain = true; + boolean ascii = true; outer: while (true) { - // SWAR: skip 8-byte chunks that contain nothing interesting. while (p + 8 <= end) { long x = chunks.getLong(p); - // Due to the byte-by-byte handling if we match an interesting byte, - // if we already know this is a non-ASCII-only string, we simply - // look for quotes and backslashes. - long m = plain ? stringScanMask(x) : quoteBackslashMask(x); + // Pick the cheapest mask that still observes every transition we + // still care about: + // plain -> control, quote, backslash and high bytes + // non-plain, ASCII -> quote, backslash and high bytes, so the + // first non-ASCII byte still stops us and + // clears ASCII_BIT + // non-plain, !ASCII -> quote and backslash only; multi-byte + // UTF-8 is skipped eight bytes at a time + long m = plain ? stringScanMask(x) + : ascii ? quoteBackslashHighMask(x) + : quoteBackslashMask(x); if (m == 0) { p += 8; } else { @@ -90,24 +100,61 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { while (p < end) { int b = data[p] & 0xFF; if (b == '"') { - return ((long) p) | (plain ? PLAIN_BIT : 0L); + return (((long) p) | (plain ? PLAIN_BIT : 0L) | (ascii ? ASCII_BIT : 0L)); } if (b == '\\') { plain = false; p += 2; // skip the backslash and the escaped byte continue outer; } - if (b < 0x20 || b >= 0x80) { + if (b < 0x20) { plain = false; p++; continue outer; } + if (b >= 0x80) { + plain = false; + ascii = false; + p++; + continue outer; + } p++; } return NOT_FOUND; } } + + /** + * Scans {@code data[start..end)} for the next backslash or control character. + * + *

The caller must guarantee the {@link data[start..end]} contains + * no non-ASCII bytes that need to be decoded.

+ * + * @return the index of the first backslash or control byte, or {@code end} + * when none is found. + */ + int scanEscape(byte[] data, ByteBuffer chunks, int start, int end) { + int p = start; + while (p + 8 <= end) { + long x = chunks.getLong(p); + long m = backslashControlMask(x); + if (m == 0) { + p += 8; + } else { + return p + (Long.numberOfTrailingZeros(m) >>> 3); + } + } + while (p < end) { + int b = data[p] & 0xFF; + if (b == '\\' || b < 0x20) { + return p; + } + p++; + } + return end; + } + /** * Returns a mask whose high bit (0x80) is set in every lane of {@code x} * that needs scalar attention: an ASCII control character (< 0x20), a @@ -124,11 +171,24 @@ private static long stringScanMask(long x) { return (control | high | quote | bslash) & HIGH_BITS; } + /** + * Like {@link #quoteBackslashMask} but for the ASCII-only decode fast path: + * flags backslashes and ASCII control characters (< 0x20). The control + * test relies on every byte being ASCII (< 0x80), which the caller + * guarantees, so no high-bit cleanup is needed. + */ + private static long backslashControlMask(long x) { + long control = (x - SPACES) & ~x; // bytes < 0x20 (ASCII) + long s = x ^ BACKSLASHES; + long bslash = (s - ONES) & ~s; + return (control | bslash) & HIGH_BITS; + } + /** * Like {@link #stringScanMask} but only flags double quotes and backslashes. - * Used once a string is known to require the decoder, so the remaining scan - * for the closing quote still skips clean chunks (including multi-byte - * UTF-8) eight bytes at a time. + * Used once a string is known to require the decoder and to already + * contain non-ASCII bytes, so the remaining scan for the closing quote skips + * clean chunks (including multi-byte UTF-8) eight bytes at a time. */ private static long quoteBackslashMask(long x) { long q = x ^ DOUBLE_QUOTES; @@ -137,4 +197,19 @@ private static long quoteBackslashMask(long x) { long bslash = (s - ONES) & ~s; return (quote | bslash) & HIGH_BITS; } + + /** + * Like {@link #quoteBackslashMask} but also flags non-ASCII bytes. Used once + * a string is known to require the decoder but is still ASCII-only, so the + * scan keeps skipping printable runs eight bytes at a time yet still stops on + * the first high byte and clears {@link #ASCII_BIT}. + */ + private static long quoteBackslashHighMask(long x) { + long high = x; // bit 0x80 set iff non-ASCII + long q = x ^ DOUBLE_QUOTES; + long quote = (q - ONES) & ~q; + long s = x ^ BACKSLASHES; + long bslash = (s - ONES) & ~s; + return (high | quote | bslash) & HIGH_BITS; + } } diff --git a/java/src/json/ext/VectorizedStringScanner.java b/java/src/json/ext/VectorizedStringScanner.java index 8a9c1e3e7..d7c4a8694 100644 --- a/java/src/json/ext/VectorizedStringScanner.java +++ b/java/src/json/ext/VectorizedStringScanner.java @@ -12,6 +12,7 @@ final class VectorizedStringScanner extends StringScanner { private static final ByteVector ZERO = ByteVector.zero(SP); private static final ByteVector TWO = ByteVector.broadcast(SP, 2); private static final ByteVector THIRTY_THREE = ByteVector.broadcast(SP, 33); + private static final ByteVector SPACE = ByteVector.broadcast(SP, 0x20); private static final ByteVector BACKSLASH = ByteVector.broadcast(SP, '\\'); private static final ByteVector DQUOTE = ByteVector.broadcast(SP, '"'); @@ -20,15 +21,18 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { final int width = SP.length(); int p = start; boolean plain = true; + boolean ascii = true; - // The same structure as the StringEncoder. The logic is + // The same structure as the StringEncoder. The logic is // duplicated for maximum inlining. outer: while (true) { while (p + width <= end) { ByteVector chunk = ByteVector.fromArray(SP, data, p); VectorMask interesting = - plain ? interestingLanes(chunk) : quoteOrBackslashLanes(chunk); + plain ? interestingLanes(chunk) + : ascii ? quoteBackslashOrHighLanes(chunk) + : quoteOrBackslashLanes(chunk); if (interesting.anyTrue()) { p += interesting.firstTrue(); break; @@ -38,14 +42,20 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { while (p < end) { int b = data[p] & 0xFF; if (b == '"') { - return ((long) p) | (plain ? PLAIN_BIT : 0L); + return ((long) p) | (plain ? PLAIN_BIT : 0L) | (ascii ? ASCII_BIT : 0L); } if (b == '\\') { plain = false; p += 2; // skip the backslash and the escaped byte continue outer; } - if (b < 0x20 || b >= 0x80) { + if (b >= 0x80) { + plain = false; + ascii = false; + p++; + continue outer; + } + if (b < 0x20) { plain = false; p++; continue outer; @@ -56,7 +66,28 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { } } - // Lanes that are control characters, double quotes, backslashes or non-ASCII. + @Override + int scanEscape(byte[] data, ByteBuffer chunks, int start, int end) { + final int width = SP.length(); + if (start + width > end) { + return super.scanEscape(data, chunks, start, end); + } + int p = start; + while (p + width <= end) { + ByteVector chunk = ByteVector.fromArray(SP, data, p); + VectorMask interesting = escapeOrControlLanes(chunk); + if (interesting.anyTrue()) { + return p + interesting.firstTrue(); + } + p += width; + } + return super.scanEscape(data, chunks, p, end); + } + + private static VectorMask escapeOrControlLanes(ByteVector chunk) { + return chunk.lt(SPACE).or(chunk.eq(BACKSLASH)); + } + private static VectorMask interestingLanes(ByteVector chunk) { VectorMask negative = chunk.lt(ZERO); VectorMask lowOrQuote = chunk.lanewise(VectorOperators.XOR, TWO) @@ -65,8 +96,11 @@ private static VectorMask interestingLanes(ByteVector chunk) { return lowOrQuote.or(chunk.eq(BACKSLASH)).or(negative); } - // Lanes that are a double quote or a backslash (non-plain phase). private static VectorMask quoteOrBackslashLanes(ByteVector chunk) { return chunk.eq(DQUOTE).or(chunk.eq(BACKSLASH)); } + + private static VectorMask quoteBackslashOrHighLanes(ByteVector chunk) { + return chunk.eq(DQUOTE).or(chunk.eq(BACKSLASH)).or(chunk.lt(ZERO)); + } } From 099340ca6a3cd9953d0ac835a4e531a8c373d7b3 Mon Sep 17 00:00:00 2001 From: Scott Myron Date: Fri, 19 Jun 2026 20:18:10 -0500 Subject: [PATCH 457/506] Push the validateUTF8String into the StringScanner to reduce the work being done if UTF-8 validation is disabled. --- java/src/json/ext/Parser.java | 2 +- java/src/json/ext/StringDecoder.java | 4 +- java/src/json/ext/StringScanner.java | 38 +++++++++++++++--- .../src/json/ext/VectorizedStringScanner.java | 39 +++++++++++++++---- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index fbe243632..e43be53e0 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -669,7 +669,7 @@ private IRubyObject parseString(boolean isName) { final byte[] data = this.data; final int contentStart = cursor + 1; // skip opening quote - long scanned = scanner.scan(data, chunks, contentStart, end); + long scanned = scanner.scan(data, chunks, contentStart, end, validateUTF8Strings); final int q = (int) scanned; if (q < 0) { throw parseError("unexpected end of input, expected closing \""); diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index 7ae96c5de..e35638a4b 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -52,8 +52,8 @@ ByteList decode(ThreadContext context, ByteList src, int start, int end) { } } - // Decodes ASCII-only strings with no UTF-8 validation. It is assumed this is only called when - // the StringScanner#scan returned a result with the ASCII_BIT bit set. + // Decodes strings with no UTF-8 validation. It is assumed the string is either ASCII-only + // or UTF-8 avlidation has been disabled. ByteList decodeNoValidate(ThreadContext context, ByteList src, int start, int end, ByteBuffer chunks) { try { init(src, start, end); diff --git a/java/src/json/ext/StringScanner.java b/java/src/json/ext/StringScanner.java index 729c81e05..605c0ad97 100644 --- a/java/src/json/ext/StringScanner.java +++ b/java/src/json/ext/StringScanner.java @@ -69,7 +69,7 @@ static StringScanner getInstance() { * {@link #ASCII_BIT} is set when the body contains no non-ASCII byte * (it may still hold escapes or control characters) */ - long scan(byte[] data, ByteBuffer chunks, int start, int end) { + long scan(byte[] data, ByteBuffer chunks, int start, int end, boolean validateUtf8) { int p = start; boolean plain = true; boolean ascii = true; @@ -86,9 +86,20 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { // clears ASCII_BIT // non-plain, !ASCII -> quote and backslash only; multi-byte // UTF-8 is skipped eight bytes at a time - long m = plain ? stringScanMask(x) - : ascii ? quoteBackslashHighMask(x) - : quoteBackslashMask(x); + // + // When UTF-8 validation is disabled, non-ASCII bytes are copied + // verbatim, so they stay on the plain fast path and the high-bit + // term drops out of every mask: plain scans for control, quote + // and backslash; non-plain scans for quote and backslash only. + long m; + if (validateUtf8) { + m = plain ? stringScanMask(x) + : ascii ? quoteBackslashHighMask(x) + : quoteBackslashMask(x); + } else { + m = plain ? controlQuoteBackslashMask(x) + : quoteBackslashMask(x); + } if (m == 0) { p += 8; } else { @@ -113,8 +124,10 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { continue outer; } if (b >= 0x80) { - plain = false; ascii = false; + if (validateUtf8) { + plain = false; + } p++; continue outer; } @@ -184,6 +197,21 @@ private static long backslashControlMask(long x) { return (control | bslash) & HIGH_BITS; } + /** + * Like {@link #stringScanMask} but omits the non-ASCII (high-bit) term: + * flags double quotes, backslashes and ASCII control characters (< 0x20) + * only. Used as the starting mask when UTF-8 validation is disabled, where + * non-ASCII bytes are copied verbatim and so stay on the plain fast path. + */ + private static long controlQuoteBackslashMask(long x) { + long control = (x - SPACES) & ~x; // bytes < 0x20 (ASCII) + long q = x ^ DOUBLE_QUOTES; + long quote = (q - ONES) & ~q; + long s = x ^ BACKSLASHES; + long bslash = (s - ONES) & ~s; + return (control | quote | bslash) & HIGH_BITS; + } + /** * Like {@link #stringScanMask} but only flags double quotes and backslashes. * Used once a string is known to require the decoder and to already diff --git a/java/src/json/ext/VectorizedStringScanner.java b/java/src/json/ext/VectorizedStringScanner.java index d7c4a8694..b63544fea 100644 --- a/java/src/json/ext/VectorizedStringScanner.java +++ b/java/src/json/ext/VectorizedStringScanner.java @@ -17,7 +17,7 @@ final class VectorizedStringScanner extends StringScanner { private static final ByteVector DQUOTE = ByteVector.broadcast(SP, '"'); @Override - long scan(byte[] data, ByteBuffer chunks, int start, int end) { + long scan(byte[] data, ByteBuffer chunks, int start, int end, boolean validateUtf8) { final int width = SP.length(); int p = start; boolean plain = true; @@ -25,14 +25,24 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { // The same structure as the StringEncoder. The logic is // duplicated for maximum inlining. + // + // When UTF-8 validation is disabled, non-ASCII lanes are copied verbatim + // and stay on the plain fast path, so the high-bit term drops out: plain + // scans for control, quote and backslash; non-plain for quote and + // backslash only. outer: while (true) { while (p + width <= end) { ByteVector chunk = ByteVector.fromArray(SP, data, p); - VectorMask interesting = - plain ? interestingLanes(chunk) - : ascii ? quoteBackslashOrHighLanes(chunk) - : quoteOrBackslashLanes(chunk); + VectorMask interesting; + if (validateUtf8) { + interesting = plain ? interestingLanes(chunk) + : ascii ? quoteBackslashOrHighLanes(chunk) + : quoteOrBackslashLanes(chunk); + } else { + interesting = plain ? controlQuoteBackslashLanes(chunk) + : quoteOrBackslashLanes(chunk); + } if (interesting.anyTrue()) { p += interesting.firstTrue(); break; @@ -50,8 +60,10 @@ long scan(byte[] data, ByteBuffer chunks, int start, int end) { continue outer; } if (b >= 0x80) { - plain = false; ascii = false; + if (validateUtf8) { + plain = false; + } p++; continue outer; } @@ -85,7 +97,9 @@ int scanEscape(byte[] data, ByteBuffer chunks, int start, int end) { } private static VectorMask escapeOrControlLanes(ByteVector chunk) { - return chunk.lt(SPACE).or(chunk.eq(BACKSLASH)); + VectorMask negative = chunk.lt(ZERO); + VectorMask control = chunk.lt(SPACE).andNot(negative); + return control.or(chunk.eq(BACKSLASH)); } private static VectorMask interestingLanes(ByteVector chunk) { @@ -100,6 +114,17 @@ private static VectorMask quoteOrBackslashLanes(ByteVector chunk) { return chunk.eq(DQUOTE).or(chunk.eq(BACKSLASH)); } + // Like interestingLanes but without the non-ASCII (high-bit) term: flags + // ASCII control characters, double quotes and backslashes only. Used as the + // plain-path mask when UTF-8 validation is disabled. + private static VectorMask controlQuoteBackslashLanes(ByteVector chunk) { + VectorMask negative = chunk.lt(ZERO); + VectorMask lowOrQuote = chunk.lanewise(VectorOperators.XOR, TWO) + .lt(THIRTY_THREE) + .andNot(negative); + return lowOrQuote.or(chunk.eq(BACKSLASH)); + } + private static VectorMask quoteBackslashOrHighLanes(ByteVector chunk) { return chunk.eq(DQUOTE).or(chunk.eq(BACKSLASH)).or(chunk.lt(ZERO)); } From cc32ef5cf58933b921de362500524ca041def7f5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sat, 26 Sep 2026 09:25:37 +0200 Subject: [PATCH 458/506] Ensure `default_sort_keys_proc` isn't reassigned. It's assignable from Ruby simply for the gem's own initialization, but it should never be re-assigned later. Fix: https://github.com/ruby/json/pull/1079 --- ext/json/ext/generator/generator.c | 3 +++ lib/json/common.rb | 8 -------- lib/json/ext.rb | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index d0d048751..0b7397c56 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1733,6 +1733,9 @@ static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc) if (!rb_obj_is_proc(proc)) { rb_raise(rb_eTypeError, "sort_key_proc must be a Proc"); } + if (default_sort_keys_proc) { + rb_raise(rb_eArgError, "sort_key_proc can only be set once"); + } return default_sort_keys_proc = proc; } diff --git a/lib/json/common.rb b/lib/json/common.rb index 82d334061..3b5e3157a 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -80,14 +80,6 @@ def parser=(parser) # :nodoc: def generator=(generator) # :nodoc: old, $VERBOSE = $VERBOSE, nil - # The default proc used when the +sort_keys+ generation option is +true+. - # It returns a new hash with the entries sorted by their keys. - sort_keys_proc = ->(hash) { hash.sort.to_h } - if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda) - sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc) - end - generator::State.default_sort_keys_proc = sort_keys_proc - @generator = generator if generator.const_defined?(:GeneratorMethods) generator_methods = generator::GeneratorMethods diff --git a/lib/json/ext.rb b/lib/json/ext.rb index da17df1e8..8c8c6f258 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -32,13 +32,22 @@ def parse Ext::Parser::Config = Ext::ParserConfig JSON.parser = Ext::Parser - if RUBY_ENGINE == 'truffleruby' + generator = if RUBY_ENGINE == 'truffleruby' require 'json/truffle_ruby/generator' - JSON.generator = JSON::TruffleRuby::Generator + JSON::TruffleRuby::Generator else require 'json/ext/generator' - JSON.generator = Generator + Generator end + + # The default proc used when the +sort_keys+ generation option is +true+. + # It returns a new hash with the entries sorted by their keys. + sort_keys_proc = ->(hash) { hash.sort.to_h } + if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda) + sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc) + end + generator::State.default_sort_keys_proc = sort_keys_proc + JSON.generator = generator end if defined?(ResumableParser) # Not yet available on JRuby From edb3b20d6d375377c42fdd9ccb4566c81324ca22 Mon Sep 17 00:00:00 2001 From: youdie006 Date: Mon, 28 Sep 2026 00:58:09 +0900 Subject: [PATCH 459/506] JRuby: keep the whole character after an invalid escape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With allow_invalid_escape, the JRuby decoder appended the code point read after a stray backslash with append(int), which writes only its low byte. "\\é" became "\xE9" and "\\あ" became "B", where the C extension keeps the character. Use writeUtf8Char, as the other code point paths do. --- java/src/json/ext/StringDecoder.java | 2 +- test/json/json_parser_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/java/src/json/ext/StringDecoder.java b/java/src/json/ext/StringDecoder.java index e35638a4b..362a0d685 100644 --- a/java/src/json/ext/StringDecoder.java +++ b/java/src/json/ext/StringDecoder.java @@ -155,7 +155,7 @@ private void handleEscapeSequence(ThreadContext context) throws IOException { break; default: if (allowInvalidEscape) { - append(character); + writeUtf8Char(character); } else { throw invalidEscape(context); } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 2fd7e2745..025bf671c 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -223,6 +223,12 @@ def test_parse_invalid_escape assert_equal "foo", parse(%("fo\\o"), allow_invalid_escape: true) end + def test_parse_invalid_escape_non_ascii + assert_equal "caf\u00e9", parse(%("caf\\\u00e9"), allow_invalid_escape: true) + assert_equal "\u3042", parse(%("\\\u3042"), allow_invalid_escape: true) + assert_equal "\u{1F600}", parse(%("\\\u{1F600}"), allow_invalid_escape: true) + end + def test_parse_arrays assert_equal([1,2,3], parse('[1,2,3]')) assert_equal([1.2,2,3], parse('[1.2,2,3]')) From f5c889de9c2a5d8ed7ee8474d46c197d7c0cf8a9 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 30 Sep 2026 22:12:20 +0900 Subject: [PATCH 460/506] Prevent crashes with non-Proc as_json callbacks Convert non-Proc callbacks through to_proc and validate the result in both configuration and the as_json setter. rb_convert_type accepts unrelated T_DATA objects without conversion, allowing them to reach Proc APIs and crash. Cover Method callbacks, invalid callback types, and to_proc returning a non-Proc object. --- ext/json/ext/generator/generator.c | 16 +++++++++++-- test/json/json_generator_test.rb | 38 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 0b7397c56..82e86e7df 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1550,6 +1550,18 @@ static VALUE cState_as_json(VALUE self) return state->as_json; } +static VALUE as_json_config(VALUE value) +{ + if (rb_obj_is_proc(value)) { + return value; + } + VALUE proc = rb_check_funcall(value, rb_intern("to_proc"), 0, NULL); + if (proc == Qundef || !rb_obj_is_proc(proc)) { + rb_raise(rb_eTypeError, "as_json must be a Proc"); + } + return proc; +} + /* * call-seq: as_json=(as_json) * @@ -1559,7 +1571,7 @@ static VALUE cState_as_json_set(VALUE self, VALUE as_json) { rb_check_frozen(self); GET_STATE(self); - RB_OBJ_WRITE(self, &state->as_json, rb_convert_type(as_json, T_DATA, "Proc", "to_proc")); + RB_OBJ_WRITE(self, &state->as_json, as_json_config(as_json)); return Qnil; } @@ -1879,7 +1891,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_strict) { state->strict = RTEST(val); } else if (key == sym_allow_duplicate_key) { state->allow_duplicate_key = RTEST(val); } else if (key == sym_as_json) { - VALUE proc = RTEST(val) ? rb_convert_type(val, T_DATA, "Proc", "to_proc") : Qfalse; + VALUE proc = RTEST(val) ? as_json_config(val) : Qfalse; state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; state_write_value(data, &state->as_json, proc); } diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1b2197735..0124521a5 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1035,6 +1035,44 @@ def test_json_generate_as_json_convert_to_proc assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: -> (o, is_key) { o.object_id }) end + def test_json_generate_as_json_method + object = Object.new + as_json = -> (o, is_key) { o.object_id }.method(:call) + assert_equal object.object_id.to_json, JSON.generate(object, strict: true, as_json: as_json) + end + + def test_state_as_json_method + object = Object.new + state = JSON.state.new(strict: true) + state.as_json = -> (o, is_key) { o.object_id }.method(:call) + assert_equal object.object_id.to_json, state.generate(object) + end + + def test_json_generate_as_json_invalid_type + omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' + [Object.new, Time.now].each do |as_json| + assert_raise(TypeError) { JSON.generate(Object.new, strict: true, as_json: as_json) } + end + end + + def test_state_as_json_invalid_type + omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' + state = JSON.state.new(strict: true) + [Object.new, Time.now].each do |as_json| + assert_raise(TypeError) { state.as_json = as_json } + end + end + + def test_as_json_to_proc_returns_invalid_type + omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' + as_json = Object.new + def as_json.to_proc + method(:to_proc) + end + assert_raise(TypeError) { JSON.generate(Object.new, strict: true, as_json: as_json) } + assert_raise(TypeError) { JSON.state.new.as_json = as_json } + end + def test_as_json_nan_does_not_call_to_json def (obj = Object.new).to_json(*) "null" From b6a5d352455b7c99e7555da7d5c4cb9a8a987ee8 Mon Sep 17 00:00:00 2001 From: Lucas Dohmen Date: Wed, 30 Sep 2026 13:05:54 +0200 Subject: [PATCH 461/506] Fix Interoperability with Zlib::GzipReader GzipReader offers both a read and to_io method. The read method returns the uncompressed file as a string. The to_io method returns the compressed file. The JSON.load method first tests for the to_io method and uses it before testing for the read method. This results in reading the compressed file, which is invalid JSON. Instead, we now test for the read method first. --- lib/json/common.rb | 4 ++-- test/json/json_common_interface_test.rb | 14 ++++++++++++-- test/json/test_helper.rb | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 3b5e3157a..a1603c06c 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -699,10 +699,10 @@ def load(source, proc = nil, allow_blank: true, **options) unless source.is_a?(String) if source.respond_to? :to_str source = source.to_str + elsif source.respond_to? :read + source = source.read elsif source.respond_to? :to_io source = source.to_io.read - elsif source.respond_to?(:read) - source = source.read end end diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index e4bf958d4..4fb161569 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -122,15 +122,25 @@ def o.to_s def test_load assert_equal @hash, JSON.load(@json) + assert_equal nil, JSON.load(nil) + assert_equal nil, JSON.load('') + tempfile = Tempfile.open('@json') tempfile.write @json tempfile.rewind assert_equal @hash, JSON.load(tempfile) + stringio = StringIO.new(@json) stringio.rewind assert_equal @hash, JSON.load(stringio) - assert_equal nil, JSON.load(nil) - assert_equal nil, JSON.load('') + + tempfile = Tempfile.open('@json.gzip') + gzw = Zlib::GzipWriter.new(tempfile) + gzw.write(@json) + gzw.close + tempfile.open + gzr = Zlib::GzipReader.new(tempfile) + assert_equal @hash, JSON.load(gzr) ensure tempfile.close! end diff --git a/test/json/test_helper.rb b/test/json/test_helper.rb index a30885c40..222ba86df 100644 --- a/test/json/test_helper.rb +++ b/test/json/test_helper.rb @@ -31,6 +31,7 @@ require 'test/unit' require 'stringio' require 'tempfile' +require 'zlib' begin require 'bigdecimal' rescue LoadError From 672da19fd50f86efe05ace7ebf37137b14b4e3f0 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 30 Sep 2026 22:29:22 +0900 Subject: [PATCH 462/506] Prevent crashes with non-Proc on_load callbacks Convert non-Proc on_load callbacks through to_proc and validate the result in the shared parser configuration. Previously, arbitrary values were stored and passed directly to rb_proc_call_with_block. Cover Method callbacks in JSON.parse, JSON::Coder, and ResumableParser, along with invalid types, invalid conversion results, and nil or false callbacks. --- ext/json/ext/parser/parser.c | 10 +++++++++- test/json/json_coder_test.rb | 10 ++++++++++ test/json/json_parser_test.rb | 26 ++++++++++++++++++++++++++ test/json/resumable_parser_test.rb | 8 ++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 5cba2feb6..c2255b3cc 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2021,7 +2021,15 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) else if (key == sym_allow_invalid_escape) { config->allow_invalid_escape = RTEST(val); } else if (key == sym_symbolize_names) { config->symbolize_names = RTEST(val); } else if (key == sym_freeze) { config->freeze = RTEST(val); } - else if (key == sym_on_load) { parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); } + else if (key == sym_on_load) { + if (RTEST(val) && !rb_obj_is_proc(val)) { + val = rb_check_funcall(val, rb_intern("to_proc"), 0, NULL); + if (val == Qundef || !rb_obj_is_proc(val)) { + rb_raise(rb_eTypeError, "on_load must be a Proc"); + } + } + parser_config_wb_write(self, &config->on_load_proc, RTEST(val) ? val : Qfalse); + } else if (key == sym_allow_duplicate_key) { config->allow_duplicate_key = RTEST(val); } else if (key == sym_decimal_class) { if (RTEST(val)) { diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 0e5e5e2f2..5ff314478 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -56,6 +56,16 @@ def test_json_coder_load_options assert_equal({a: 1}, coder.load('{"a":1}')) end + def test_json_coder_load_with_on_load_method + on_load = ->(value) { Integer === value ? value + 1 : value }.method(:call) + coder = JSON::Coder.new(on_load: on_load) + assert_equal [2], coder.load('[1]') + end + + def test_json_coder_on_load_invalid_type + assert_raise(TypeError) { JSON::Coder.new(on_load: 'x') } + end + def test_json_coder_dump_NaN_or_Infinity coder = JSON::Coder.new { |o| o.inspect } assert_equal "NaN", coder.load(coder.dump(Float::NAN)) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 025bf671c..be2807d30 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -38,6 +38,32 @@ def test_parsing assert_equal 'test', parser.parse end + def test_on_load_method + on_load = ->(value) { Integer === value ? value + 1 : value }.method(:call) + assert_equal [2], JSON.parse('[1]', on_load: on_load) + end + + def test_on_load_invalid_type + ['x', Object.new, Time.now].each do |on_load| + assert_raise(TypeError) { JSON.parse('[1]', on_load: on_load) } + end + end + + def test_on_load_invalid_to_proc + on_load = Object.new + def on_load.to_proc + method(:to_proc) + end + assert_raise(TypeError) { JSON.parse('[1]', on_load: on_load) } + end + + def test_on_load_falsy + [nil, false].each do |on_load| + config = JSON::Parser::Config.new(on_load: on_load) + assert_equal [1], config.parse('[1]') + end + end + def test_parser_reset parser = Parser.new('{"a":"b"}') assert_equal({ 'a' => 'b' }, parser.parse) diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 2777d50c5..b663d3a67 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -106,6 +106,14 @@ def test_nested_parse_error end end + def test_on_load_method + on_load = ->(value) { Integer === value ? value + 1 : value }.method(:call) + parser = new_parser(on_load: on_load) + parser << '[1]' + assert parser.parse + assert_equal [2], parser.value + end + def test_parse_document_direct @parser << '[true]' assert_equal true, @parser.parse From 452f31b56964c36409e10b533541cb2ecd460e4b Mon Sep 17 00:00:00 2001 From: ydah Date: Thu, 1 Oct 2026 12:24:08 +0900 Subject: [PATCH 463/506] Prevent ResumableParser from stalling on invalid object keys --- ext/json/ext/parser/parser.c | 2 +- test/json/resumable_parser_test.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c2255b3cc..d1d33b14a 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1815,7 +1815,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo if (RB_LIKELY(peek(state) == '"')) { VALUE string = json_parse_string(state, config, true); if (UNDEF_P(string)) { - if (resumable) { + if (resumable && eos(state)) { state->cursor = start; return false; } else { diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index b663d3a67..a704e9edd 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -170,6 +170,7 @@ def test_parse_byte_by_byte_object assert_resumed_parsing('{ }') assert_resumed_parsing('{"test" : true}') assert_resumed_parsing('{ "test":12, "value" : { "key": 42} }') + assert_resumed_parsing('{"te\u0000st":true}') end def test_parse_byte_by_byte_string @@ -234,6 +235,18 @@ def test_nul_byte_is_a_syntax_error assert_parse_error "{\"a\":1,\x00}" # object key after ',' end + def test_nul_after_backslash_in_object_key_is_a_syntax_error + assert_parse_error "{\"key\\\x00\":1}" + assert_parse_error "{\"a\":1,\"key\\\x00\":2}" + end + + def test_nul_after_backslash_in_object_key_across_feeds + @parser << "{\"key\\" + refute @parser.parse + @parser << "\x00\":1}" + assert_raise(JSON::ParserError) { @parser.parse } + end + def test_incomplete_input_at_structural_positions_resumes # Counterpart of test_nul_byte_is_a_syntax_error: a genuine EOS at the same positions must # stay incomplete (return false), not raise -- this is what distinguishes EOS from a NUL. From f7206147284f077b200af9867f4debe5948df419 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 30 Sep 2026 23:11:55 +0900 Subject: [PATCH 464/506] Prevent use-after-free when strings change during IO generation --- ext/json/ext/fbuffer/fbuffer.h | 4 +++ ext/json/ext/generator/generator.c | 10 +++++-- java/src/json/ext/Generator.java | 2 ++ java/src/json/ext/StringEncoder.java | 7 +++-- test/json/json_generator_test.rb | 39 ++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/fbuffer/fbuffer.h b/ext/json/ext/fbuffer/fbuffer.h index cc44ea030..3f95f1f76 100644 --- a/ext/json/ext/fbuffer/fbuffer.h +++ b/ext/json/ext/fbuffer/fbuffer.h @@ -169,6 +169,10 @@ static inline void fbuffer_append_reserved_char(FBuffer *fb, char chr) static void fbuffer_append_str(FBuffer *fb, VALUE str) { + if (RB_UNLIKELY(fb->io)) { + // Growing the buffer may flush it and invoke arbitrary Ruby code. + str = rb_str_new_frozen(str); + } const char *ptr; size_t len; RSTRING_GETMEM(str, ptr, len); diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 82e86e7df..3c961240e 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -892,12 +892,17 @@ ALWAYS_INLINE(static) VALUE ensure_valid_encoding(struct generate_json_data *dat static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { + VALUE str = obj; + if (RB_UNLIKELY(buffer->io)) { + // IO writes can mutate the original string while we are reading it. + str = rb_str_new_frozen(str); + } fbuffer_append_char(buffer, '"'); long len; search_state search; search.buffer = buffer; - RSTRING_GETMEM(obj, search.ptr, len); + RSTRING_GETMEM(str, search.ptr, len); search.cursor = search.ptr; search.end = search.ptr + len; @@ -908,7 +913,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data search.chunk_end = NULL; #endif /* HAVE_SIMD */ - switch (json_str_coderange(obj)) { + switch (json_str_coderange(str)) { case ENC_CODERANGE_7BIT: case ENC_CODERANGE_VALID: if (RB_UNLIKELY(data->state->ascii_only)) { @@ -924,6 +929,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data break; } fbuffer_append_char(buffer, '"'); + RB_GC_GUARD(str); } static void generate_json_string(FBuffer *buffer, struct generate_json_data *data, VALUE obj) diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index ce1b5b57e..fddb0ea32 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -722,6 +722,7 @@ static RubyString generateFragmentNew(ThreadContext context, Session session, IR static void generateFragment(ThreadContext context, Session session, IRubyObject object, OutputStream buffer) throws IOException { RubyString result = generateFragmentNew(context, session, object); + if (buffer instanceof BufferedOutputStream) result = result.newFrozen(); ByteList bytes = result.getByteList(); buffer.write(bytes.unsafeBytes(), bytes.begin(), bytes.length()); } @@ -814,6 +815,7 @@ static RubyString generateGenericNew(ThreadContext context, Session session, IRu static void generateGeneric(ThreadContext context, Session session, IRubyObject object, OutputStream buffer) throws IOException { RubyString result = generateGenericNew(context, session, object); + if (buffer instanceof BufferedOutputStream) result = result.newFrozen(); ByteList bytes = result.getByteList(); buffer.write(bytes.unsafeBytes(), bytes.begin(), bytes.length()); } diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index 4eea11c62..e059fc993 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -5,6 +5,7 @@ */ package json.ext; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.Constructor; @@ -192,11 +193,13 @@ static StringEncoder createBasicEncoder() { void generate(ThreadContext context, RubyString object, OutputStream buffer) throws IOException { object = ensureValidEncoding(context, object); - ByteList byteList = object.getByteList(); + // IO writes can mutate the original string while we are reading it. + RubyString source = buffer instanceof BufferedOutputStream ? object.newFrozen() : object; + ByteList byteList = source.getByteList(); init(byteList); out = buffer; append('"'); - switch (object.scanForCodeRange()) { + switch (source.scanForCodeRange()) { case StringSupport.CR_7BIT: case StringSupport.CR_VALID: encode(byteList); diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 0124521a5..a901840d9 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -91,6 +91,45 @@ def test_dump_deprecated_limit assert_equal '[1]', io.string end + def test_dump_string_mutated_during_io_write + [{}, {ascii_only: true}, {script_safe: true}, {buffer_initial_length: 1}].each do |options| + ['a', "a\né/"].each do |pattern| + string = pattern * 100_000 + assert_dump_preserves_string_during_io_write(string, string, options) + end + end + end + + def test_dump_fragment_mutated_during_io_write + string = JSON.generate('a' * 100_000) + assert_dump_preserves_string_during_io_write(JSON::Fragment.new(string), string) + end + + def test_dump_to_json_result_mutated_during_io_write + string = JSON.generate('a' * 100_000) + object = Object.new + object.define_singleton_method(:to_json) { |*| string } + assert_dump_preserves_string_during_io_write(object, string) + end + + def assert_dump_preserves_string_during_io_write(object, string, options = {}) + expected = JSON.dump([object], options) + io = StringIO.new + mutated = false + io.define_singleton_method(:write) do |chunk| + unless mutated + string.setbyte(0, 'b'.ord) + string.replace('changed') + mutated = true + GC.start + end + super(chunk) + end + assert_same io, JSON.dump([object], io, options) + assert_equal 'changed', string + assert_equal true, expected == io.string, 'IO output must preserve the original string' + end + def test_not_frozen [ [[], '[]'], From 3643231dfd32287a09d637aeaf86ea2776b193e8 Mon Sep 17 00:00:00 2001 From: ydah Date: Thu, 1 Oct 2026 15:39:12 +0900 Subject: [PATCH 465/506] Correct State as_json and depth setter documentation --- ext/json/ext/generator/generator.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 3c961240e..be490171f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1548,7 +1548,7 @@ static VALUE cState_array_nl_set(VALUE self, VALUE array_nl) /* * call-seq: as_json() * - * This string is put at the end of a line that holds a JSON array. + * Returns the callback used to convert unsupported objects in strict mode. */ static VALUE cState_as_json(VALUE self) { @@ -1571,7 +1571,7 @@ static VALUE as_json_config(VALUE value) /* * call-seq: as_json=(as_json) * - * This string is put at the end of a line that holds a JSON array. + * Sets the Proc used to convert unsupported objects in strict mode. */ static VALUE cState_as_json_set(VALUE self, VALUE as_json) { @@ -1818,8 +1818,7 @@ static VALUE cState_depth(VALUE self) /* * call-seq: depth=(depth) * - * This sets the maximum level of data structure nesting in the generated JSON - * to the integer depth, max_nesting = 0 if no maximum should be checked. + * Sets the current depth of data structure nesting to the integer +depth+. */ static VALUE cState_depth_set(VALUE self, VALUE depth) { From 0c46c27fae7d757bc5fd081175d39529e2a7997f Mon Sep 17 00:00:00 2001 From: ydah Date: Thu, 1 Oct 2026 15:39:12 +0900 Subject: [PATCH 466/506] Remove the leading space from Fragment conversion errors --- lib/json/common.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index a1603c06c..b07f80809 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -224,7 +224,7 @@ def detailed_message(...) Fragment = Struct.new(:json) do def initialize(json) unless string = String.try_convert(json) - raise TypeError, " no implicit conversion of #{json.class} into String" + raise TypeError, "no implicit conversion of #{json.class} into String" end super(string) From 0edc03a14ebccc6639f9bedededd2a631c1a2a1c Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 1 Oct 2026 19:38:28 +0900 Subject: [PATCH 467/506] Fix tempfile leak The `tempfile` variable refers to different things in the first half and the second half. --- test/json/json_common_interface_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 4fb161569..0fbc2df7a 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -129,6 +129,7 @@ def test_load tempfile.write @json tempfile.rewind assert_equal @hash, JSON.load(tempfile) + tempfile.close! stringio = StringIO.new(@json) stringio.rewind From c484f8fa82f9ec9b2294e80e7c27fe270230235c Mon Sep 17 00:00:00 2001 From: ydah Date: Fri, 2 Oct 2026 20:28:26 +0900 Subject: [PATCH 468/506] Preserve complete UTF-8 characters in parse error snippets --- ext/json/ext/parser/parser.c | 16 +++++++--------- test/json/json_parser_test.rb | 27 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d1d33b14a..a13a0e935 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -631,17 +631,15 @@ static VALUE build_parse_error_message(const char *format, JSON_ParserState *sta } if (len) { - buffer[0] = '\''; - MEMCPY(buffer + 1, ptr, char, len); - - while (buffer[len] >= 0x80 && buffer[len] < 0xC0) { // Is continuation byte - len--; - } - - if (buffer[len] >= 0xC0) { // multibyte character start - len--; + if (len == PARSE_ERROR_FRAGMENT_LEN) { + // Only trim when the byte limit splits a multibyte character. + while (len && (unsigned char)ptr[len] >= 0x80 && (unsigned char)ptr[len] < 0xC0) { + len--; + } } + buffer[0] = '\''; + MEMCPY(buffer + 1, ptr, char, len); buffer[len + 1] = '\''; buffer[len + 2] = '\0'; ptr = (const char *)buffer; diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index be2807d30..2c856341f 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -868,7 +868,7 @@ def test_parse_error_snippet assert_equal "unexpected character: 'aああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abあああああああああああああああああああああああ") } - assert_equal "unexpected character: 'abあああああああああ' at line 1 column 1", error.message + assert_equal "unexpected character: 'abああああああああああ' at line 1 column 1", error.message error = assert_raise(JSON::ParserError) { JSON.parse("abcあああああああああああああああああああああああ") } assert_equal "unexpected character: 'abcあああああああああ' at line 1 column 1", error.message @@ -881,6 +881,31 @@ def test_parse_error_snippet error = assert_raise(JSON::ParserError) { JSON.parse("@") } assert_equal "unexpected character: '@' at line 1 column 1", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse('"a" 日本') } + assert_equal "unexpected token at end of stream '日本' at line 1 column 5", error.message + + error = assert_raise(JSON::ParserError) { JSON.parse('[1, é') } + assert_equal "unexpected character: 'é' at line 1 column 5", error.message + end + + def test_parse_error_snippet_multibyte_boundaries + omit "JRuby errors don't contain positions" if RUBY_ENGINE == "jruby" + + ['é', '日', '😀'].each do |char| + ['', ' ', "\n", "\t", "\r"].each do |suffix| + error = assert_raise(JSON::ParserError) { JSON.parse(char + suffix) } + assert_equal "unexpected character: '#{char}' at line 1 column 1", error.message + end + + (28..32).each do |padding| + prefix = 'a' * padding + fragment = prefix + char * ((32 - padding) / char.bytesize) + error = assert_raise(JSON::ParserError) { JSON.parse(prefix + char * 20) } + assert_equal "unexpected character: '#{fragment}' at line 1 column 1", error.message + assert_predicate error.message, :valid_encoding? + end + end end def test_parse_error_json_path From 590dd5448d594dc763d8ef1f77f7ea4abaced683 Mon Sep 17 00:00:00 2001 From: ydah Date: Fri, 2 Oct 2026 21:51:51 +0900 Subject: [PATCH 469/506] Fix parser error positions for newlines and multibyte characters --- ext/json/ext/parser/parser.c | 14 +++++--------- java/src/json/ext/Parser.java | 20 +++++++------------- lib/json/common.rb | 2 +- test/json/json_parser_test.rb | 27 +++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index a13a0e935..c912a8eec 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -592,19 +592,15 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum } const char *cursor = state->cursor; - long column = 0; long line = 1; - while (cursor >= state->start) { - if (*cursor-- == '\n') { - line++; - break; - } - column++; + while (cursor > state->start && cursor[-1] != '\n') { + cursor--; } + long column = rb_enc_strlen(cursor, state->cursor, enc_utf8) + 1; - while (cursor >= state->start) { - if (*cursor-- == '\n') { + while (cursor > state->start) { + if (*--cursor == '\n') { line++; } } diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index e43be53e0..88bc054ef 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -22,6 +22,7 @@ import org.jruby.runtime.builtin.IRubyObject; import org.jruby.util.ByteList; import org.jruby.util.ConvertBytes; +import org.jruby.util.StringSupport; import org.jruby.util.ConvertDouble.DoubleConverter; import org.jruby.util.ConvertBytes; import org.jruby.util.ConvertDouble.DoubleConverter; @@ -897,22 +898,15 @@ private IRubyObject getConstant(String name) { } private long cursorPosition(int position) { - long column = 0; + position = Math.min(position, end); int i = position; - if (i >= end) { - column = i - end + 1; - i = end - 1; - } int line = 1; - while (i >= begin) { - if (data[i--] == '\n') { - line++; - break; - } - column++; + while (i > begin && data[i - 1] != '\n') { + i--; } - while (i >= begin) { - if (data[i--] == '\n') { + long column = StringSupport.strLength(UTF8Encoding.INSTANCE, data, i, position) + 1; + while (i > begin) { + if (data[--i] == '\n') { line++; } } diff --git a/lib/json/common.rb b/lib/json/common.rb index b07f80809..e1a8f6814 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -138,7 +138,7 @@ class ParserError < JSONError # Is +nil+ when raised by JSON::ResumableParser. attr_reader :line - # Column number where the parser encountered an error. + # One-based column number where the parser encountered an error, counted in Unicode codepoints. # Is +nil+ when raised by JSON::ResumableParser. attr_reader :column diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 2c856341f..d9e3256bd 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -935,6 +935,33 @@ def test_parse_error_position assert_equal 17, error.column end + def test_parse_error_position_at_newline + omit "JRuby string decoding errors don't contain positions" if RUBY_ENGINE == "jruby" + + ["[1,\n\"ab\ncd\"]", "[1,\n\"日本\ncd\"]"].each do |json| + error = assert_raise(JSON::ParserError) { JSON.parse(json) } + assert_equal 2, error.line + assert_equal 4, error.column + assert_include error.message, 'at line 2 column 4' + end + end + + def test_parse_error_position_with_multibyte_characters + { + '["日本", @]' => [1, 8], + "[\n\"é日😀\", @" => [2, 8], + '{"日本":{"a":1,"a":2}}' => [1, 7], + '["日本",' => [1, 7], + "[1,\n" => [2, 1], + '' => [1, 1], + }.each do |json, (line, column)| + error = assert_raise(JSON::ParserError) { JSON.parse(json) } + assert_equal line, error.line + assert_equal column, error.column + assert_include error.message, "at line #{line} column #{column}" + end + end + def test_parse_error_json_path_on_load assert_parse_error_at "$" do JSON.load('{"a": {"b": {"c":', -> (obj) { From f103c8953fa7692c29a24ba38d7a5b34b7a79d21 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 2 Oct 2026 14:14:10 +0200 Subject: [PATCH 470/506] Implement rfc8785 Fix: https://github.com/ruby/json/issues/1038 Directly lift https://github.com/dryruby/json-canonicalization At least for now, implementing most of the logic in Ruby make it shareable between the 3 generator implementations, largely reducing the maintenance cost. I don't particularly expect canonicalization to be a hotspot, but if it happens to be, I'd be open to specialize the implementation, within reason, at least in the C generator. --- ext/json/ext/generator/generator.c | 80 +++++++++++++++++-- java/src/json/ext/Generator.java | 17 +++- java/src/json/ext/GeneratorState.java | 34 ++++++++ lib/json.rb | 2 + lib/json/ext.rb | 61 ++++++++++++-- lib/json/ext/generator/state.rb | 3 +- lib/json/truffle_ruby/generator.rb | 38 +++++++-- test/json/fixtures/rfc8785/input/arrays.json | 8 ++ test/json/fixtures/rfc8785/input/french.json | 6 ++ .../fixtures/rfc8785/input/structures.json | 8 ++ test/json/fixtures/rfc8785/input/unicode.json | 3 + test/json/fixtures/rfc8785/input/values.json | 5 ++ test/json/fixtures/rfc8785/input/wierd.json | 11 +++ test/json/fixtures/rfc8785/output/arrays.json | 1 + test/json/fixtures/rfc8785/output/french.json | 1 + .../fixtures/rfc8785/output/structures.json | 1 + .../json/fixtures/rfc8785/output/unicode.json | 1 + test/json/fixtures/rfc8785/output/values.json | 1 + test/json/fixtures/rfc8785/output/wierd.json | 1 + test/json/json_generator_test.rb | 32 ++++++++ 20 files changed, 292 insertions(+), 22 deletions(-) create mode 100644 test/json/fixtures/rfc8785/input/arrays.json create mode 100644 test/json/fixtures/rfc8785/input/french.json create mode 100644 test/json/fixtures/rfc8785/input/structures.json create mode 100644 test/json/fixtures/rfc8785/input/unicode.json create mode 100644 test/json/fixtures/rfc8785/input/values.json create mode 100644 test/json/fixtures/rfc8785/input/wierd.json create mode 100644 test/json/fixtures/rfc8785/output/arrays.json create mode 100644 test/json/fixtures/rfc8785/output/french.json create mode 100644 test/json/fixtures/rfc8785/output/structures.json create mode 100644 test/json/fixtures/rfc8785/output/unicode.json create mode 100644 test/json/fixtures/rfc8785/output/values.json create mode 100644 test/json/fixtures/rfc8785/output/wierd.json diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index be490171f..fbef1980b 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -27,14 +27,16 @@ typedef struct JSON_Generator_StateStruct { bool ascii_only; bool script_safe; bool strict; + bool rfc8785; VALUE sort_keys; } JSON_Generator_State; -static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc; +static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc, + rfc8785_number_formater_proc, rfc8785_sort_keys_proc; static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, - sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys; + sym_ascii_only, sym_depth, sym_buffer_initial_length, sym_script_safe, sym_strict, sym_as_json, sym_sort_keys, sym_rfc8785; #define GET_STATE_TO(self, state) \ @@ -1052,7 +1054,10 @@ static inline long increase_depth(struct generate_json_data *data) static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { - if (RB_UNLIKELY(data->state->sort_keys)) { + if (RB_UNLIKELY(data->state->rfc8785)) { + obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 1, &obj, Qnil); + Check_Type(obj, T_HASH); + } else if (RB_UNLIKELY(data->state->sort_keys)) { obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil); Check_Type(obj, T_HASH); } @@ -1180,6 +1185,13 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data return; } + if (RB_UNLIKELY(data->state->rfc8785)) { + VALUE str = rb_proc_call_with_block(rfc8785_number_formater_proc, 1, &obj, Qnil); + Check_Type(str, T_STRING); + fbuffer_append_str(buffer, str); + return; + } + /* This implementation writes directly into the buffer. We reserve * the 32 characters that fpconv_dtoa states as its maximum. */ @@ -1757,6 +1769,28 @@ static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc) return default_sort_keys_proc = proc; } +static VALUE cState_set_rfc8785_number_formater_proc(VALUE self, VALUE proc) +{ + if (!rb_obj_is_proc(proc)) { + rb_raise(rb_eTypeError, "rfc8785_number_formater_proc must be a Proc"); + } + if (rfc8785_number_formater_proc) { + rb_raise(rb_eArgError, "rfc8785_number_formater_proc can only be set once"); + } + return rfc8785_number_formater_proc = proc; +} + +static VALUE cState_set_rfc8785_sort_keys_proc(VALUE self, VALUE proc) +{ + if (!rb_obj_is_proc(proc)) { + rb_raise(rb_eTypeError, "rfc8785_sort_keys_proc must be a Proc"); + } + if (rfc8785_sort_keys_proc) { + rb_raise(rb_eArgError, "rfc8785_sort_keys_proc can only be set once"); + } + return rfc8785_sort_keys_proc = proc; +} + static VALUE normalize_sort_keys(VALUE value) { if (rb_obj_is_proc(value)) { @@ -1775,7 +1809,7 @@ static VALUE normalize_sort_keys(VALUE value) * * Get the value of sort_keys. */ -static VALUE cState_sort_keys_p(VALUE self) +static VALUE cState_sort_keys(VALUE self) { GET_STATE(self); return state->sort_keys; @@ -1795,7 +1829,33 @@ static VALUE cState_sort_keys_set(VALUE self, VALUE value) rb_check_frozen(self); GET_STATE(self); RB_OBJ_WRITE(self, &state->sort_keys, normalize_sort_keys(value)); - return Qnil; + return state->sort_keys; +} + + +/* + * call-seq: rfc8785? + * + * Get whether RFC8785 (canonicalization) is enabled. + */ +static VALUE cState_rfc8785_p(VALUE self) +{ + GET_STATE(self); + return state->rfc8785 ? Qtrue : Qfalse; +} + +/* + * call-seq: rfc8785= + * + * Sets whether RFC8785 (canonicalization) is enabled. + */ + +static VALUE cState_rfc8785_set(VALUE self, VALUE value) +{ + rb_check_frozen(self); + GET_STATE(self); + state->rfc8785 = RTEST(value); + return state->rfc8785 ? Qtrue : Qfalse; } static VALUE cState_allow_duplicate_key_p(VALUE self) @@ -1895,6 +1955,7 @@ static int configure_state_i(VALUE key, VALUE val, VALUE _arg) else if (key == sym_script_safe) { state->script_safe = RTEST(val); } else if (key == sym_strict) { state->strict = RTEST(val); } else if (key == sym_allow_duplicate_key) { state->allow_duplicate_key = RTEST(val); } + else if (key == sym_rfc8785) { state->rfc8785 = RTEST(val); } else if (key == sym_as_json) { VALUE proc = RTEST(val) ? as_json_config(val) : Qfalse; state->as_json_single_arg = proc && rb_proc_arity(proc) == 1; @@ -1990,6 +2051,8 @@ void Init_generator(void) VALUE mGenerator = rb_define_module_under(mExt, "Generator"); rb_global_variable(&default_sort_keys_proc); + rb_global_variable(&rfc8785_number_formater_proc); + rb_global_variable(&rfc8785_sort_keys_proc); rb_global_variable(&eGeneratorError); eGeneratorError = rb_path2class("JSON::GeneratorError"); @@ -2001,6 +2064,8 @@ void Init_generator(void) rb_define_alloc_func(cState, cState_s_allocate); rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1); rb_define_singleton_method(cState, "default_sort_keys_proc=", cState_set_default_sort_keys_proc, 1); + rb_define_singleton_method(cState, "rfc8785_number_formater_proc=", cState_set_rfc8785_number_formater_proc, 1); + rb_define_singleton_method(cState, "rfc8785_sort_keys_proc=", cState_set_rfc8785_sort_keys_proc, 1); rb_define_method(cState, "initialize", cState_initialize, -1); rb_define_alias(cState, "initialize", "initialize"); // avoid method redefinition warnings @@ -2038,8 +2103,10 @@ void Init_generator(void) rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1); rb_define_method(cState, "generate", cState_generate, -1); rb_define_method(cState, "_generate_no_fallback", cState_generate_no_fallback, -1); - rb_define_method(cState, "sort_keys", cState_sort_keys_p, 0); + rb_define_method(cState, "sort_keys", cState_sort_keys, 0); rb_define_method(cState, "sort_keys=", cState_sort_keys_set, 1); + rb_define_method(cState, "rfc8785?", cState_rfc8785_p, 0); + rb_define_method(cState, "rfc8785=", cState_rfc8785_set, 1); rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0); @@ -2069,6 +2136,7 @@ void Init_generator(void) sym_as_json = ID2SYM(rb_intern("as_json")); sym_allow_duplicate_key = ID2SYM(rb_intern("allow_duplicate_key")); sym_sort_keys = ID2SYM(rb_intern("sort_keys")); + sym_rfc8785 = ID2SYM(rb_intern("rfc8785")); usascii_encindex = rb_usascii_encindex(); utf8_encindex = rb_utf8_encindex(); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index fddb0ea32..a38c31d83 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -403,11 +403,10 @@ void generate(ThreadContext context, Session session, RubyFloat object, OutputSt } static void generateFloat(ThreadContext context, Session session, RubyFloat object, OutputStream buffer) throws IOException { + final GeneratorState state = session.getState(context); double value = object.getValue(); if (Double.isInfinite(value) || Double.isNaN(value)) { - GeneratorState state = session.getState(context); - if (!state.allowNaN()) { if (state.strict() && state.getAsJSON() != null) { IRubyObject castedValue = state.getAsJSON().call(context, object, context.getRuntime().getFalse()); @@ -421,7 +420,13 @@ static void generateFloat(ThreadContext context, Session session, RubyFloat obje } } - buffer.write(Double.toString(value).getBytes(UTF_8)); + if (state.rfc8785()) { + RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormaterProc; + RubyString numberString = (RubyString)Helpers.invoke(context, numberProc, "call", object); + buffer.write(numberString.toString().getBytes(UTF_8)); + } else { + buffer.write(Double.toString(value).getBytes(UTF_8)); + } } private static final byte[] EMPTY_ARRAY_BYTES = "[]".getBytes(); @@ -570,8 +575,12 @@ static void generateHash(ThreadContext context, Session session, RubyHash object } RubyProc sortKeysProc = state.getSortKeysProc(); + if (state.rfc8785()) { + sortKeysProc = (RubyProc)GeneratorState.rfc8785SortKeysProc; + } + if (sortKeysProc != null) { - object = (RubyHash) Helpers.invoke(context, sortKeysProc, "call", object); + object = (RubyHash)Helpers.invoke(context, sortKeysProc, "call", object); } final ByteList objectNl = state.getObjectNl(); diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 3fdf4f276..778500ef7 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -37,6 +37,8 @@ public class GeneratorState extends RubyObject { private boolean allowDuplicateKey = false; private static IRubyObject defaultSortKeyProc; + public static IRubyObject rfc8785NumberFormaterProc; + public static IRubyObject rfc8785SortKeysProc; /** * The indenting unit string. Will be repeated several times for larger @@ -112,6 +114,8 @@ public class GeneratorState extends RubyObject { */ private IRubyObject sortKeys; + private boolean rfc8785 = false; + /** * The current depth (inside a #to_json call) */ @@ -172,6 +176,18 @@ public static IRubyObject setDefaultSortKeyProc(IRubyObject klass, IRubyObject p return proc; } + @JRubyMethod(meta=true, name="rfc8785_number_formater_proc=") + public static IRubyObject setRfc8785NumberFormaterProc(IRubyObject klass, IRubyObject proc) { + rfc8785NumberFormaterProc = proc; + return proc; + } + + @JRubyMethod(meta=true, name="rfc8785_sort_keys_proc=") + public static IRubyObject setRfc8785SortKeysProc(IRubyObject klass, IRubyObject proc) { + rfc8785SortKeysProc = proc; + return proc; + } + /** * State#initialize(opts = {}) *

@@ -493,6 +509,22 @@ public IRubyObject sort_keys_set(ThreadContext context, IRubyObject sortKeys) { return sortKeys; } + @JRubyMethod(name="rfc8785?") + public IRubyObject rfc8785_p(ThreadContext context) { + return RubyBoolean.newBoolean(context, rfc8785); + } + + @JRubyMethod(name="rfc8785=") + public IRubyObject rfc8785_set(ThreadContext context, IRubyObject rfc8785) { + checkFrozen(); + this.rfc8785 = rfc8785.isTrue(); + return rfc8785; + } + + public boolean rfc8785() { + return this.rfc8785; + } + public int getDepth() { return depth; } @@ -568,6 +600,8 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { if (opts.hasKey("sort_keys")) sortKeys = normalizeSortKeys(context, opts.get("sort_keys")); + rfc8785 = opts.getBool("rfc8785", rfc8785); + opts.ensureEmpty(); return this; diff --git a/lib/json.rb b/lib/json.rb index f5f5ade89..576d86de9 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -414,6 +414,8 @@ # When +true+, keys are sorted lexicographically. When a \Proc, it receives # the entire \Hash and must return a \Hash with its pairs in the desired # order, allowing for arbitrary sort orders. +# - Option +rfc8785+ (boolean) controls whether the generated JSON will be canonicalized +# as defined in RFC8785. # # In this example, +obj+ is used first to generate the shortest # \JSON data (no whitespace), then again with all formatting options diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 8c8c6f258..19683f024 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -28,6 +28,18 @@ def parse end end + class << self + if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda) + def shareable_lambda(block) # :nodoc: + Ractor.shareable_lambda(&block) + end + else + def shareable_lambda(block) # :nodoc: + block + end + end + end + require 'json/ext/parser' Ext::Parser::Config = Ext::ParserConfig JSON.parser = Ext::Parser @@ -42,11 +54,50 @@ def parse # The default proc used when the +sort_keys+ generation option is +true+. # It returns a new hash with the entries sorted by their keys. - sort_keys_proc = ->(hash) { hash.sort.to_h } - if defined?(::Ractor) && Ractor.respond_to?(:shareable_lambda) - sort_keys_proc = Ractor.shareable_lambda(&sort_keys_proc) - end - generator::State.default_sort_keys_proc = sort_keys_proc + generator::State.default_sort_keys_proc = shareable_lambda(->(hash) { + hash.sort.to_h + }) + + # Directly lifted from Gregg Kellogg's json-canonicalization + generator::State.rfc8785_number_formater_proc = shareable_lambda(->(num) { + if num.zero? + "0" + else + if num < 0 + num, sign = -num, '-' + end + native_rep = "%.15E" % num + decimal, exponential = native_rep.split('E') + exp_val = exponential.to_i + exponential = exp_val > 0 ? ('+' + exp_val.to_s) : exp_val.to_s + + integral, fractional = decimal.split('.') + fractional = fractional.sub(/0+$/, '') # Remove trailing zeros + + if exp_val > 0 && exp_val < 21 + while exp_val > 0 + integral += fractional.to_s[0] || '0' + fractional = fractional.to_s[1..-1] + exp_val -= 1 + end + exponential = nil + elsif exp_val == 0 + exponential = nil + elsif exp_val < 0 && exp_val > -7 + # Small numbers are shown as 0.etc with e-6 as lower limit + fractional, integral, exponential = integral + fractional.to_s, '0', nil + fractional = ("0" * (-exp_val - 1)) + fractional + end + + fractional = nil if fractional.to_s.empty? + sign.to_s + integral + (fractional ? ".#{fractional}" : '') + (exponential ? "e#{exponential}" : '') + end + }) + + generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash) { + hash.sort_by { |k| k.to_s.encode(Encoding::UTF_16) }.to_h + }) + JSON.generator = generator end diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index a9578b9bd..2715a03f6 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -54,7 +54,8 @@ def to_h strict: strict?, depth: depth, buffer_initial_length: buffer_initial_length, - sort_keys: sort_keys + sort_keys: sort_keys, + rfc8785: rfc8785?, } allow_duplicate_key = allow_duplicate_key? diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index fe5daafff..99e5e7ec6 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -112,6 +112,8 @@ def self.valid_utf8?(string) # while generating a JSON text from a Ruby data structure. class State singleton_class.attr_accessor :default_sort_keys_proc # :nodoc: + singleton_class.attr_accessor :rfc8785_number_formater_proc # :nodoc: + singleton_class.attr_accessor :rfc8785_sort_keys_proc # :nodoc: def self.generate(obj, opts = nil, io = nil) new(opts).generate(obj, io) @@ -168,6 +170,7 @@ def initialize(opts = nil) @max_nesting = 100 @sort_keys = false @allow_duplicate_key = false + @rfc8785 = false _configure(**opts) if opts end @@ -209,6 +212,9 @@ def initialize(opts = nil) # desired order. attr_reader :sort_keys + # Controls whether RFC8785 (canonicalization) mode is enabled. Otherwise returns false. + attr_writer :rfc8785 + def sort_keys=(value) # :nodoc: type_error = false @sort_keys = case value @@ -288,6 +294,11 @@ def strict? @strict end + # Returns true, if RFC8785 (canonicalization) mode is enabled. Otherwise returns false. + def rfc8785? + @rfc8785 + end + # Configure this State instance with the Hash _opts_, and return # itself. def configure(options) @@ -308,7 +319,7 @@ def configure(options) array_nl: @array_nl, allow_nan: @allow_nan, as_json: @as_json, ascii_only: @ascii_only, sort_keys: @sort_keys, depth: @depth, buffer_initial_length: @buffer_initial_length, allow_duplicate_key: @allow_duplicate_key, script_safe: @script_safe, strict: @strict, - max_nesting: @max_nesting + max_nesting: @max_nesting, rfc8785: @rfc8785 ) if depth.negative? raise ArgumentError, "depth must be >= 0 (got #{depth})" @@ -333,6 +344,8 @@ def configure(options) @script_safe = script_safe @strict = strict @max_nesting = max_nesting || 0 + self.rfc8785 = rfc8785 + self end @@ -366,12 +379,9 @@ def generate(obj, anIO = nil) !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) and !@sort_keys result = generate_json(obj, ''.dup) else - if @sort_keys - obj = @sort_keys.call(obj) - end - result = obj.to_json(self) end + JSON::TruffleRuby::Generator.valid_utf8?(result) or raise GeneratorError.new( "source sequence #{result.inspect} is illegal/malformed utf-8", obj @@ -393,6 +403,11 @@ def generate(obj, anIO = nil) buf << '{' first = true key_type = nil + if @rfc8785 + obj = State.rfc8785_sort_keys_proc.call(obj) + elsif @sort_keys + obj = @sort_keys.call(obj) + end obj.each_pair do |k,v| if first key_type = k.class @@ -524,7 +539,15 @@ def json_transform(state) first = true key_type = nil indent = !state.object_nl.empty? - each { |key, value| + hash = self + + if state.rfc8785? + hash = State.rfc8785_sort_keys_proc.call(hash) + elsif state.sort_keys + hash = state.sort_keys.call(hash) + end + + hash.each { |key, value| if first key_type = key.class else @@ -658,6 +681,7 @@ module Float # Returns a JSON string representation for this Float number. def to_json(state = nil, *args) state = State.from_state(state) + if infinite? || nan? if state.allow_nan? to_s @@ -679,6 +703,8 @@ def to_json(state = nil, *args) else raise GeneratorError.new("#{self} not allowed in JSON", self) end + elsif state.rfc8785? + State.rfc8785_number_formater_proc.call(self) else to_s end diff --git a/test/json/fixtures/rfc8785/input/arrays.json b/test/json/fixtures/rfc8785/input/arrays.json new file mode 100644 index 000000000..9154bdf15 --- /dev/null +++ b/test/json/fixtures/rfc8785/input/arrays.json @@ -0,0 +1,8 @@ +[ + 56, + { + "d": true, + "10": null, + "1": [ ] + } +] \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/input/french.json b/test/json/fixtures/rfc8785/input/french.json new file mode 100644 index 000000000..d4ad2e4ed --- /dev/null +++ b/test/json/fixtures/rfc8785/input/french.json @@ -0,0 +1,6 @@ +{ + "peach": "This sorting order", + "péché": "is wrong according to French", + "pêche": "but canonicalization MUST", + "sin": "ignore locale" +} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/input/structures.json b/test/json/fixtures/rfc8785/input/structures.json new file mode 100644 index 000000000..eb71efb84 --- /dev/null +++ b/test/json/fixtures/rfc8785/input/structures.json @@ -0,0 +1,8 @@ +{ + "1": {"f": {"f": "hi","F": 5} ,"\n": 56.0}, + "10": { }, + "": "empty", + "a": { }, + "111": [ {"e": "yes","E": "no" } ], + "A": { } +} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/input/unicode.json b/test/json/fixtures/rfc8785/input/unicode.json new file mode 100644 index 000000000..9ec5cc7db --- /dev/null +++ b/test/json/fixtures/rfc8785/input/unicode.json @@ -0,0 +1,3 @@ +{ + "Unnormalized Unicode":"A\u030a" +} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/input/values.json b/test/json/fixtures/rfc8785/input/values.json new file mode 100644 index 000000000..f7712c2fb --- /dev/null +++ b/test/json/fixtures/rfc8785/input/values.json @@ -0,0 +1,5 @@ +{ + "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], + "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", + "literals": [null, true, false] +} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/input/wierd.json b/test/json/fixtures/rfc8785/input/wierd.json new file mode 100644 index 000000000..940bec4f9 --- /dev/null +++ b/test/json/fixtures/rfc8785/input/wierd.json @@ -0,0 +1,11 @@ +{ + "\u20ac": "Euro Sign", + "\r": "Carriage Return", + "\u000a": "Newline", + "1": "One", + "\u0080": "Control\u007f", + "\ud83d\ude02": "Smiley", + "\u00f6": "Latin Small Letter O With Diaeresis", + "\ufb33": "Hebrew Letter Dalet With Dagesh", + "": "Browser Challenge" +} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/arrays.json b/test/json/fixtures/rfc8785/output/arrays.json new file mode 100644 index 000000000..5efb93db7 --- /dev/null +++ b/test/json/fixtures/rfc8785/output/arrays.json @@ -0,0 +1 @@ +[56,{"1":[],"10":null,"d":true}] \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/french.json b/test/json/fixtures/rfc8785/output/french.json new file mode 100644 index 000000000..2e15cd1b2 --- /dev/null +++ b/test/json/fixtures/rfc8785/output/french.json @@ -0,0 +1 @@ +{"peach":"This sorting order","péché":"is wrong according to French","pêche":"but canonicalization MUST","sin":"ignore locale"} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/structures.json b/test/json/fixtures/rfc8785/output/structures.json new file mode 100644 index 000000000..dc21e2424 --- /dev/null +++ b/test/json/fixtures/rfc8785/output/structures.json @@ -0,0 +1 @@ +{"":"empty","1":{"\n":56,"f":{"F":5,"f":"hi"}},"10":{},"111":[{"E":"no","e":"yes"}],"A":{},"a":{}} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/unicode.json b/test/json/fixtures/rfc8785/output/unicode.json new file mode 100644 index 000000000..ee60fd121 --- /dev/null +++ b/test/json/fixtures/rfc8785/output/unicode.json @@ -0,0 +1 @@ +{"Unnormalized Unicode":"Å"} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/values.json b/test/json/fixtures/rfc8785/output/values.json new file mode 100644 index 000000000..29b720b6e --- /dev/null +++ b/test/json/fixtures/rfc8785/output/values.json @@ -0,0 +1 @@ +{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"} \ No newline at end of file diff --git a/test/json/fixtures/rfc8785/output/wierd.json b/test/json/fixtures/rfc8785/output/wierd.json new file mode 100644 index 000000000..62c83a333 --- /dev/null +++ b/test/json/fixtures/rfc8785/output/wierd.json @@ -0,0 +1 @@ +{"\n":"Newline","\r":"Carriage Return","1":"One","":"Browser Challenge","€":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😂":"Smiley","דּ":"Hebrew Letter Dalet With Dagesh"} \ No newline at end of file diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index a901840d9..0a20357f0 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -366,6 +366,7 @@ def test_state_defaults space: "", space_before: "", sort_keys: false, + rfc8785: false, }.sort_by { |n,| n.to_s }.to_h, state.to_h.sort_by { |n,| n.to_s }.to_h) state = JSON::State.new(allow_duplicate_key: true) @@ -385,6 +386,7 @@ def test_state_defaults space: "", space_before: "", sort_keys: false, + rfc8785: false, }.sort_by { |n,| n.to_s }, state.to_h.sort_by { |n,| n.to_s }) end @@ -1227,4 +1229,34 @@ def test_large_depth_raises end end + def test_rfc8785_numbers + assert_rfc8785 '-9007199254740992', -9007199254740992 + assert_rfc8785 '0', 0 + assert_rfc8785 '0.000001', 0.000001 + assert_rfc8785 '1e+21', 1e+21 + assert_rfc8785 '9.999999999999997e+22', 9.999999999999997e+22 + assert_rfc8785 '9.999999999999997e-7', 9.999999999999997e-7 + assert_rfc8785 '9007199254740992', 9007199254740992 + assert_rfc8785 '9007199254740994', 9007199254740994 + assert_rfc8785 '9007199254740996', 9007199254740996 + assert_rfc8785 '999999999999999700000', 999999999999999700000 + assert_rfc8785 '999999999999999900000', 999999999999999900000 + assert_rfc8785 '333333333.3333333', 333333333.33333329 + end + + fixtures_path = File.expand_path('../fixtures/rfc8785/', __FILE__) + Dir[File.join(fixtures_path, "input/*.json", __FILE__)].each do |input| + filename = File.basename(input) + name, _ = File.basename(filename, ".json") + expected = File.join(fixtures_path, 'output', filename) + define_method("test_rfc8785_#{name}") do + assert_rfc8785(File.read(output), JSON.load_file(input)) + end + end + + private + + def assert_rfc8785(expected, value) + assert_equal(expected, JSON.generate(value, rfc8785: true)) + end end From 42cc52a89caaa84319d52403d5477246846db163 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 5 Oct 2026 22:37:31 +0900 Subject: [PATCH 471/506] Open gzip in binary mode It is binary format, read in text mode stops at 0x1a (^Z). --- test/json/json_common_interface_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 0fbc2df7a..cf11c8f0e 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -135,7 +135,7 @@ def test_load stringio.rewind assert_equal @hash, JSON.load(stringio) - tempfile = Tempfile.open('@json.gzip') + tempfile = Tempfile.open('@json.gzip', mode: File::BINARY) gzw = Zlib::GzipWriter.new(tempfile) gzw.write(@json) gzw.close From b63b4f1506e9ead59e084cf4018126534db50d35 Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 5 Oct 2026 19:49:57 +0900 Subject: [PATCH 472/506] Honor as_json and reject disabled strict mode in JSON::Coder --- lib/json/common.rb | 9 +++++++-- lib/json/truffle_ruby/generator.rb | 17 +++++++++++++++-- test/json/json_coder_test.rb | 26 ++++++++++++++++++++++++++ test/json/json_generator_test.rb | 5 ++--- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index e1a8f6814..11f564083 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -827,7 +827,12 @@ class Coder # # puts MyApp::API_JSON_CODER.dump(Time.now.utc) # => "2025-01-21T08:41:44.286Z" # - def initialize(object_class: nil, array_class: nil, on_load: nil, **options, &as_json) + # The conversion callback can also be passed as +as_json:+. If both are given, the block takes precedence. + # Passing +false+ or +nil+ for +strict:+ raises ArgumentError. + # + def initialize(object_class: nil, array_class: nil, on_load: nil, strict: true, **options, &as_json) + raise ArgumentError, "JSON::Coder requires strict mode" unless strict + if object_class || array_class on_load = ParserOptions.on_load(on_load, object_class, array_class) end @@ -839,7 +844,7 @@ def initialize(object_class: nil, array_class: nil, on_load: nil, **options, &as @state = State.new( **generator_options, strict: true, - as_json: as_json, + as_json: as_json || generator_options[:as_json], ).freeze end diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 99e5e7ec6..de3bcf366 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -192,7 +192,16 @@ def initialize(opts = nil) attr_accessor :array_nl # This proc converts unsupported types into native JSON types. - attr_accessor :as_json + attr_reader :as_json + + def as_json=(value) + raise FrozenError, "can't modify frozen #{self.class}: #{inspect}" if frozen? + value = value.to_proc if !(Proc === value) && value.respond_to?(:to_proc) + unless Proc === value + raise TypeError, "as_json must be a Proc" + end + @as_json = value + end # This integer returns the maximum level of data structure nesting in # the generated JSON, max_nesting = 0 if no maximum is checked. @@ -335,7 +344,11 @@ def configure(options) @object_nl = object_nl || '' @array_nl = array_nl || '' @allow_nan = allow_nan || false - @as_json = as_json || false + if as_json + self.as_json = as_json + else + @as_json = false + end @ascii_only = ascii_only || false self.sort_keys = sort_keys @depth = depth diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 5ff314478..574222bf2 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -10,6 +10,32 @@ def test_json_coder_with_proc assert_equal %(["[Object object]"]), coder.dump([Object.new]) end + def test_json_coder_with_as_json_option + callback = ->(object, is_key) { 42 } + [callback, callback.method(:call)].each do |as_json| + coder = JSON::Coder.new(as_json: as_json) + assert_equal '[42]', coder.dump([Object.new]) + end + end + + def test_json_coder_block_overrides_as_json_option + coder = JSON::Coder.new(as_json: ->(object) { 1 }) { 2 } + assert_equal '[2]', coder.dump([Object.new]) + end + + def test_json_coder_as_json_invalid_type + assert_raise(TypeError) { JSON::Coder.new(as_json: 'x') } + end + + def test_json_coder_rejects_disabling_strict_mode + [false, nil].each do |strict| + error = assert_raise(ArgumentError) { JSON::Coder.new(strict: strict) } + assert_equal 'JSON::Coder requires strict mode', error.message + end + assert_raise(JSON::GeneratorError) { JSON::Coder.new.dump(Object.new) } + assert_raise(JSON::GeneratorError) { JSON::Coder.new(strict: true).dump(Object.new) } + end + def test_json_coder_with_proc_with_unsupported_value coder = JSON::Coder.new do |object, is_key| assert_equal false, is_key diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 0a20357f0..326780177 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1086,26 +1086,25 @@ def test_state_as_json_method object = Object.new state = JSON.state.new(strict: true) state.as_json = -> (o, is_key) { o.object_id }.method(:call) + assert_kind_of Proc, state.as_json assert_equal object.object_id.to_json, state.generate(object) end def test_json_generate_as_json_invalid_type - omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' [Object.new, Time.now].each do |as_json| assert_raise(TypeError) { JSON.generate(Object.new, strict: true, as_json: as_json) } end end def test_state_as_json_invalid_type - omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' state = JSON.state.new(strict: true) [Object.new, Time.now].each do |as_json| assert_raise(TypeError) { state.as_json = as_json } + assert_raise(TypeError) { state.configure(as_json: as_json) } end end def test_as_json_to_proc_returns_invalid_type - omit 'TruffleRuby does not convert as_json to Proc' if RUBY_ENGINE == 'truffleruby' as_json = Object.new def as_json.to_proc method(:to_proc) From 08e72297b252dc5b48eb5932d5b6299603872084 Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 5 Oct 2026 20:47:53 +0900 Subject: [PATCH 473/506] Exclude custom instance variables from State#to_h --- lib/json/ext/generator/state.rb | 5 ----- lib/json/truffle_ruby/generator.rb | 25 ++++++++++++++++++------- test/json/json_generator_test.rb | 10 ++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/lib/json/ext/generator/state.rb b/lib/json/ext/generator/state.rb index 2715a03f6..92c8fc480 100644 --- a/lib/json/ext/generator/state.rb +++ b/lib/json/ext/generator/state.rb @@ -63,11 +63,6 @@ def to_h result[:allow_duplicate_key] = allow_duplicate_key end - instance_variables.each do |iv| - iv = iv.to_s[1..-1] - result[iv.to_sym] = self[iv] - end - result end diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index de3bcf366..6a755659d 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -369,13 +369,24 @@ def allow_duplicate_key? # :nodoc: # Returns the configuration instance variables as a hash, that can be # passed to the configure method. def to_h - result = {} - instance_variables.each do |iv| - key = iv.to_s[1..-1] - result[key.to_sym] = instance_variable_get(iv) - end - - result + { + indent: @indent, + space: @space, + space_before: @space_before, + object_nl: @object_nl, + array_nl: @array_nl, + as_json: @as_json, + allow_nan: @allow_nan, + ascii_only: @ascii_only, + max_nesting: @max_nesting, + script_safe: @script_safe, + strict: @strict, + depth: @depth, + buffer_initial_length: @buffer_initial_length, + sort_keys: @sort_keys, + allow_duplicate_key: @allow_duplicate_key, + rfc8785: @rfc8785, + } end alias to_hash to_h diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 326780177..2fbe3f9f6 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -660,6 +660,16 @@ def test_json_state_to_h_roundtrip assert_equal state.to_h, JSON.state.new(state.to_h).to_h end + def test_json_state_to_h_ignores_instance_variables + state = JSON.state.new(indent: ' ') + expected = state.to_h + state.instance_variable_set(:@custom, 42) + state.instance_variable_set(:@enabled, false) + + assert_equal expected, state.to_h + assert_equal expected, state.to_hash + end + def test_json_generate assert_raise JSON::GeneratorError do generate(["\xea"]) From 8bab92d63be4e3345ffeb6d931be1c9120ff799e Mon Sep 17 00:00:00 2001 From: ydah Date: Tue, 6 Oct 2026 20:53:07 +0900 Subject: [PATCH 474/506] Reject non-ASCII BINARY strings during JSON generation --- ext/json/ext/generator/generator.c | 17 +--------------- java/src/json/ext/StringEncoder.java | 21 ++------------------ test/json/json_generator_test.rb | 29 +++++++++++++++------------- 3 files changed, 19 insertions(+), 48 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index fbef1980b..45f95fb94 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -75,7 +75,7 @@ static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *dat static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj); static void generate_json_fragment(FBuffer *buffer, struct generate_json_data *data, VALUE obj); -static int usascii_encindex, utf8_encindex, binary_encindex; +static int usascii_encindex, utf8_encindex; NORETURN(static void) raise_generator_error_str(VALUE invalid_object, VALUE str) { @@ -865,20 +865,6 @@ NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, } } - if (RB_ENCODING_GET_INLINED(str) == binary_encindex) { - VALUE utf8_string = rb_enc_associate_index(rb_str_dup(str), utf8_encindex); - switch (rb_enc_str_coderange(utf8_string)) { - case ENC_CODERANGE_7BIT: - return utf8_string; - case ENC_CODERANGE_VALID: - // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work. - // TODO: Raise in 3.0.0 - rb_warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0"); - return utf8_string; - break; - } - } - return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); } @@ -2140,7 +2126,6 @@ void Init_generator(void) usascii_encindex = rb_usascii_encindex(); utf8_encindex = rb_utf8_encindex(); - binary_encindex = rb_ascii8bit_encindex(); rb_require("json/ext/generator/state"); diff --git a/java/src/json/ext/StringEncoder.java b/java/src/json/ext/StringEncoder.java index e059fc993..347635cad 100644 --- a/java/src/json/ext/StringEncoder.java +++ b/java/src/json/ext/StringEncoder.java @@ -13,7 +13,6 @@ import java.nio.charset.StandardCharsets; import org.jcodings.Encoding; -import org.jcodings.specific.ASCIIEncoding; import org.jcodings.specific.USASCIIEncoding; import org.jcodings.specific.UTF8Encoding; import org.jruby.Ruby; @@ -229,28 +228,12 @@ static RubyString ensureValidEncoding(ThreadContext context, RubyString str) { return str; } - return tryWeirdEncodings(context, str, encoding); + return tryWeirdEncodings(context, str); } - private static RubyString tryWeirdEncodings(ThreadContext context, RubyString str, Encoding encoding) { + private static RubyString tryWeirdEncodings(ThreadContext context, RubyString str) { Ruby runtime = context.runtime; - RubyString utf8String; - - if (encoding == ASCIIEncoding.INSTANCE) { - utf8String = str.strDup(runtime); - utf8String.setEncoding(UTF8Encoding.INSTANCE); - switch (utf8String.getCodeRange()) { - case StringSupport.CR_7BIT: - return utf8String; - case StringSupport.CR_VALID: - // For historical reason, we silently reinterpret binary strings as UTF-8 if it would work. - // TODO: Raise in 3.0.0 - runtime.getWarnings().warn("JSON.generate: UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0"); - return utf8String; - } - } - try { str = (RubyString) str.encode(context, runtime.getEncodingService().convertEncodingToRubyEncoding(UTF8Encoding.INSTANCE)); } catch (RaiseException re) { diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 2fbe3f9f6..80f761e7b 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1049,22 +1049,25 @@ def to_s end end - if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java" - def test_valid_utf8_in_different_encoding - utf8_string = "€™" - wrong_encoding_string = utf8_string.b - # This behavior is historical. Not necessary desirable. We should deprecated it. - # The pure and java version of the gem already don't behave this way. - assert_warning(/UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0/) do - assert_equal utf8_string.to_json, wrong_encoding_string.to_json - end - - assert_warning(/UTF-8 string passed as BINARY, this will raise an encoding error in json 3.0/) do - assert_equal JSON.dump(utf8_string), JSON.dump(wrong_encoding_string) - end + def test_valid_utf8_in_binary_encoding + string = "€™".b.freeze + assert_raise(JSON::GeneratorError) { string.to_json } + assert_raise(JSON::GeneratorError) { JSON.dump(string) } + [string, [string], { string => 1 }, { value: string }].each do |object| + error = assert_raise(JSON::GeneratorError) { JSON.generate(object) } + assert_same string, error.invalid_object + assert_kind_of Encoding::UndefinedConversionError, error.cause end end + def test_ascii_in_binary_encoding + string = "ascii".b + assert_equal '"ascii"', string.to_json + assert_equal '"ascii"', JSON.dump(string) + assert_equal '["ascii"]', JSON.generate([string]) + assert_equal '{"ascii":1}', JSON.generate(string => 1) + end + def test_nonutf8_encoding assert_equal("\"5\u{b0}\"", "5\xb0".dup.force_encoding(Encoding::ISO_8859_1).to_json) end From 37a48ef6b8ab528c352b56f2351af7123164ed66 Mon Sep 17 00:00:00 2001 From: ydah Date: Mon, 5 Oct 2026 23:15:50 +0900 Subject: [PATCH 475/506] Report the attempted depth in generator nesting errors --- ext/json/ext/generator/generator.c | 7 +++---- java/src/json/ext/GeneratorState.java | 10 ++++------ test/json/json_generator_test.rb | 12 ++++++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 45f95fb94..b2a35bcb2 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1031,11 +1031,10 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) static inline long increase_depth(struct generate_json_data *data) { JSON_Generator_State *state = data->state; - long depth = ++data->depth; - if (RB_UNLIKELY(depth > state->max_nesting && state->max_nesting)) { - rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", --data->depth); + if (RB_UNLIKELY(data->depth >= state->max_nesting && state->max_nesting)) { + rb_raise(eNestingError, "nesting of %ld is too deep. Did you try to serialize objects with circular references?", data->depth + 1); } - return depth; + return ++data->depth; } static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 778500ef7..97809134c 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -649,9 +649,8 @@ public RubyHash to_h(ThreadContext context) { } public int increaseDepth(ThreadContext context) { - depth++; checkMaxNesting(context); - return depth; + return ++depth; } public int decreaseDepth() { @@ -659,13 +658,12 @@ public int decreaseDepth() { } /** - * Checks if the current depth is allowed as per this state's options. + * Checks if increasing the depth is allowed as per this state's options. * @param context The current context */ private void checkMaxNesting(ThreadContext context) { - if (maxNesting != 0 && depth > maxNesting) { - depth--; - throw Utils.newException(context, Utils.M_NESTING_ERROR, "nesting of " + depth + " is too deep. Did you try to serialize objects with circular references?"); + if (maxNesting != 0 && depth >= maxNesting) { + throw Utils.newException(context, Utils.M_NESTING_ERROR, "nesting of " + (depth + 1) + " is too deep. Did you try to serialize objects with circular references?"); } } } diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 80f761e7b..831443bf8 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -477,6 +477,18 @@ def test_depth_nesting_error assert_raise(JSON::NestingError) { JSON.pretty_generate(ary) } end + def test_nesting_error_reports_attempted_depth + [[[]], { a: {} }].each do |object| + [0, 1].each do |depth| + state = JSON.state.new(depth: depth, max_nesting: depth + 1) + error = assert_raise(JSON::NestingError) { state.generate(object) } + assert_match(/\Anesting of #{depth + 2} is too deep\./, error.message) + assert_equal depth, state.depth + assert_equal '[]', state.generate([]) + end + end + end + def test_depth_nesting_error_to_json ary = []; ary << ary s = JSON.state.new(depth: 1) From 193df5585202d40ea1e3800d650b114b117e49f2 Mon Sep 17 00:00:00 2001 From: ydah Date: Tue, 6 Oct 2026 21:44:31 +0900 Subject: [PATCH 476/506] Preserve float precision in RFC 8785 number formatting --- lib/json/ext.rb | 4 +-- test/json/json_generator_test.rb | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 19683f024..1c7965d08 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -66,8 +66,8 @@ def shareable_lambda(block) # :nodoc: if num < 0 num, sign = -num, '-' end - native_rep = "%.15E" % num - decimal, exponential = native_rep.split('E') + native_rep = num.to_s + decimal, exponential = native_rep.split('e') exp_val = exponential.to_i exponential = exp_val > 0 ? ('+' + exp_val.to_s) : exp_val.to_s diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 831443bf8..75aa39dee 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1266,6 +1266,48 @@ def test_rfc8785_numbers assert_rfc8785 '999999999999999700000', 999999999999999700000 assert_rfc8785 '999999999999999900000', 999999999999999900000 assert_rfc8785 '333333333.3333333', 333333333.33333329 + assert_rfc8785 '0.30000000000000004', 0.1 + 0.2 + end + + # https://www.rfc-editor.org/rfc/rfc8785.html#appendix-B + { + '0000000000000000' => '0', + '8000000000000000' => '0', + '0000000000000001' => '5e-324', + '8000000000000001' => '-5e-324', + '7fefffffffffffff' => '1.7976931348623157e+308', + 'ffefffffffffffff' => '-1.7976931348623157e+308', + '4340000000000000' => '9007199254740992', + 'c340000000000000' => '-9007199254740992', + '4430000000000000' => '295147905179352830000', + '7fffffffffffffff' => nil, + '7ff0000000000000' => nil, + '44b52d02c7e14af5' => '9.999999999999997e+22', + '44b52d02c7e14af6' => '1e+23', + '44b52d02c7e14af7' => '1.0000000000000001e+23', + '444b1ae4d6e2ef4e' => '999999999999999700000', + '444b1ae4d6e2ef4f' => '999999999999999900000', + '444b1ae4d6e2ef50' => '1e+21', + '3eb0c6f7a0b5ed8c' => '9.999999999999997e-7', + '3eb0c6f7a0b5ed8d' => '0.000001', + '41b3de4355555553' => '333333333.3333332', + '41b3de4355555554' => '333333333.33333325', + '41b3de4355555555' => '333333333.3333333', + '41b3de4355555556' => '333333333.3333334', + '41b3de4355555557' => '333333333.33333343', + 'becbf647612f3696' => '-0.0000033333333333333333', + '43143ff3c1cb0959' => '1424953923781206.2', + }.each do |bits, expected| + define_method("test_rfc8785_number_#{bits}") do + omit "Float#to_s is not always shortest on this engine" unless RUBY_ENGINE == "ruby" + number = [bits].pack('H*').unpack('G').first + if expected + assert_rfc8785 expected, number + assert_equal number, JSON.parse(JSON.generate(number, rfc8785: true)).to_f + else + assert_raise(JSON::GeneratorError) { JSON.generate(number, rfc8785: true) } + end + end end fixtures_path = File.expand_path('../fixtures/rfc8785/', __FILE__) From 808577ae1d6b32712d7cbe4377caf29bba70e19f Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 7 Oct 2026 22:50:43 +0900 Subject: [PATCH 477/506] Sort RFC 8785 object properties by their keys --- lib/json/ext.rb | 2 +- test/json/json_generator_test.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 1c7965d08..9c7e090bb 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -95,7 +95,7 @@ def shareable_lambda(block) # :nodoc: }) generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash) { - hash.sort_by { |k| k.to_s.encode(Encoding::UTF_16) }.to_h + hash.sort_by { |k,| k.to_s.encode(Encoding::UTF_16) }.to_h }) JSON.generator = generator diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 75aa39dee..02cff38cd 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1253,6 +1253,36 @@ def test_large_depth_raises end end + def test_rfc8785_key_order + assert_rfc8785 '{"a":1,"a b":2}', { 'a' => 1, 'a b' => 2 } + assert_rfc8785 '{"a":2,"a!":1}', { 'a!' => 1, 'a' => 2 } + assert_rfc8785 '{"\n":1,"A":2}', { "\n" => 1, 'A' => 2 } + assert_rfc8785 '[{"a":1,"a b":2}]', [{ 'a b' => 2, 'a' => 1 }] + end + + def test_rfc8785_mixed_key_order + assert_rfc8785 '{"10":2,"9":1}', { '9' => 1, 10 => 2 } + assert_rfc8785 '{"a":1,"b":2}', { a: 1, 'b' => 2 } + end + + def test_rfc8785_key_order_with_default_external_encoding + # https://www.rfc-editor.org/rfc/rfc8785.html#section-3.2.3 + hash = { "\u20ac" => 1, "\r" => 2, "\ufb33" => 3, '1' => 4, + "\u{1f600}" => 5, "\u0080" => 6, "\u00f6" => 7 } + expected = "{\"\\r\":2,\"1\":4,\"\u0080\":6,\"\u00f6\":7,\"\u20ac\":1,\"\u{1f600}\":5,\"\ufb33\":3}" + verbose = $VERBOSE + previous_encoding = Encoding.default_external + $VERBOSE = nil + outputs = [Encoding::UTF_8, Encoding::US_ASCII].map do |encoding| + Encoding.default_external = encoding + JSON.generate(hash, rfc8785: true) + end + assert_equal [expected, expected], outputs + ensure + Encoding.default_external = previous_encoding + $VERBOSE = verbose + end + def test_rfc8785_numbers assert_rfc8785 '-9007199254740992', -9007199254740992 assert_rfc8785 '0', 0 From 3e3807c82a03d09d2db9a2a1425d6d1635f55ae8 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 7 Oct 2026 23:05:27 +0900 Subject: [PATCH 478/506] Fix RFC 8785 number formatter spelling --- ext/json/ext/generator/generator.c | 18 +++++++++--------- java/src/json/ext/Generator.java | 2 +- java/src/json/ext/GeneratorState.java | 8 ++++---- lib/json/ext.rb | 2 +- lib/json/truffle_ruby/generator.rb | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index b2a35bcb2..bdc34b71f 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -32,7 +32,7 @@ typedef struct JSON_Generator_StateStruct { } JSON_Generator_State; static VALUE mJSON, cState, cFragment, eGeneratorError, eNestingError, Encoding_UTF_8, default_sort_keys_proc, - rfc8785_number_formater_proc, rfc8785_sort_keys_proc; + rfc8785_number_formatter_proc, rfc8785_sort_keys_proc; static ID i_to_s, i_to_json, i_new, i_encode; static VALUE sym_indent, sym_space, sym_space_before, sym_object_nl, sym_array_nl, sym_max_nesting, sym_allow_nan, sym_allow_duplicate_key, @@ -1171,7 +1171,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data } if (RB_UNLIKELY(data->state->rfc8785)) { - VALUE str = rb_proc_call_with_block(rfc8785_number_formater_proc, 1, &obj, Qnil); + VALUE str = rb_proc_call_with_block(rfc8785_number_formatter_proc, 1, &obj, Qnil); Check_Type(str, T_STRING); fbuffer_append_str(buffer, str); return; @@ -1754,15 +1754,15 @@ static VALUE cState_set_default_sort_keys_proc(VALUE self, VALUE proc) return default_sort_keys_proc = proc; } -static VALUE cState_set_rfc8785_number_formater_proc(VALUE self, VALUE proc) +static VALUE cState_set_rfc8785_number_formatter_proc(VALUE self, VALUE proc) { if (!rb_obj_is_proc(proc)) { - rb_raise(rb_eTypeError, "rfc8785_number_formater_proc must be a Proc"); + rb_raise(rb_eTypeError, "rfc8785_number_formatter_proc must be a Proc"); } - if (rfc8785_number_formater_proc) { - rb_raise(rb_eArgError, "rfc8785_number_formater_proc can only be set once"); + if (rfc8785_number_formatter_proc) { + rb_raise(rb_eArgError, "rfc8785_number_formatter_proc can only be set once"); } - return rfc8785_number_formater_proc = proc; + return rfc8785_number_formatter_proc = proc; } static VALUE cState_set_rfc8785_sort_keys_proc(VALUE self, VALUE proc) @@ -2036,7 +2036,7 @@ void Init_generator(void) VALUE mGenerator = rb_define_module_under(mExt, "Generator"); rb_global_variable(&default_sort_keys_proc); - rb_global_variable(&rfc8785_number_formater_proc); + rb_global_variable(&rfc8785_number_formatter_proc); rb_global_variable(&rfc8785_sort_keys_proc); rb_global_variable(&eGeneratorError); @@ -2049,7 +2049,7 @@ void Init_generator(void) rb_define_alloc_func(cState, cState_s_allocate); rb_define_singleton_method(cState, "from_state", cState_from_state_s, 1); rb_define_singleton_method(cState, "default_sort_keys_proc=", cState_set_default_sort_keys_proc, 1); - rb_define_singleton_method(cState, "rfc8785_number_formater_proc=", cState_set_rfc8785_number_formater_proc, 1); + rb_define_singleton_method(cState, "rfc8785_number_formatter_proc=", cState_set_rfc8785_number_formatter_proc, 1); rb_define_singleton_method(cState, "rfc8785_sort_keys_proc=", cState_set_rfc8785_sort_keys_proc, 1); rb_define_method(cState, "initialize", cState_initialize, -1); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index a38c31d83..0f6bc8248 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -421,7 +421,7 @@ static void generateFloat(ThreadContext context, Session session, RubyFloat obje } if (state.rfc8785()) { - RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormaterProc; + RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormatterProc; RubyString numberString = (RubyString)Helpers.invoke(context, numberProc, "call", object); buffer.write(numberString.toString().getBytes(UTF_8)); } else { diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index 97809134c..d702ae8a0 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -37,7 +37,7 @@ public class GeneratorState extends RubyObject { private boolean allowDuplicateKey = false; private static IRubyObject defaultSortKeyProc; - public static IRubyObject rfc8785NumberFormaterProc; + public static IRubyObject rfc8785NumberFormatterProc; public static IRubyObject rfc8785SortKeysProc; /** @@ -176,9 +176,9 @@ public static IRubyObject setDefaultSortKeyProc(IRubyObject klass, IRubyObject p return proc; } - @JRubyMethod(meta=true, name="rfc8785_number_formater_proc=") - public static IRubyObject setRfc8785NumberFormaterProc(IRubyObject klass, IRubyObject proc) { - rfc8785NumberFormaterProc = proc; + @JRubyMethod(meta=true, name="rfc8785_number_formatter_proc=") + public static IRubyObject setRfc8785NumberFormatterProc(IRubyObject klass, IRubyObject proc) { + rfc8785NumberFormatterProc = proc; return proc; } diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 9c7e090bb..4443d6687 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -59,7 +59,7 @@ def shareable_lambda(block) # :nodoc: }) # Directly lifted from Gregg Kellogg's json-canonicalization - generator::State.rfc8785_number_formater_proc = shareable_lambda(->(num) { + generator::State.rfc8785_number_formatter_proc = shareable_lambda(->(num) { if num.zero? "0" else diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 6a755659d..852ad4e55 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -112,7 +112,7 @@ def self.valid_utf8?(string) # while generating a JSON text from a Ruby data structure. class State singleton_class.attr_accessor :default_sort_keys_proc # :nodoc: - singleton_class.attr_accessor :rfc8785_number_formater_proc # :nodoc: + singleton_class.attr_accessor :rfc8785_number_formatter_proc # :nodoc: singleton_class.attr_accessor :rfc8785_sort_keys_proc # :nodoc: def self.generate(obj, opts = nil, io = nil) @@ -728,7 +728,7 @@ def to_json(state = nil, *args) raise GeneratorError.new("#{self} not allowed in JSON", self) end elsif state.rfc8785? - State.rfc8785_number_formater_proc.call(self) + State.rfc8785_number_formatter_proc.call(self) else to_s end From 3d9d6dd32d0185a39583bac7e12e3e9b07dd4544 Mon Sep 17 00:00:00 2001 From: ydah Date: Thu, 8 Oct 2026 07:34:03 +0900 Subject: [PATCH 479/506] Canonicalize integers as doubles in RFC 8785 mode --- ext/json/ext/generator/generator.c | 19 +++++++++++--- java/src/json/ext/Generator.java | 30 ++++++++++++++------- lib/json.rb | 2 ++ lib/json/ext.rb | 6 +++++ lib/json/truffle_ruby/generator.rb | 10 +++++-- test/json/json_generator_test.rb | 42 ++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 14 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index bdc34b71f..32f38a58d 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1135,13 +1135,28 @@ static void generate_json_true(FBuffer *buffer, struct generate_json_data *data, fbuffer_append(buffer, "true", 4); } +static void generate_json_rfc8785_number(FBuffer *buffer, VALUE obj) +{ + VALUE str = rb_proc_call_with_block(rfc8785_number_formatter_proc, 1, &obj, Qnil); + Check_Type(str, T_STRING); + fbuffer_append_str(buffer, str); +} + static void generate_json_fixnum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { + if (RB_UNLIKELY(data->state->rfc8785)) { + generate_json_rfc8785_number(buffer, obj); + return; + } fbuffer_append_long(buffer, FIX2LONG(obj)); } static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { + if (RB_UNLIKELY(data->state->rfc8785)) { + generate_json_rfc8785_number(buffer, obj); + return; + } VALUE tmp = rb_funcall(obj, i_to_s, 0); fbuffer_append_str(buffer, StringValue(tmp)); } @@ -1171,9 +1186,7 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data } if (RB_UNLIKELY(data->state->rfc8785)) { - VALUE str = rb_proc_call_with_block(rfc8785_number_formatter_proc, 1, &obj, Qnil); - Check_Type(str, T_STRING); - fbuffer_append_str(buffer, str); + generate_json_rfc8785_number(buffer, obj); return; } diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 0f6bc8248..f191993d7 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -174,8 +174,8 @@ private static void generateFor(ThreadContext context, S case TRUE : buffer.write(TRUE_STRING); return; case FALSE : buffer.write(FALSE_STRING); return; case FLOAT : generateFloat(context, session, (RubyFloat) object, buffer); return; - case FIXNUM : generateFixnum(session, (RubyFixnum) object, buffer); return; - case BIGNUM : generateBignum((RubyBignum) object, buffer); return; + case FIXNUM : generateFixnum(context, session, (RubyFixnum) object, buffer); return; + case BIGNUM : generateBignum(context, session, (RubyBignum) object, buffer); return; case SYMBOL : generateSymbol(context, session, (RubySymbol) object, buffer); return; case STRING : if (Helpers.metaclass(object) != context.runtime.getString()) break; @@ -345,11 +345,15 @@ void generate(ThreadContext context, Session session, T object, OutputStream buf private static class BignumHandler extends Handler { @Override void generate(ThreadContext context, Session session, RubyBignum object, OutputStream buffer) throws IOException { - generateBignum(object, buffer); + generateBignum(context, session, object, buffer); } } - private static void generateBignum(RubyBignum object, OutputStream buffer) throws IOException { + private static void generateBignum(ThreadContext context, Session session, RubyBignum object, OutputStream buffer) throws IOException { + if (session.getState(context).rfc8785()) { + generateRfc8785Number(context, object, buffer); + return; + } BigInteger bigInt = object.getValue(); buffer.write(bigInt.toString().getBytes(UTF_8)); } @@ -357,11 +361,15 @@ private static void generateBignum(RubyBignum object, OutputStream buffer) throw private static class FixnumHandler extends Handler { @Override void generate(ThreadContext context, Session session, RubyFixnum object, OutputStream buffer) throws IOException { - generateFixnum(session, object, buffer); + generateFixnum(context, session, object, buffer); } } - static void generateFixnum(Session session, RubyFixnum object, OutputStream buffer) throws IOException { + static void generateFixnum(ThreadContext context, Session session, RubyFixnum object, OutputStream buffer) throws IOException { + if (session.getState(context).rfc8785()) { + generateRfc8785Number(context, object, buffer); + return; + } long i = object.getLongValue(); if (i == 0) { buffer.write('0'); @@ -421,14 +429,18 @@ static void generateFloat(ThreadContext context, Session session, RubyFloat obje } if (state.rfc8785()) { - RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormatterProc; - RubyString numberString = (RubyString)Helpers.invoke(context, numberProc, "call", object); - buffer.write(numberString.toString().getBytes(UTF_8)); + generateRfc8785Number(context, object, buffer); } else { buffer.write(Double.toString(value).getBytes(UTF_8)); } } + private static void generateRfc8785Number(ThreadContext context, IRubyObject object, OutputStream buffer) throws IOException { + RubyProc numberProc = (RubyProc)GeneratorState.rfc8785NumberFormatterProc; + RubyString numberString = (RubyString)Helpers.invoke(context, numberProc, "call", object); + buffer.write(numberString.toString().getBytes(UTF_8)); + } + private static final byte[] EMPTY_ARRAY_BYTES = "[]".getBytes(); private static class ArrayHandler extends Handler> { @Override diff --git a/lib/json.rb b/lib/json.rb index 576d86de9..72ca9f1de 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -416,6 +416,8 @@ # order, allowing for arbitrary sort orders. # - Option +rfc8785+ (boolean) controls whether the generated JSON will be canonicalized # as defined in RFC8785. +# Integers are converted to Float before formatting and may be rounded. +# Integers outside the finite Float range raise JSON::GeneratorError. # # In this example, +obj+ is used first to generate the shortest # \JSON data (no whitespace), then again with all formatting options diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 4443d6687..7382867a2 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -60,6 +60,12 @@ def shareable_lambda(block) # :nodoc: # Directly lifted from Gregg Kellogg's json-canonicalization generator::State.rfc8785_number_formatter_proc = shareable_lambda(->(num) { + if Integer === num + if num > Float::MAX || num < -Float::MAX + raise GeneratorError.new("Integer out of range for RFC 8785", num) + end + num = num.to_f + end if num.zero? "0" else diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 852ad4e55..748a18c1d 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -473,7 +473,7 @@ def generate(obj, anIO = nil) buf << obj.to_json(self) end when Integer - buf << obj.to_s + buf << (@rfc8785 ? State.rfc8785_number_formatter_proc.call(obj) : obj.to_s) when Symbol if @strict fast_serialize_string(obj.name, buf) @@ -698,7 +698,13 @@ def json_transform(state) module Integer # Returns a JSON string representation for this Integer number. - def to_json(*) to_s end + def to_json(state = nil, *) + if state && State.from_state(state).rfc8785? + State.rfc8785_number_formatter_proc.call(self) + else + to_s + end + end end module Float diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 02cff38cd..ee781194d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1283,6 +1283,48 @@ def test_rfc8785_key_order_with_default_external_encoding $VERBOSE = verbose end + { + 0 => '0', + 1 => '1', + 2**53 - 1 => '9007199254740991', + 2**53 => '9007199254740992', + 2**53 + 1 => '9007199254740992', + 2**53 + 3 => '9007199254740996', + 2**63 - 1 => '9223372036854776000', + 2**68 => '295147905179352830000', + 10**20 => '100000000000000000000', + 10**21 => '1e+21', + Float::MAX.to_i => '1.7976931348623157e+308', + }.each do |integer, expected| + define_method("test_rfc8785_integer_#{integer}") do + [1, -1].each do |sign| + number = sign * integer + json = sign < 0 && integer != 0 ? "-#{expected}" : expected + assert_rfc8785 json, number + assert_rfc8785 %({"n":[#{json}]}), { 'n' => [number] } + assert_equal json, number.to_json(rfc8785: true) + assert_equal json, JSON.generate(number, rfc8785: true, max_nesting: false) + assert_equal json, JSON::Coder.new(rfc8785: true).dump(number) + io = StringIO.new + JSON.dump(number, io, rfc8785: true) + assert_equal json, io.string + assert_equal number.to_s, JSON.generate(number) + end + end + end + + def test_rfc8785_rejects_out_of_range_integers + [Float::MAX.to_i + 1, 10**400].each do |integer| + [integer, -integer].each do |number| + error = assert_raise(JSON::GeneratorError) { JSON.generate(number, rfc8785: true) } + assert_same number, error.invalid_object + assert_equal 'Integer out of range for RFC 8785', error.message + assert_raise(JSON::GeneratorError) { JSON.dump([number], rfc8785: true) } + assert_equal number.to_s, JSON.generate(number) + end + end + end + def test_rfc8785_numbers assert_rfc8785 '-9007199254740992', -9007199254740992 assert_rfc8785 '0', 0 From 49e8df0e1639e7f825e9cd2cf665ccfc6d97d279 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 7 Oct 2026 23:39:35 +0900 Subject: [PATCH 480/506] Reject incompatible RFC 8785 generation options --- ext/json/ext/generator/generator.c | 19 +++++++++ java/src/json/ext/Generator.java | 2 + java/src/json/ext/GeneratorState.java | 17 ++++++++ lib/json.rb | 4 ++ lib/json/common.rb | 2 +- lib/json/truffle_ruby/generator.rb | 41 +++++++++++++++----- test/json/json_generator_test.rb | 56 +++++++++++++++++++++++++++ 7 files changed, 130 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 32f38a58d..9af35a086 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1307,10 +1307,28 @@ static void generate_json_no_fallback(FBuffer *buffer, struct generate_json_data generate_json_general(buffer, data, obj, false); } +static void validate_rfc8785(JSON_Generator_State *state) +{ + if (!state->rfc8785) return; + + const char *option = state->indent ? "indent" : + state->space ? "space" : + state->space_before ? "space_before" : + state->object_nl ? "object_nl" : + state->array_nl ? "array_nl" : + state->ascii_only ? "ascii_only" : + state->script_safe ? "script_safe" : + state->allow_nan ? "allow_nan" : NULL; + if (option) { + rb_raise(rb_eArgError, "%s cannot be used with rfc8785", option); + } +} + static VALUE generate_json_try(VALUE d) { struct generate_json_data *data = (struct generate_json_data *)d; + validate_rfc8785(data->state); data->func(data->buffer, data, data->obj); return fbuffer_finalize(data->buffer); @@ -1990,6 +2008,7 @@ static void configure_state(JSON_Generator_State *state, VALUE vstate, VALUE con rb_hash_foreach(config, configure_state_i, (VALUE)&data); raise_argument_error_on_unknown_keywords(data.unknown_keywords); + validate_rfc8785(state); } static VALUE cState_configure(VALUE self, VALUE opts) diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index f191993d7..d653e582c 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -56,6 +56,7 @@ static RubyString generateJson(ThreadContext context, T static RubyString generateJson(ThreadContext context, T object, Handler handler, IRubyObject arg0) { Session session = new Session(arg0); GeneratorState state = session.getState(context); + state.validateRfc8785(context); int depth = state.depth; try { return handler.generateNew(context, session, object); @@ -87,6 +88,7 @@ static RubyString generateJson(ThreadContext context, T GeneratorState config, IRubyObject io) { Session session = new Session(config); GeneratorState state = session.getState(context); + state.validateRfc8785(context); int depth = state.depth; try { diff --git a/java/src/json/ext/GeneratorState.java b/java/src/json/ext/GeneratorState.java index d702ae8a0..a0f3dfc21 100644 --- a/java/src/json/ext/GeneratorState.java +++ b/java/src/json/ext/GeneratorState.java @@ -525,6 +525,22 @@ public boolean rfc8785() { return this.rfc8785; } + public void validateRfc8785(ThreadContext context) { + if (!rfc8785) return; + + String option = !indent.isEmpty() ? "indent" : + !space.isEmpty() ? "space" : + !spaceBefore.isEmpty() ? "space_before" : + !objectNl.isEmpty() ? "object_nl" : + !arrayNl.isEmpty() ? "array_nl" : + asciiOnly ? "ascii_only" : + scriptSafe ? "script_safe" : + allowNaN ? "allow_nan" : null; + if (option != null) { + throw context.runtime.newArgumentError(option + " cannot be used with rfc8785"); + } + } + public int getDepth() { return depth; } @@ -603,6 +619,7 @@ public IRubyObject _configure(ThreadContext context, IRubyObject vOpts) { rfc8785 = opts.getBool("rfc8785", rfc8785); opts.ensureEmpty(); + validateRfc8785(context); return this; } diff --git a/lib/json.rb b/lib/json.rb index 72ca9f1de..1969b026d 100644 --- a/lib/json.rb +++ b/lib/json.rb @@ -418,6 +418,10 @@ # as defined in RFC8785. # Integers are converted to Float before formatting and may be rounded. # Integers outside the finite Float range raise JSON::GeneratorError. +# Nonempty +indent+, +space+, +space_before+, +object_nl+, or +array_nl+, +# and enabled +ascii_only+, +script_safe+, or +allow_nan+ raise ArgumentError. +# JSON.dump defaults +allow_nan+ to +false+ when +rfc8785+ is enabled. +# JSON::Fragment contents are inserted as is; the caller must ensure they are canonical. # # In this example, +obj+ is used first to generate the shortest # \JSON data (no whitespace), then again with all formatting options diff --git a/lib/json/common.rb b/lib/json/common.rb index 11f564083..01a4be18c 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -763,7 +763,7 @@ def dump(obj, anIO = nil, _deprecated_limit = nil, kwargs = nil) end opts = { - allow_nan: true, + allow_nan: !(kwargs && kwargs[:rfc8785]), } opts[:max_nesting] = _deprecated_limit if _deprecated_limit opts.merge!(kwargs) if kwargs diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 748a18c1d..895b09e75 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -127,7 +127,7 @@ def self.from_state(opts) if opts case when self === opts - return opts + return opts.validate_rfc8785 when opts.respond_to?(:to_hash) return new(opts.to_hash) when opts.respond_to?(:to_h) @@ -308,6 +308,22 @@ def rfc8785? @rfc8785 end + def validate_rfc8785 # :nodoc: + if @rfc8785 + option = if !@indent.empty? then :indent + elsif !@space.empty? then :space + elsif !@space_before.empty? then :space_before + elsif !@object_nl.empty? then :object_nl + elsif !@array_nl.empty? then :array_nl + elsif @ascii_only then :ascii_only + elsif @script_safe then :script_safe + elsif @allow_nan then :allow_nan + end + raise ArgumentError, "#{option} cannot be used with rfc8785" if option + end + self + end + # Configure this State instance with the Hash _opts_, and return # itself. def configure(options) @@ -359,7 +375,7 @@ def configure(options) @max_nesting = max_nesting || 0 self.rfc8785 = rfc8785 - self + validate_rfc8785 end def allow_duplicate_key? # :nodoc: @@ -399,6 +415,7 @@ def generate(obj, anIO = nil) return dup.generate(obj, anIO) if frozen? depth = @depth + validate_rfc8785 if @indent.empty? and @space.empty? and @space_before.empty? and @object_nl.empty? and @array_nl.empty? and !@ascii_only and !@script_safe and @max_nesting == 0 and (!@strict || Symbol === obj) and !@sort_keys result = generate_json(obj, ''.dup) @@ -542,10 +559,12 @@ module Hash def to_json(state = nil, *) state = State.from_state(state) depth = state.depth - state.check_max_nesting - json_transform(state) - ensure - state.depth = depth + begin + state.check_max_nesting + json_transform(state) + ensure + state.depth = depth + end end private @@ -642,10 +661,12 @@ module Array def to_json(state = nil, *) state = State.from_state(state) depth = state.depth - state.check_max_nesting - json_transform(state) - ensure - state.depth = depth + begin + state.check_max_nesting + json_transform(state) + ensure + state.depth = depth + end end private diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index ee781194d..a5f697ded 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1325,6 +1325,62 @@ def test_rfc8785_rejects_out_of_range_integers end end + { + indent: ' ', space: ' ', space_before: ' ', object_nl: "\n", array_nl: "\n", + ascii_only: true, script_safe: true, allow_nan: true, + }.each do |option, value| + define_method("test_rfc8785_rejects_#{option}") do + options = { rfc8785: true, option => value } + error = assert_raise(ArgumentError) { JSON.generate({ 'a' => [1] }, options) } + assert_equal "#{option} cannot be used with rfc8785", error.message + assert_raise(ArgumentError) { JSON.generate(nil, options.to_a.reverse.to_h) } + assert_raise(ArgumentError) { [1].to_json(options) } + assert_raise(ArgumentError) { { 'a' => 1 }.to_json(options) } + assert_raise(ArgumentError) { JSON::Coder.new(**options).dump([1]) } + + state = JSON::State.new(option => value) + state.rfc8785 = true + io = StringIO.new + assert_raise(ArgumentError) { state.generate([1], io) } + assert_equal '', io.string + assert_raise(ArgumentError) { [1].to_json(state) } + assert_equal 0, state.depth + assert_raise(ArgumentError) { state.freeze.generate([1]) } + end + end + + def test_rfc8785_accepts_disabled_options + options = { rfc8785: true, indent: '', space: '', space_before: '', + object_nl: '', array_nl: '', ascii_only: false, + script_safe: false, allow_nan: false } + assert_equal '["é/",1]', JSON.generate(['é/', 1], options) + end + + def test_rfc8785_rejects_pretty_generation + assert_raise(ArgumentError) { JSON.pretty_generate({ 'a' => 1 }, rfc8785: true) } + end + + def test_rfc8785_dump_disables_allow_nan_by_default + assert_equal '[1]', JSON.dump([1], rfc8785: true) + [Float::NAN, Float::INFINITY, -Float::INFINITY].each do |number| + assert_raise(JSON::GeneratorError) { JSON.dump([number], rfc8785: true) } + end + assert_raise(ArgumentError) { JSON.dump([1], rfc8785: true, allow_nan: true) } + end + + def test_rfc8785_fragments + fragment = JSON::Fragment.new('[1,2]') + assert_rfc8785 '[1,2]', fragment + assert_rfc8785 '[[1,2]]', [fragment] + assert_rfc8785 '{"a":[1,2]}', { 'a' => fragment } + assert_equal '[[1,2]]', JSON.generate([fragment], rfc8785: true, max_nesting: false) + assert_equal '[1,2]', JSON::Coder.new(rfc8785: true).dump(fragment) + assert_equal '[1,2]', fragment.to_json(rfc8785: true) + + # Fragment contents are the caller's responsibility and are inserted as is. + assert_rfc8785 '[1, 2]', JSON::Fragment.new('[1, 2]') + end + def test_rfc8785_numbers assert_rfc8785 '-9007199254740992', -9007199254740992 assert_rfc8785 '0', 0 From cdcfdb80664613c832d35da52da8d1df42832544 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 8 Oct 2026 12:52:57 +0200 Subject: [PATCH 481/506] Better defend against `to_s` or `encode` not returning a T_STRING --- ext/json/ext/generator/generator.c | 25 ++++++++++++++--------- test/json/json_generator_test.rb | 32 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9af35a086..380622235 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -770,6 +770,12 @@ static void vstate_spill(struct generate_json_data *data) RB_OBJ_WRITTEN(vstate, Qundef, state->sort_keys); } +static inline VALUE json_to_s(VALUE obj) +{ + VALUE tmp = rb_funcall(obj, i_to_s, 0); + return StringValue(tmp); +} + static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj) { if (RB_UNLIKELY(!data->vstate)) { @@ -865,7 +871,12 @@ NOINLINE(static) VALUE convert_invalid_encoding(struct generate_json_data *data, } } - return rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); + str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str); + Check_Type(str, T_STRING); + if (!valid_json_string_p(str)) { + raise_generator_error(str, "source sequence is illegal/malformed utf-8"); + } + return str; } ALWAYS_INLINE(static) VALUE ensure_valid_encoding(struct generate_json_data *data, VALUE str, bool as_json_called, bool is_key) @@ -1114,9 +1125,7 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d Check_Type(tmp, T_STRING); fbuffer_append_str(buffer, tmp); } else { - tmp = rb_funcall(obj, i_to_s, 0); - Check_Type(tmp, T_STRING); - generate_json_string(buffer, data, tmp); + generate_json_string(buffer, data, json_to_s(obj)); } } @@ -1157,8 +1166,7 @@ static void generate_json_bignum(FBuffer *buffer, struct generate_json_data *dat generate_json_rfc8785_number(buffer, obj); return; } - VALUE tmp = rb_funcall(obj, i_to_s, 0); - fbuffer_append_str(buffer, StringValue(tmp)); + fbuffer_append_str(buffer, json_to_s(obj)); } static void generate_json_float(FBuffer *buffer, struct generate_json_data *data, VALUE obj) @@ -1177,11 +1185,10 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data return; } } - raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", rb_funcall(obj, i_to_s, 0)); + raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", json_to_s(obj)); } - VALUE tmp = rb_funcall(obj, i_to_s, 0); - fbuffer_append_str(buffer, tmp); + fbuffer_append_str(buffer, json_to_s(obj)); return; } diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index a5f697ded..fb38b1ab8 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1223,6 +1223,38 @@ def test_frozen end end + def test_adversary_encode + fake_string = Class.new(String) do + def to_s + self + end + + def encode(*) + 42 + end + end + + assert_raise TypeError, JSON::GeneratorError do + JSON.generate({fake_string.new("\xff".b) => 1}) + end + + no_to_json = Class.new do + undef_method :to_json + + def initialize(str) + @str = str + end + + def to_s + @str + end + end + + assert_raise TypeError, JSON::GeneratorError do + JSON.generate([no_to_json.new(fake_string.new("\xff".b))]) + end + end + # The case when the State is frozen is tested in JSONCoderTest#test_nesting_recovery def test_nesting_recovery state = JSON::State.new From 88b358d8b2d28e0fba14db30f7942c0cfbf14be7 Mon Sep 17 00:00:00 2001 From: ydah Date: Thu, 8 Oct 2026 19:55:33 +0900 Subject: [PATCH 482/506] Sort RFC 8785 keys after as_json conversion --- ext/json/ext/generator/generator.c | 3 +- java/src/json/ext/Generator.java | 10 +++---- lib/json/ext.rb | 28 ++++++++++++++++- lib/json/truffle_ruby/generator.rb | 6 ++-- test/json/json_coder_test.rb | 48 ++++++++++++++++++++++++++++++ test/json/json_generator_test.rb | 19 ++++++++++++ 6 files changed, 103 insertions(+), 11 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 380622235..9413bf517 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1051,7 +1051,8 @@ static inline long increase_depth(struct generate_json_data *data) static void generate_json_object(FBuffer *buffer, struct generate_json_data *data, VALUE obj) { if (RB_UNLIKELY(data->state->rfc8785)) { - obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 1, &obj, Qnil); + VALUE args[2] = {obj, data->state->strict ? data->state->as_json : Qnil}; + obj = rb_proc_call_with_block(rfc8785_sort_keys_proc, 2, args, Qnil); Check_Type(obj, T_HASH); } else if (RB_UNLIKELY(data->state->sort_keys)) { obj = rb_proc_call_with_block(data->state->sort_keys, 1, &obj, Qnil); diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index d653e582c..3b05812f5 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -588,13 +588,11 @@ static void generateHash(ThreadContext context, Session session, RubyHash object return; } - RubyProc sortKeysProc = state.getSortKeysProc(); if (state.rfc8785()) { - sortKeysProc = (RubyProc)GeneratorState.rfc8785SortKeysProc; - } - - if (sortKeysProc != null) { - object = (RubyHash)Helpers.invoke(context, sortKeysProc, "call", object); + IRubyObject asJSON = state.strict() ? state.as_json_get(context) : context.nil; + object = (RubyHash)Helpers.invoke(context, GeneratorState.rfc8785SortKeysProc, "call", object, asJSON); + } else if (state.getSortKeysProc() != null) { + object = (RubyHash)Helpers.invoke(context, state.getSortKeysProc(), "call", object); } final ByteList objectNl = state.getObjectNl(); diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 7382867a2..7d7b09b27 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -100,7 +100,33 @@ def shareable_lambda(block) # :nodoc: end }) - generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash) { + generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash, as_json) { + if as_json + converted = {} + hash.each do |key, value| + string = (String === key || Symbol === key) ? key.to_s : nil + unless string && (string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?)) + key = as_json.call(key, true) + unless String === key || Symbol === key + raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) + end + begin + string = key.to_s.encode(Encoding::UTF_8) + rescue EncodingError => error + raise GeneratorError.new(error.message, key) + end + unless string.valid_encoding? + raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) + end + key = string + end + if converted.key?(key) + raise GeneratorError.new("detected duplicate key #{key.inspect}", key) + end + converted[key] = value + end + hash = converted + end hash.sort_by { |k,| k.to_s.encode(Encoding::UTF_16) }.to_h }) diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 895b09e75..1cacd76c5 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -445,7 +445,7 @@ def generate(obj, anIO = nil) first = true key_type = nil if @rfc8785 - obj = State.rfc8785_sort_keys_proc.call(obj) + obj = State.rfc8785_sort_keys_proc.call(obj, @strict && @as_json) elsif @sort_keys obj = @sort_keys.call(obj) end @@ -585,7 +585,7 @@ def json_transform(state) hash = self if state.rfc8785? - hash = State.rfc8785_sort_keys_proc.call(hash) + hash = State.rfc8785_sort_keys_proc.call(hash, state.strict? && state.as_json) elsif state.sort_keys hash = state.sort_keys.call(hash) end @@ -596,7 +596,7 @@ def json_transform(state) else if key_type && !state.allow_duplicate_key? && key_type != key.class key_type = nil # stop checking - JSON.send(:on_mixed_keys_hash, self) + JSON.send(:on_mixed_keys_hash, hash) end result << delim end diff --git a/test/json/json_coder_test.rb b/test/json/json_coder_test.rb index 574222bf2..b91431821 100755 --- a/test/json/json_coder_test.rb +++ b/test/json/json_coder_test.rb @@ -64,6 +64,54 @@ def test_json_coder_hash_key assert_equal "Integer not allowed as object key in JSON", error.message end + def test_json_coder_rfc8785_converted_keys + time = Time.utc(1999) + calls = [] + coder = JSON::Coder.new(rfc8785: true) do |object, is_key| + calls << [object, is_key] + object.year.to_s + end + input = { time => 1, '1999!' => 2, Time.utc(2000) => 3 } + assert_equal '{"1999":1,"1999!":2,"2000":3}', coder.dump(input) + assert_equal [[time, true], [Time.utc(2000), true]], calls + assert_equal [time, '1999!', Time.utc(2000)], input.keys + end + + def test_json_coder_rfc8785_converted_key_validation + object = Object.new + coder = JSON::Coder.new(rfc8785: true) { 42 } + error = assert_raise(JSON::GeneratorError) { coder.dump(object => 1) } + assert_equal 'Integer not allowed as object key in JSON', error.message + assert_equal 42, error.invalid_object + + coder = JSON::Coder.new(rfc8785: true) { :a } + assert_equal '{"a":1,"b":2}', coder.dump('b' => 2, object => 1) + + invalid = "\xFF" + calls = 0 + coder = JSON::Coder.new(rfc8785: true) { calls += 1; invalid } + error = assert_raise(JSON::GeneratorError) { coder.dump(object => 1) } + assert_same invalid, error.invalid_object + assert_equal 1, calls + end + + def test_json_coder_rfc8785_converted_key_collision + coder = JSON::Coder.new(rfc8785: true) { 'a' } + assert_raise(JSON::GeneratorError) { coder.dump(Object.new => 1, 'a' => 2) } + assert_raise(JSON::GeneratorError) { coder.dump(Object.new => 1, Object.new => 2) } + assert_raise(JSON::GeneratorError) { coder.dump(Object.new => 1, :a => 2) } + end + + def test_json_coder_rfc8785_invalid_encoding_key + calls = [] + coder = JSON::Coder.new(rfc8785: true) do |object, is_key| + calls << [object, is_key] + 'a' + end + assert_equal '{"a":1,"b":2}', coder.dump('b' => 2, "\xFF" => 1) + assert_equal [["\xFF", true]], calls + end + def test_json_coder_options coder = JSON::Coder.new(array_nl: "\n") do |object| 42 diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index fb38b1ab8..7cf76c958 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1297,6 +1297,25 @@ def test_rfc8785_mixed_key_order assert_rfc8785 '{"a":1,"b":2}', { a: 1, 'b' => 2 } end + def test_rfc8785_key_order_with_as_json + key = Object.new + def key.to_s + raise 'must not sort the original key' + end + as_json = ->(object, is_key) { is_key ? 'a' : object } + options = { rfc8785: true, strict: true, as_json: as_json } + object = [{ 'b' => 2, key => 1 }] + expected = '[{"a":1,"b":2}]' + assert_equal expected, JSON.generate(object, options) + assert_equal expected, object.to_json(options) + state = JSON::State.new(options) + io = StringIO.new + state.generate(object, io) + assert_equal expected, io.string + + assert_equal '{"1":1,"b":2}', JSON.generate({ 'b' => 2, 1 => 1 }, options.merge(strict: false)) + end + def test_rfc8785_key_order_with_default_external_encoding # https://www.rfc-editor.org/rfc/rfc8785.html#section-3.2.3 hash = { "\u20ac" => 1, "\r" => 2, "\ufb33" => 3, '1' => 4, From df19d581f0d570569d4d23a171d49e76bfad8c9d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Sat, 10 Oct 2026 03:54:59 +0900 Subject: [PATCH 483/506] Use rb_len_t for String lengths RSTRING_LEN returns rb_len_t on Ruby master, so the variables receiving it follow. Older Rubies get a long typedef and build as before. https://bugs.ruby-lang.org/issues/22400 Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/generator/generator.c | 2 +- ext/json/ext/json.h | 4 ++++ ext/json/ext/parser/parser.c | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 9413bf517..2352c7cec 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -898,7 +898,7 @@ static void raw_generate_json_string(FBuffer *buffer, struct generate_json_data } fbuffer_append_char(buffer, '"'); - long len; + rb_len_t len; search_state search; search.buffer = buffer; RSTRING_GETMEM(str, search.ptr, len); diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index be69bf589..f2be3e552 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -5,6 +5,10 @@ #include "ruby/encoding.h" #include +#ifndef HAVE_RB_LEN_T +typedef long rb_len_t; +#endif + #ifndef RBIMPL_ASSERT_OR_ASSUME # define RBIMPL_ASSERT_OR_ASSUME(x) #endif diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index c912a8eec..80bc12d20 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -138,7 +138,7 @@ ALWAYS_INLINE(static) int rstring_cache_memcmp(const char *str, const char *rptr ALWAYS_INLINE(static) int rstring_cache_cmp(const char *str, const long length, VALUE rstring) { const char *rstring_ptr; - long rstring_length; + rb_len_t rstring_length; RSTRING_GETMEM(rstring, rstring_ptr, rstring_length); @@ -2044,7 +2044,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) const char *method_name_beg = last_colon + 1; long before_len = method_name_beg - name_cstr; - long len = RSTRING_LEN(name) - before_len; + rb_len_t len = RSTRING_LEN(name) - before_len; VALUE method_name = rb_str_substr(name, before_len, len); config->decimal_method_id = SYM2ID(rb_str_intern(method_name)); } else { @@ -2137,7 +2137,7 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) .head = 1, }; - long len; + rb_len_t len; const char *start; RSTRING_GETMEM(Vsource, start, len); @@ -2456,7 +2456,7 @@ static VALUE cResumableParser_feed(VALUE self, VALUE str) rb_str_append(parser->buffer, str); } - long len; + rb_len_t len; const char *start; RSTRING_GETMEM(parser->buffer, start, len); parser->state.start = start; @@ -2762,7 +2762,7 @@ static VALUE cResumableParser_rest(VALUE self) size_t offset = parser->state.cursor - parser->state.start; const char *ptr; - long len; + rb_len_t len; RSTRING_GETMEM(parser->buffer, ptr, len); return rb_utf8_str_new(ptr + offset, len - offset); } From e7a6313b72519eac3b5689d799959593bf8d8521 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Sat, 10 Oct 2026 03:55:23 +0900 Subject: [PATCH 484/506] Use rb_len_t for the column of parse errors The column is a character offset counted by rb_enc_strlen(), which returns rb_len_t. The line number stays long. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/json.h | 2 ++ ext/json/ext/parser/parser.c | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ext/json/ext/json.h b/ext/json/ext/json.h index f2be3e552..a402182d0 100644 --- a/ext/json/ext/json.h +++ b/ext/json/ext/json.h @@ -7,6 +7,8 @@ #ifndef HAVE_RB_LEN_T typedef long rb_len_t; +# define LEN2NUM LONG2NUM +# define PRIdLEN "ld" #endif #ifndef RBIMPL_ASSERT_OR_ASSUME diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 80bc12d20..d35c14955 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -580,7 +580,7 @@ static inline char peek(JSON_ParserState *state) return *state->cursor; } -static void cursor_position(JSON_ParserState *state, long *line_out, long *column_out) +static void cursor_position(JSON_ParserState *state, long *line_out, rb_len_t *column_out) { JSON_ASSERT(!state->parser); JSON_ASSERT(state->cursor); @@ -597,7 +597,7 @@ static void cursor_position(JSON_ParserState *state, long *line_out, long *colum while (cursor > state->start && cursor[-1] != '\n') { cursor--; } - long column = rb_enc_strlen(cursor, state->cursor, enc_utf8) + 1; + rb_len_t column = rb_enc_strlen(cursor, state->cursor, enc_utf8) + 1; while (cursor > state->start) { if (*--cursor == '\n') { @@ -675,11 +675,11 @@ static VALUE json_path_new(JSON_ParserState *state, VALUE duplicate_key) return path; } -static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, long column, bool eos) +static VALUE parse_error_new(JSON_ParserState *state, VALUE message, long line, rb_len_t column, bool eos) { VALUE exc = rb_exc_new_str(eParserError, message); rb_ivar_set(exc, i_at_line, LONG2NUM(line)); - rb_ivar_set(exc, i_at_column, LONG2NUM(column)); + rb_ivar_set(exc, i_at_column, LEN2NUM(column)); rb_ivar_set(exc, i_at_json_path, json_path_new(state, Qundef)); return exc; } @@ -697,9 +697,10 @@ NORETURN(static) void raise_parse_error(const char *format, JSON_ParserState *st } } else { VALUE message = build_parse_error_message(format, state); - long line, column; + long line; + rb_len_t column; cursor_position(state, &line, &column); - rb_str_catf(message, " at line %ld column %ld", line, column); + rb_str_catf(message, " at line %ld column %"PRIdLEN, line, column); rb_exc_raise(parse_error_new(state, message, line, column, eos)); } } @@ -1231,9 +1232,10 @@ NORETURN(static) void raise_duplicate_key_error(JSON_ParserState *state, VALUE d if (state->parser) { // line and columns can't be accurate in resumable exc = parse_error_new(state, message, 0, 0, false); } else { - long line, column; + long line; + rb_len_t column; cursor_position(state, &line, &column); - rb_str_catf(message, " at line %ld column %ld", line, column); + rb_str_catf(message, " at line %ld column %"PRIdLEN, line, column); exc = parse_error_new(state, message, line, column, false); } rb_ivar_set(exc, i_at_json_path, json_path_new(state, duplicate_key)); From a9cd923436310276f3a8721cc2ab3600236ae6b5 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Sat, 10 Oct 2026 04:03:49 +0900 Subject: [PATCH 485/506] Use rb_len_t for lengths given to the String API These lengths come from pointer differences and go to functions that take rb_len_t, so they no longer pass through long on the way. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/parser/parser.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d35c14955..696a878a9 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -79,7 +79,7 @@ static rb_encoding *enc_utf8; #define JSON_RVALUE_CACHE_MAX_ENTRY_LENGTH 55 -static inline VALUE build_interned_string(const char *str, const long length) +static inline VALUE build_interned_string(const char *str, const rb_len_t length) { # ifdef HAVE_RB_ENC_INTERNED_STR return rb_enc_interned_str(str, length, enc_utf8); @@ -89,7 +89,7 @@ static inline VALUE build_interned_string(const char *str, const long length) # endif } -static inline VALUE build_symbol(const char *str, const long length) +static inline VALUE build_symbol(const char *str, const rb_len_t length) { return rb_str_intern(build_interned_string(str, length)); } @@ -890,12 +890,12 @@ static inline VALUE build_string(const char *start, const char *end, bool intern VALUE result; # ifdef HAVE_RB_ENC_INTERNED_STR if (intern) { - result = rb_enc_interned_str(start, (long)(end - start), enc_utf8); + result = rb_enc_interned_str(start, (rb_len_t)(end - start), enc_utf8); } else { - result = rb_utf8_str_new(start, (long)(end - start)); + result = rb_utf8_str_new(start, (rb_len_t)(end - start)); } # else - result = rb_utf8_str_new(start, (long)(end - start)); + result = rb_utf8_str_new(start, (rb_len_t)(end - start)); if (intern) { result = rb_funcall(rb_str_freeze(result), i_uminus, 0); } @@ -1089,7 +1089,7 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser typedef VALUE (*json_number_decode_func_t)(const char *ptr); -static inline VALUE json_decode_large_number(const char *start, long len, json_number_decode_func_t func) +static inline VALUE json_decode_large_number(const char *start, rb_len_t len, json_number_decode_func_t func) { if (RB_LIKELY(len < MAX_NUMBER_STACK_BUFFER)) { char buffer[MAX_NUMBER_STACK_BUFFER]; @@ -1112,7 +1112,7 @@ static VALUE json_decode_inum(const char *buffer) return rb_cstr2inum(buffer, 10); } -NOINLINE(static) VALUE json_decode_large_integer(const char *start, long len) +NOINLINE(static) VALUE json_decode_large_integer(const char *start, rb_len_t len) { return json_decode_large_number(start, len, json_decode_inum); } @@ -1153,7 +1153,7 @@ static VALUE json_decode_dnum(const char *buffer) return DBL2NUM(rb_cstr_to_dbl(buffer, 1)); } -NOINLINE(static) VALUE json_decode_large_float(const char *start, long len) +NOINLINE(static) VALUE json_decode_large_float(const char *start, rb_len_t len) { return json_decode_large_number(start, len, json_decode_dnum); } @@ -2045,7 +2045,7 @@ static int parser_config_init_i(VALUE key, VALUE val, VALUE data) parser_config_wb_write(self, &config->decimal_class, rb_path_to_class(mod_path)); const char *method_name_beg = last_colon + 1; - long before_len = method_name_beg - name_cstr; + rb_len_t before_len = method_name_beg - name_cstr; rb_len_t len = RSTRING_LEN(name) - before_len; VALUE method_name = rb_str_substr(name, before_len, len); config->decimal_method_id = SYM2ID(rb_str_intern(method_name)); From 21193d0be87294cc69a4149f798070a73e3eb3cc Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Sat, 10 Oct 2026 07:09:54 +0900 Subject: [PATCH 486/506] Match rb_hash_bulk_insert fallback to Ruby's declaration Ruby declares it with an rb_len_t count, and a mismatching definition fails builds where the fallback is compiled against those headers. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/parser/parser.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 696a878a9..70e34edb7 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -20,9 +20,9 @@ static int utf8_encindex; #ifndef HAVE_RB_HASH_BULK_INSERT // For TruffleRuby static void -rb_hash_bulk_insert(long count, const VALUE *pairs, VALUE hash) +rb_hash_bulk_insert(rb_len_t count, const VALUE *pairs, VALUE hash) { - long index = 0; + rb_len_t index = 0; while (index < count) { VALUE name = pairs[index++]; VALUE value = pairs[index++]; From 8df32ab07485db5fc288046056ae8bf4e34e9366 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 12 Oct 2026 10:34:31 +0900 Subject: [PATCH 487/506] Index the generated JSON array by rb_len_t An int index overflows on an Array longer than INT_MAX, which LP64 and now mswin can hold. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/generator/generator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 2352c7cec..38c88f231 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1098,7 +1098,7 @@ static void generate_json_array(FBuffer *buffer, struct generate_json_data *data fbuffer_append_char(buffer, '['); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); - for (int i = 0; i < RARRAY_LEN(obj); i++) { + for (rb_len_t i = 0; i < RARRAY_LEN(obj); i++) { if (i > 0) { fbuffer_append_char(buffer, ','); if (RB_UNLIKELY(data->state->array_nl)) fbuffer_append_str(buffer, data->state->array_nl); From 429c1094585d7f5c144d1accb536192ad31e0314 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 09:28:28 +0200 Subject: [PATCH 488/506] More hardening against adversarial `encode` methods --- ext/json/ext/parser/parser.c | 4 +--- test/json/json_parser_test.rb | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 70e34edb7..b3b6bfdca 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -2113,9 +2113,7 @@ static VALUE cParser_parse(JSON_ParserConfig *config, VALUE src) // Ensure the string isn't mutated under us. // The classic API to use is `rb_str_locktmp`, but then we'd // need to use `rb_protect` to make sure we always unlock. - if (Vsource == src) { - Vsource = rb_str_new_frozen(Vsource); - } + Vsource = rb_str_new_frozen(Vsource); VALUE rvalue_stack_buffer[RVALUE_STACK_INITIAL_CAPA]; rvalue_stack value_stack = { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index d9e3256bd..0b9d5fc96 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1045,6 +1045,41 @@ def test_mutating_source_string_during_parsing assert_equal expected, actual end + def test_adversary_encode + fake_string = Class.new(String) do + def encode(*) + 42 + end + end + + assert_raise TypeError, Encoding::InvalidByteSequenceError do + JSON.parse(fake_string.new("\xff".b, encoding: Encoding::UTF_16)) + end + + if RUBY_ENGINE != "jruby" + fake_string = Class.new(String) do + attr_reader :encoded_str + + def initialize(...) + super + @encoded_str = JSON.dump(1000.times.to_a) + end + + def encode(*) + @encoded_str + end + end + + fake = fake_string.new("blah", encoding: Encoding::UTF_16) + assert_equal 1000.times.to_a, JSON.parse(fake, on_load: ->(v) { + if v == 20 + fake.encoded_str.clear + end + v + }) + end + end + private def assert_equal_float(expected, actual, delta = 1e-2) From c1a4372837233df9d78c9f9ff0e34a09272bf59e Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 15:45:27 +0200 Subject: [PATCH 489/506] ResumableParser: fix handling of incomplete unicode escape `spe` is the string end, so if the escape is incomplete, it doesn't mean we reached the end of stream, hence it's a syntax error, not an EOS error. --- ext/json/ext/parser/parser.c | 2 +- test/json/resumable_parser_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index b3b6bfdca..52b6e2250 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -753,7 +753,7 @@ static const signed char digit_values[256] = { static uint32_t unescape_unicode(JSON_ParserState *state, const char *sp, const char *spe) { if (RB_UNLIKELY(sp > spe - 4)) { - raise_eos_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); + raise_syntax_error_at("incomplete unicode character escape sequence at %s", state, sp - 2); } const unsigned char *p = (const unsigned char *)sp; diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index a704e9edd..dbf3ea7b6 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -179,6 +179,12 @@ def test_parse_byte_by_byte_string assert_resumed_parsing('"te\\u2028st"') assert_resumed_parsing(JSON.generate("te\u2028st")) assert_resumed_parsing(JSON.generate("te \u2028 st")) + + parser = JSON::ResumableParser.new + parser << '"\u12"' + assert_raise JSON::ParserError do + parser.parse + end end def test_parse_byte_by_byte_numbers From 1b67eb3db207ad17336b70b7a6b1b794885f0a0b Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 15:50:51 +0200 Subject: [PATCH 490/506] Fix parsing of unescaped NULL bytes in strings --- ext/json/ext/parser/parser.c | 4 ++-- test/json/json_parser_test.rb | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 52b6e2250..288521e1c 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1046,13 +1046,13 @@ NOINLINE(static) VALUE json_string_unescape(JSON_ParserState *state, JSON_Parser p = ++pe; break; } - case 0: - return Qundef; default: if ((unsigned char)*pe < 0x20) { if (!config->allow_control_characters) { if (*pe == '\n') { raise_syntax_error_at("Invalid unescaped newline character (\\n) in string: %s", state, pe - 1); + } else if (*pe == '\0') { + raise_syntax_error_at("Invalid unescaped NULL character (\\0) in string: %s", state, pe - 1); } raise_syntax_error_at("invalid ASCII control character in string: %s", state, pe - 1); } diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 0b9d5fc96..75e99904d 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -239,6 +239,8 @@ def test_parse_control_char_and_backslash end JSON.parse(%("#{'a' * 30}#{backslash_and_control_char}"), allow_control_characters: true, allow_invalid_escape: true) + + assert_equal ["a\u0000b", 2], JSON.parse("[\"a\\\x00b\", 2]", allow_control_characters: true, allow_invalid_escape: true) end def test_parse_invalid_escape From 6ea62016fb726739439b8a68454eb54550e70094 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 16:10:06 +0200 Subject: [PATCH 491/506] Fix parsing of number with out of range negative exponents --- ext/json/ext/parser/parser.c | 6 +++--- java/src/json/ext/Parser.java | 6 +++--- test/json/json_parser_test.rb | 10 ++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index 288521e1c..d86c347ad 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -1169,11 +1169,11 @@ static inline VALUE json_decode_float(JSON_ParserConfig *config, uint64_t mantis return rb_funcallv(config->decimal_class, config->decimal_method_id, 1, &text); } - if (RB_UNLIKELY(exponent > INT32_MAX)) { + if (RB_UNLIKELY(exponent >= (int64_t)INT32_MAX)) { return negative ? CMinusInfinity : CInfinity; } - if (RB_UNLIKELY(exponent < INT32_MIN)) { + if (RB_UNLIKELY(exponent <= INT32_MIN)) { return rb_float_new(negative ? -0.0 : 0.0); } @@ -1514,7 +1514,7 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } if (RB_UNLIKELY(exponent_digits >= 20 || abs_exponent > (uint64_t)INT64_MAX)) { - exponent = negative_exponent ? INT64_MIN : INT64_MAX; + exponent = negative_exponent ? INT32_MIN : INT32_MAX; } else { exponent = negative_exponent ? -(int64_t)abs_exponent : (int64_t)abs_exponent; } diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 88bc054ef..6fa5d4383 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -634,7 +634,7 @@ private IRubyObject parseNumber(int numberStart, byte first, boolean negative) { } if (exponent_digits >= 20 || Long.compareUnsigned(abs_exponent, Long.MAX_VALUE) > 0) { - exponent = negative_exponent ? Long.MIN_VALUE : Long.MAX_VALUE; + exponent = negative_exponent ? Integer.MIN_VALUE : Integer.MAX_VALUE; } else { exponent = negative_exponent ? -abs_exponent : abs_exponent; } @@ -655,8 +655,8 @@ private IRubyObject parseNumber(int numberStart, byte first, boolean negative) { IRubyObject decodeFloat(ThreadContext context, long mantissa, int mantissa_digits, long exponent, boolean negative, int start) { if (config.decimalClass != null) return config.decimalFactory.apply(context, absSubSequence(start, cursor)); - if (exponent > Integer.MAX_VALUE) return getConstant(negative ? CONST_MINUS_INFINITY : CONST_INFINITY); - if (exponent < Integer.MIN_VALUE) return RubyFloat.newFloat(context.runtime, negative ? -0.0 : 0.0); + if (exponent >= Integer.MAX_VALUE) return getConstant(negative ? CONST_MINUS_INFINITY : CONST_INFINITY); + if (exponent <= Integer.MIN_VALUE) return RubyFloat.newFloat(context.runtime, negative ? -0.0 : 0.0); // Ryu has rounding issues with subnormals around 1e-310 (< 2.225e-308) if (mantissa_digits > 17 || mantissa_digits + exponent < -307) { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 75e99904d..47d84b398 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -147,6 +147,12 @@ def test_parse_numbers assert_raise(ParserError) { parse('-Infinity') } assert_equal(-1.0/0, parse('-Infinity', allow_nan: true)) capture_output { assert_equal(Float::INFINITY, parse("23456789012E666")) } + + assert_parse_float "1.5e-99999999999999999999" + assert_parse_float "1.5e-9223372036854775808" + assert_parse_float "0.0e-99999999999999999999" + assert_parse_float "-1.5e-99999999999999999999" + assert_parse_float "123.456e-18446744073709551615" end INTEGER_FAST_PATH_BOUNDARIES = [ @@ -1084,6 +1090,10 @@ def encode(*) private + def assert_parse_float(float_str) + assert_equal Float(float_str), JSON.parse(float_str) + end + def assert_equal_float(expected, actual, delta = 1e-2) Array === expected and expected = expected.first Array === actual and actual = actual.first From 78d6a83a3cdbc8c0fea6112d89cfe9d34164e733 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 16:27:22 +0200 Subject: [PATCH 492/506] parser.c: only update the stack once fully spilled --- ext/json/ext/parser/parser.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ext/json/ext/parser/parser.c b/ext/json/ext/parser/parser.c index d86c347ad..b456a4e23 100644 --- a/ext/json/ext/parser/parser.c +++ b/ext/json/ext/parser/parser.c @@ -323,14 +323,17 @@ static const rb_data_type_t JSON_Parser_rvalue_stack_type = { static rvalue_stack *rvalue_stack_spill(rvalue_stack *old_stack, VALUE *handle, rvalue_stack **stack_ref) { rvalue_stack *stack; - *handle = TypedData_Make_Struct(0, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack); - *stack_ref = stack; + VALUE stack_handle = TypedData_Make_Struct(0, rvalue_stack, &JSON_Parser_rvalue_stack_type, stack); MEMCPY(stack, old_stack, rvalue_stack, 1); stack->capa = old_stack->capa << 1; stack->ptr = ALLOC_N(VALUE, stack->capa); stack->type = RVALUE_STACK_HEAP_ALLOCATED; MEMCPY(stack->ptr, old_stack->ptr, VALUE, old_stack->head); + + *handle = stack_handle; + *stack_ref = stack; + return stack; } From e19f7d75a613ecd3f4fd7033c06925bd8c982cfe Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 12 Oct 2026 16:37:32 +0200 Subject: [PATCH 493/506] Handle invalid encoding in JSON path keys --- lib/json/common.rb | 5 ++++- test/json/json_parser_test.rb | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/json/common.rb b/lib/json/common.rb index 01a4be18c..4b2d6d87f 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -171,7 +171,10 @@ def build_json_path(segments) when Integer "[#{segment}]" when String, Symbol - if segment.match?(/\A[a-zA-Z\$\_][a-zA-Z\$\_0-9]*\z/) + if !segment.valid_encoding? + segment = segment.b.gsub(/["\\]/n, { '"' => '\\"', '\\' => '\\\\' }).force_encoding(Encoding::UTF_8) + %{["#{segment}"]} + elsif segment.match?(/\A[a-zA-Z\$\_][a-zA-Z\$\_0-9]*\z/) ".#{segment}" else segment = segment.to_s.gsub(/["\\]/, { '"' => '\\"', '\\' => '\\\\' }) diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 47d84b398..3607be33a 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -931,6 +931,10 @@ def test_parse_error_json_path assert_parse_error_at "$[4]", '[1,2,3,4,5' assert_parse_error_at "$[5]", '[1,2,3,4,5,' assert_parse_error_at "$[5]", '[1,2,3,4,5,]' + + if RUBY_ENGINE != "jruby" + assert_parse_error_at %($["\xff"][1]), %({"\xff":[1,x]}) + end end def test_parse_error_position From d09e383d19bb84d6209aa00999476eaeb4ccbd53 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 13 Oct 2026 09:15:24 +0200 Subject: [PATCH 494/506] Refactor `rfc8785_sort_keys_proc` --- lib/json/ext.rb | 86 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 26 deletions(-) diff --git a/lib/json/ext.rb b/lib/json/ext.rb index 7d7b09b27..75d0e4e3d 100644 --- a/lib/json/ext.rb +++ b/lib/json/ext.rb @@ -38,6 +38,56 @@ def shareable_lambda(block) # :nodoc: block end end + + def valid_encoding?(str) # :nodoc: + str.ascii_only? || (str.encoding == Encoding::UTF_8 && str.valid_encoding?) + end + + def coerce_key(key, as_json) # :nodoc: + as_json_called = false + + unless String === key || Symbol === key + key = as_json.call(key, true) + as_json_called = true + end + + if Symbol === key + key = key.to_s + end + + unless String === key + raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) + end + + unless valid_encoding?(key) + if as_json_called + raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) + end + + key = as_json.call(key, true) + unless String === key + raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) + end + + unless Ext.valid_encoding?(key) + raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) + end + end + + key + end + + def on_duplicate_key(hash, as_json) + set = {} + hash.each_key do |key| + key = coerce_key(key, as_json) + if set[key] + raise GeneratorError.new("detected duplicate key #{key.inspect}", key) + else + set[key] = true + end + end + end end require 'json/ext/parser' @@ -101,33 +151,17 @@ def shareable_lambda(block) # :nodoc: }) generator::State.rfc8785_sort_keys_proc = shareable_lambda(->(hash, as_json) { - if as_json - converted = {} - hash.each do |key, value| - string = (String === key || Symbol === key) ? key.to_s : nil - unless string && (string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?)) - key = as_json.call(key, true) - unless String === key || Symbol === key - raise GeneratorError.new("#{key.class} not allowed as object key in JSON", key) - end - begin - string = key.to_s.encode(Encoding::UTF_8) - rescue EncodingError => error - raise GeneratorError.new(error.message, key) - end - unless string.valid_encoding? - raise GeneratorError.new("source sequence is illegal/malformed utf-8", key) - end - key = string - end - if converted.key?(key) - raise GeneratorError.new("detected duplicate key #{key.inspect}", key) - end - converted[key] = value - end - hash = converted + converted = if as_json + hash.transform_keys { |k| Ext.coerce_key(k, as_json) } + else + hash.transform_keys(&:to_s) end - hash.sort_by { |k,| k.to_s.encode(Encoding::UTF_16) }.to_h + + if converted.size < hash.size + Ext.on_duplicate_key(hash, as_json) + end + + converted.sort_by { |k,| k.encode(Encoding::UTF_16) }.to_h }) JSON.generator = generator From a15b2ba603bf4692b9066dcc884c96f8984777e1 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 13 Oct 2026 10:03:21 +0200 Subject: [PATCH 495/506] Release 3.1.0 --- CHANGES.md | 7 +++++++ lib/json/version.rb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9ef0ce94a..d7c7ad118 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,14 @@ ### Unreleased +### 2026-10-13 (3.1.0) + +* Added `rfc8785` generator option to canonicalize the output. Replaces the `json-canonicalization` with numerous correctness fixes. +* Use the `rb_len_t` type introduced in Ruby 4.1 to allow for proper 64bit support on LP64 platforms (Windows). * JRuby: parser errors now include the position (`line`, `column` and the message suffix) as well as `json_path`, matching the C extension. +* Improved the accuracy of parser error's `line`, `column` and snipets in presence of multibyte characters. +* Fix `JSON.load` to work correctly when provided a `Zlib::GzipReader` as IO source. +* Reject non-ASCII BINARY strings during JSON generation. This was deprecated since a long time and scheduled to be removed for 3.0. ### 2026-09-09 (3.0.2) diff --git a/lib/json/version.rb b/lib/json/version.rb index 318272def..4d37baaa7 100644 --- a/lib/json/version.rb +++ b/lib/json/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module JSON - VERSION = '3.0.2' + VERSION = '3.1.0' end From faaccd3785ddbcbaf6d35855c997a8dff4066f78 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 13 Oct 2026 19:19:14 +0900 Subject: [PATCH 496/506] json: Use SSE2 on x86-64 without checking at runtime __builtin_cpu_supports refers to __cpu_model in compiler-rt, which clang-cl does not link, so generator.so and parser.so failed with an undefined symbol. SSE2 is part of the x86-64 baseline, as NEON is for AArch64, where no check is made either. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/simd/simd.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 611b41b06..721972c9e 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -187,12 +187,8 @@ ALWAYS_INLINE(static) TARGET_SSE2 int string_scan_simd_sse2(const char **ptr, co static inline SIMD_Implementation find_simd_implementation(void) { - // TODO Revisit. I think the SSE version now only uses SSE2 instructions. - if (__builtin_cpu_supports("sse2")) { - return SIMD_SSE2; - } - - return SIMD_NONE; + // SSE2 is part of the x86-64 baseline. + return SIMD_SSE2; } #endif /* HAVE_X86INTRIN_H */ From fae76b9eb4723a32dc91d06edbf32248a6472622 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 15 Oct 2026 13:49:51 +0900 Subject: [PATCH 497/506] Drop the leftovers of runtime SSE2 detection Neither the `__builtin_cpu_supports` probe in conf.rb nor `cpuid.h` has a use now that SSE2 is assumed on x86-64. The probe kept SIMD off on GCC older than 4.8, and the `HAVE_CPUID_H` guard broke builds that have `x86intrin.h` but not `cpuid.h` with a redefinition of `find_simd_implementation`. Co-Authored-By: Claude Opus 5.5 --- ext/json/ext/simd/conf.rb | 7 ++----- ext/json/ext/simd/simd.h | 4 ---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/ext/json/ext/simd/conf.rb b/ext/json/ext/simd/conf.rb index 76f774bc9..9b87473d5 100644 --- a/ext/json/ext/simd/conf.rb +++ b/ext/json/ext/simd/conf.rb @@ -1,16 +1,15 @@ case RbConfig::CONFIG['host_cpu'] when /^(arm|aarch64)/ # Try to compile a small program using NEON instructions - header, type, init, extra = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)', nil + header, type, init = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)' when /^(x86_64|x64)/ - header, type, init, extra = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)', 'if (__builtin_cpu_supports("sse2")) { printf("OK"); }' + header, type, init = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)' end if header if have_header(header) && try_compile(<<~SRC, '-Werror=implicit-function-declaration') #{cpp_include(header)} int main(int argc, char **argv) { #{type} test = #{init}; - #{extra} if (argc > 100000) printf("%p", &test); return 0; } @@ -20,5 +19,3 @@ puts "Disable SIMD" end end - -have_header('cpuid.h') diff --git a/ext/json/ext/simd/simd.h b/ext/json/ext/simd/simd.h index 721972c9e..2f3f838a9 100644 --- a/ext/json/ext/simd/simd.h +++ b/ext/json/ext/simd/simd.h @@ -143,7 +143,6 @@ ALWAYS_INLINE(static) int string_scan_simd_neon(const char **ptr, const char *en #define HAVE_SIMD 1 #define HAVE_SIMD_SSE2 1 -#ifdef HAVE_CPUID_H #define FIND_SIMD_IMPLEMENTATION_DEFINED 1 #if defined(__clang__) || defined(__GNUC__) @@ -182,9 +181,6 @@ ALWAYS_INLINE(static) TARGET_SSE2 int string_scan_simd_sse2(const char **ptr, co return 0; } -#include -#endif /* HAVE_CPUID_H */ - static inline SIMD_Implementation find_simd_implementation(void) { // SSE2 is part of the x86-64 baseline. From c839870678fd7506eede270b24fbf3484053b361 Mon Sep 17 00:00:00 2001 From: Sebastian Turner Date: Mon, 19 Oct 2026 08:47:29 -0700 Subject: [PATCH 498/506] Fix ParserError#json_path with symbolize_names: true Signed-off-by: Sebastian Turner --- CHANGES.md | 2 ++ lib/json/common.rb | 3 ++- test/json/json_parser_test.rb | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d7c7ad118..1e1b5a9ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,8 @@ ### Unreleased +* Fix `JSON::ParserError#json_path` raising `NoMethodError` when the document was parsed with `symbolize_names: true`. + ### 2026-10-13 (3.1.0) * Added `rfc8785` generator option to canonicalize the output. Replaces the `json-canonicalization` with numerous correctness fixes. diff --git a/lib/json/common.rb b/lib/json/common.rb index 4b2d6d87f..bf96f4137 100644 --- a/lib/json/common.rb +++ b/lib/json/common.rb @@ -171,13 +171,14 @@ def build_json_path(segments) when Integer "[#{segment}]" when String, Symbol + segment = segment.to_s if !segment.valid_encoding? segment = segment.b.gsub(/["\\]/n, { '"' => '\\"', '\\' => '\\\\' }).force_encoding(Encoding::UTF_8) %{["#{segment}"]} elsif segment.match?(/\A[a-zA-Z\$\_][a-zA-Z\$\_0-9]*\z/) ".#{segment}" else - segment = segment.to_s.gsub(/["\\]/, { '"' => '\\"', '\\' => '\\\\' }) + segment = segment.gsub(/["\\]/, { '"' => '\\"', '\\' => '\\\\' }) %{["#{segment}"]} end else diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3607be33a..3f8575c8a 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1004,6 +1004,18 @@ def test_parse_error_json_path_key_escaping assert_parse_error_at '$.foo["1x"]', '{"foo": {"1x": xyz}}' end + def test_parse_error_json_path_symbolize_names + assert_parse_error_at "$.a[1].b" do + JSON.parse('{"a": [1, {"b": xyz}]}', symbolize_names: true) + end + assert_parse_error_at '$["hello world"]' do + JSON.parse('{"hello world": xyz}', symbolize_names: true) + end + assert_parse_error_at "$.x.a" do + JSON.parse('{"x": {"a": 1, "a": 2}}', symbolize_names: true) + end + end + def test_parse_error_json_path_duplicate_key assert_parse_error_at "$.a", '{"a": 1, "a": 2}' assert_parse_error_at "$.x.a", '{"x": {"a": 1, "b": 2, "a": 3}}' From 9d783a28903539d5e3a0fac18407d3875c10120c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 2 Nov 2026 15:00:55 +0100 Subject: [PATCH 499/506] Fix RFC tests not running Because of the trailing __FILE__, no files were matched and the block didn't run, which was hiding the error in the block defining the test (`output` local variable not existing). --- test/json/json_generator_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 7cf76c958..8de5f80f9 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1490,12 +1490,12 @@ def test_rfc8785_numbers end fixtures_path = File.expand_path('../fixtures/rfc8785/', __FILE__) - Dir[File.join(fixtures_path, "input/*.json", __FILE__)].each do |input| + Dir[File.join(fixtures_path, "input/*.json")].each do |input| filename = File.basename(input) name, _ = File.basename(filename, ".json") expected = File.join(fixtures_path, 'output', filename) define_method("test_rfc8785_#{name}") do - assert_rfc8785(File.read(output), JSON.load_file(input)) + assert_rfc8785(File.read(expected), JSON.load_file(input)) end end From 4b431671b1506b01b10d9275f829f33ed26f0395 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 2 Nov 2026 15:54:41 +0100 Subject: [PATCH 500/506] Update changelog with 2.18.2 release --- CHANGES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1e1b5a9ed..079fd0c49 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -140,6 +140,11 @@ Numerous rarely used aliases have been removed. * Fix `allow_blank` parsing option to no longer allow invalid types (e.g. `load([], allow_blank: true)` now raise a type error). * Add `allow_invalid_escape` parsing option to ignore backslashes that aren't followed by one of the valid escape characters. +### 2026-11-02 (2.18.2) + +* Backport handling of unescaped control characters preceeded by a backslash from 2.19.3. + GHSA-cjpr-jw2g-fw99. + ### 2026-02-03 (2.18.1) * Fix a potential crash in very specific circumstance if GC triggers during a call to `to_json` From dc731b3a767c73b66551b26922bb13a00c7785b5 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 3 Nov 2026 10:30:18 +0900 Subject: [PATCH 501/506] Test on macos-15 instead of the retired macos-14 GitHub Actions retired the macOS 14 images on 2026-11-02 (actions/runner-images#13518). macos-15 keeps the arm64 job one release behind macos-latest, which is now macOS 26. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d98aa5715..1d8d18e21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - { os: ubuntu-24.04-arm, ruby: 3.4 } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DISABLE_SIMD=1" } - { os: ubuntu-latest , ruby: 3.4, env: "JSON_DEBUG=1" } - - { os: macos-14, ruby: 3.4 } + - { os: macos-15, ruby: 3.4 } - { os: windows-latest , ruby: mswin } # ruby/ruby windows CI - { os: macos-latest , ruby: jruby-9.4 } # Ruby 3.1 - { os: ubuntu-latest , ruby: jruby-9.4 } # Ruby 3.1 From 1c67ef2aa9aaf412d37b808f9923dc4879c732c4 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 3 Nov 2026 11:46:08 +0900 Subject: [PATCH 502/506] Fix failures on non-UTF-8 environments --- test/json/json_generator_test.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 8de5f80f9..1d9318fd5 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1495,7 +1495,7 @@ def test_rfc8785_numbers name, _ = File.basename(filename, ".json") expected = File.join(fixtures_path, 'output', filename) define_method("test_rfc8785_#{name}") do - assert_rfc8785(File.read(expected), JSON.load_file(input)) + assert_rfc8785(File.read(expected, encoding: Encoding::UTF_8), JSON.load_file(input)) end end From 0292a445dfdbc450683c782275dd95c6b81e8bf8 Mon Sep 17 00:00:00 2001 From: Sampo Kuokkanen Date: Wed, 4 Nov 2026 21:31:14 +0900 Subject: [PATCH 503/506] Java parser: don't let the caller mutate the bytes being parsed ParserSession reads the source string's backing byte[] directly, but the string itself was never marked as shared. A decimal_class callback that calls source.tr! could then rewrite the bytes the parser is still reading. The existing test_mutating_source_string_during_parsing only passes on JRuby by accident: with an unknown coderange String#tr! happens to build a new buffer. Once the coderange is known (for example after source.ascii_only?), tr! writes in place and the parser sees the change. Freeze a copy of the source before parsing, like rb_str_new_frozen in parser.c. RubyString#newFrozen shares the ByteList and marks the original as shared, so a later mutation copies first. --- java/src/json/ext/Parser.java | 5 ++++- test/json/json_parser_test.rb | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/java/src/json/ext/Parser.java b/java/src/json/ext/Parser.java index 6fa5d4383..eb5ce3a9a 100644 --- a/java/src/json/ext/Parser.java +++ b/java/src/json/ext/Parser.java @@ -164,7 +164,10 @@ private RubyString convertEncoding(ThreadContext context, RubyString source) { */ @JRubyMethod public IRubyObject parse(ThreadContext context, IRubyObject source) { - return new ParserSession(this, convertEncoding(context, source.convertToString()), info).parse(context); + // Like rb_str_new_frozen in parser.c: the caller's string becomes shared, so mutating it during parsing + // (e.g. from a decimal_class callback) copies its bytes instead of changing the ones being parsed. + RubyString frozenSource = convertEncoding(context, source.convertToString()).newFrozen(); + return new ParserSession(this, frozenSource, info).parse(context); } private RubyFloat createFloat(final ThreadContext context, final ByteList num) { diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 3f8575c8a..0549f6f4c 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1069,6 +1069,23 @@ def test_mutating_source_string_during_parsing assert_equal expected, actual end + def test_mutating_source_string_with_known_coderange_during_parsing + expected = ([1] * 100) + [2.3] + ([1] * 100) + source = JSON.generate(expected) + source.ascii_only? # caches the coderange, which lets JRuby's String#tr! write in place + expected.delete_at(100) + + fake_decimal_class = Class.new + fake_decimal_class.define_method(:initialize) do |number| + source.tr!('1', '0') + number.to_f + end + + actual = JSON.parse(source, decimal_class: fake_decimal_class) + actual.delete_at(100) + assert_equal expected, actual + end + def test_adversary_encode fake_string = Class.new(String) do def encode(*) From bbb92b5f871d2e6a293c1c3f454f1d8fa151215a Mon Sep 17 00:00:00 2001 From: Stephan Kaag <209887080+stephanmoneybird@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:59:37 +0100 Subject: [PATCH 504/506] Check duplicate keys when a non String/Symbol key follows a String/Symbol key Fix: #1125 Co-Authored-By: Jean Boussier --- CHANGES.md | 1 + ext/json/ext/generator/generator.c | 3 +++ lib/json/truffle_ruby/generator.rb | 12 ++++++++++-- test/json/json_generator_test.rb | 25 +++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 079fd0c49..f405db416 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ### Unreleased * Fix `JSON::ParserError#json_path` raising `NoMethodError` when the document was parsed with `symbolize_names: true`. +* Fix `JSON.generate` not detecting duplicate keys such as `{ "1" => 1, 1 => 2 }`. ### 2026-10-13 (3.1.0) diff --git a/ext/json/ext/generator/generator.c b/ext/json/ext/generator/generator.c index 38c88f231..87da10027 100644 --- a/ext/json/ext/generator/generator.c +++ b/ext/json/ext/generator/generator.c @@ -1020,6 +1020,9 @@ json_object_i(VALUE key, VALUE val, VALUE _arg) raise_generator_error(key, "%"PRIsVALUE" not allowed as object key in JSON", CLASS_OF(key)); } } + + json_inspect_hash_with_mixed_keys(arg); + key_to_s = rb_convert_type(key, T_STRING, "String", "to_s"); break; } diff --git a/lib/json/truffle_ruby/generator.rb b/lib/json/truffle_ruby/generator.rb index 1cacd76c5..f73708f2d 100644 --- a/lib/json/truffle_ruby/generator.rb +++ b/lib/json/truffle_ruby/generator.rb @@ -451,7 +451,11 @@ def generate(obj, anIO = nil) end obj.each_pair do |k,v| if first - key_type = k.class + if ::String == k.class || ::Symbol == k.class + key_type = k.class + else + JSON.send(:on_mixed_keys_hash, obj) + end else if key_type && !@allow_duplicate_key && key_type != k.class key_type = nil # stop checking @@ -592,7 +596,11 @@ def json_transform(state) hash.each { |key, value| if first - key_type = key.class + if ::String == key.class || ::Symbol == key.class + key_type = key.class + else + JSON.send(:on_mixed_keys_hash, hash) + end else if key_type && !state.allow_duplicate_key? && key_type != key.class key_type = nil # stop checking diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index 1d9318fd5..bfc324104 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1202,11 +1202,32 @@ def test_generate_duplicate_keys_allowed end def test_generate_duplicate_keys_disallowed - hash = { foo: 1, "foo" => 2 } + assert_duplicate_key({ foo: 1, "foo" => 2 }) + assert_duplicate_key({ "1" => 1, 1 => 2 }) + assert_duplicate_key({ :"1" => 1, 1 => 2 }) + + fake_string = Class.new do + def initialize(str) + @str = str + end + + def to_s + @str.dup + end + end + + assert_duplicate_key({ fake_string.new("a") => 1, fake_string.new("a") => 2 }) + end + + def assert_duplicate_key(hash) error = assert_raise JSON::GeneratorError do JSON.generate(hash, allow_duplicate_key: false) end - assert_equal %(detected duplicate key "foo" in #{hash.inspect}), error.message + assert_equal %(detected duplicate key "#{hash.keys.first}" in #{hash.inspect}), error.message + end + + def test_generate_mixed_keys_without_duplicates + assert_equal %({"1":1,"2":2}), JSON.generate({ "1" => 1, 2 => 2 }, allow_duplicate_key: false) end def test_frozen From 8bda05413e5f154da639740476a55d154e628912 Mon Sep 17 00:00:00 2001 From: Erik Berlin Date: Thu, 5 Nov 2026 11:58:48 -0800 Subject: [PATCH 505/506] Run the RFC 8785 number tests on JRuby 10.1.3 and later The RFC 8785 number formatter uses Float#to_s, so these tests were skipped on every engine but CRuby. JRuby 10.1.3 gives the shortest round-trip digits in Float#to_s (https://github.com/jruby/jruby/pull/9916), and all 26 tests pass there. Older JRuby versions print the smallest subnormal as 4.9e-324, so they still skip them. Part of https://github.com/ruby/json/issues/1123 --- test/json/json_generator_test.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index bfc324104..ca1af4c4d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1469,6 +1469,11 @@ def test_rfc8785_numbers assert_rfc8785 '0.30000000000000004', 0.1 + 0.2 end + # RFC 8785 numbers are formatted with Float#to_s, which only gives the + # shortest round-trip digits on JRuby 10.1.3 and later. + SHORTEST_FLOAT_TO_S = RUBY_ENGINE == "ruby" || + (RUBY_ENGINE == "jruby" && Gem::Version.new(JRUBY_VERSION) >= Gem::Version.new("10.1.3")) + # https://www.rfc-editor.org/rfc/rfc8785.html#appendix-B { '0000000000000000' => '0', @@ -1499,7 +1504,7 @@ def test_rfc8785_numbers '43143ff3c1cb0959' => '1424953923781206.2', }.each do |bits, expected| define_method("test_rfc8785_number_#{bits}") do - omit "Float#to_s is not always shortest on this engine" unless RUBY_ENGINE == "ruby" + omit "Float#to_s is not always shortest on this engine" unless SHORTEST_FLOAT_TO_S number = [bits].pack('H*').unpack('G').first if expected assert_rfc8785 expected, number From cf987b116dadba97960de61d9fbde9022c956dcf Mon Sep 17 00:00:00 2001 From: Erik Berlin Date: Thu, 5 Nov 2026 12:11:57 -0800 Subject: [PATCH 506/506] Java generator: raise TypeError for a String key whose to_s is broken When a Hash key is a String subclass, both generators call its to_s. generator.c raises a TypeError in convert_string_subclass if to_s doesn't return a String. The Java generator passed the result on, then fell back to TypeConverter.convertToType, which returned the key itself because it already is a String. So JSON.generate(s => 1) produced {"test":1} on JRuby instead of raising. Raise the same TypeError, with the same message, in castKey. The class names go through to_s like PRIsVALUE does, so anonymous classes show as # the way they do on CRuby. This lets test_string_subclass_with_broken_to_s run on JRuby. Part of https://github.com/ruby/json/issues/1123 --- CHANGES.md | 1 + java/src/json/ext/Generator.java | 9 ++++++++- test/json/json_generator_test.rb | 2 -- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f405db416..2bad52aed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ * Fix `JSON::ParserError#json_path` raising `NoMethodError` when the document was parsed with `symbolize_names: true`. * Fix `JSON.generate` not detecting duplicate keys such as `{ "1" => 1, 1 => 2 }`. +* JRuby: raise `TypeError` when a String subclass used as a Hash key has a `to_s` that doesn't return a String, like the C extension. ### 2026-10-13 (3.1.0) diff --git a/java/src/json/ext/Generator.java b/java/src/json/ext/Generator.java index 3b05812f5..750be6bf6 100644 --- a/java/src/json/ext/Generator.java +++ b/java/src/json/ext/Generator.java @@ -631,7 +631,14 @@ private static IRubyObject castKey(ThreadContext context, IRubyObject key) { if (keyClass == runtime.getString()) { return key; } else { - return key.callMethod(context, "to_s"); + // C: convert_string_subclass + IRubyObject keyStr = key.callMethod(context, "to_s"); + if (!(keyStr instanceof RubyString)) { + IRubyObject className = keyClass.getRealClass().callMethod(context, "to_s"); + throw runtime.newTypeError("can't convert " + className + " to String (" + + className + "#to_s gives " + keyStr.getMetaClass().getRealClass().callMethod(context, "to_s") + ")"); + } + return keyStr; } } else if (keyClass == runtime.getSymbol()) { return ((RubySymbol) key).id2name(context); diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index ca1af4c4d..c4b1a8ace 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -1054,8 +1054,6 @@ def to_s s = klass.new("test") assert_equal '["test"]', JSON.generate([s]) - omit("Can't figure out how to match behavior in java code") if RUBY_PLATFORM == "java" - assert_raise TypeError do JSON.generate(s => 1) end