diff --git a/jq.pyx b/jq.pyx index 125ebfc..0e2abca 100644 --- a/jq.pyx +++ b/jq.pyx @@ -43,6 +43,7 @@ cdef extern from "jv.h": jv_parser* jv_parser_new(int) void jv_parser_free(jv_parser*) void jv_parser_set_buf(jv_parser*, const char*, int, int) + int jv_parser_remaining(jv_parser*) jv jv_parser_next(jv_parser*) @@ -171,11 +172,11 @@ cdef class _JSONParser(object): """ cdef jv value while True: - # If we have no bytes to parse - if self._bytes is None: - # Ready some more + # If the parser has no buffer set/left + if not jv_parser_remaining(self._parser): + # Supply it with some bytes self._ready_next_bytes() - # Parse whatever we've readied, if any + # Get next value from the parser value = jv_parser_next(self._parser) if jv_is_valid(value): if self._packed: @@ -189,12 +190,10 @@ cdef class _JSONParser(object): message = jv_string_value(error_message).decode("utf8") jv_free(error_message) raise JSONParseError(message) - else: - jv_free(value) - # If we didn't ready any bytes - if self._bytes is None: - raise StopIteration - self._bytes = None + jv_free(value) + # If we supplied no bytes last time + if self._bytes is None: + raise StopIteration cdef bint _ready_next_bytes(self) except 1: cdef char* cbytes diff --git a/tests/jq_tests.py b/tests/jq_tests.py index 16b4011..ccd32d8 100644 --- a/tests/jq_tests.py +++ b/tests/jq_tests.py @@ -177,6 +177,58 @@ def unicode_strings_can_be_used_as_input(): ) +@istest +def record_separator_character_accepted_in_input(): + assert_equal( + [], + list(jq.compile(".").input(text='\x1e')) + ) + assert_equal( + [], + list(jq.compile(".").input(text='\x1e\x1e')) + ) + assert_equal( + [{}], + list(jq.compile(".").input(text='\x1e{}')) + ) + assert_equal( + [{}], + list(jq.compile(".").input(text='\x1e\x1e{}')) + ) + assert_equal( + [{}], + list(jq.compile(".").input(text='{}\x1e')) + ) + assert_equal( + [{}], + list(jq.compile(".").input(text='{}\x1e\x1e')) + ) + assert_equal( + [{}], + list(jq.compile(".").input(text='\x1e{}\x1e')) + ) + assert_equal( + [{},[]], + list(jq.compile(".").input(text='{}\x1e[]')) + ) + assert_equal( + [{},[]], + list(jq.compile(".").input(text='{}\x1e\x1e[]')) + ) + assert_equal( + [{},[]], + list(jq.compile(".").input(text='\x1e{}\x1e[]')) + ) + assert_equal( + [{},[]], + list(jq.compile(".").input(text='{}\x1e[]\x1e')) + ) + assert_equal( + [{},[]], + list(jq.compile(".").input(text='\x1e{}\x1e[]\x1e')) + ) + + @istest def unicode_strings_can_be_used_as_programs(): assert_equal( @@ -204,6 +256,21 @@ def parse_json_all_inputs_accepted(): assert_equal(True, next(jq.parse_json(text=b"true"))) assert_equal(True, next(jq.parse_json(text_iter=iter([b"true"])))) +@istest +def parse_json_record_separator_character_accepted(): + assert_equal([], list(jq.parse_json(text='\x1e'))) + assert_equal([], list(jq.parse_json(text='\x1e\x1e'))) + assert_equal([{}], list(jq.parse_json(text='\x1e{}'))) + assert_equal([{}], list(jq.parse_json(text='\x1e\x1e{}'))) + assert_equal([{}], list(jq.parse_json(text='{}\x1e'))) + assert_equal([{}], list(jq.parse_json(text='{}\x1e\x1e'))) + assert_equal([{}], list(jq.parse_json(text='\x1e{}\x1e'))) + assert_equal([{},[]], list(jq.parse_json(text='{}\x1e[]'))) + assert_equal([{},[]], list(jq.parse_json(text='{}\x1e\x1e[]'))) + assert_equal([{},[]], list(jq.parse_json(text='\x1e{}\x1e[]'))) + assert_equal([{},[]], list(jq.parse_json(text='{}\x1e[]\x1e'))) + assert_equal([{},[]], list(jq.parse_json(text='\x1e{}\x1e[]\x1e'))) + @istest def parse_json_file_works(): fp = io.StringIO('{"abc": "def"}')