Bugfix: lexer loses unmatched "hold char" between top-level forms.

Test case: file containing 4(prinl 3). Scanner consumes 4 and (.
The ( is lost when the scanner is reset for the next call to yyparse,
resulting in jut prinl being read and interpreted as a variable.

* parser.c (prime_parser): If present, append hold byte to priming
string. Takes parser_t * instead of parser, and returns void now.

* parser.l (reset_scanner): Now returns int value, the value
of the scanner's yy_hold_char variable which is nonzero when
the scanner is hanging on to an unmatched byte of input.

* parser.h (reset_scanner, prime_parser): Declarations updated.

* parser.y (parse): Pass hold byte returned by reset_scanner to
prime_parser.

1 files changed, 11 insertions, 6 deletions

diff --git a/parser.c b/parser.c
index aa85a7f4..97c8f44a 100644
--- a/parser.c
+++ b/parser.c
@@ -127,19 +127,24 @@ static val ensure_parser(val stream, val primer)
   return set(cdr_l(cell), parser(stream, one, primer));
 }
 
-val prime_parser(val parser)
+void prime_parser(parser_t *p, int hold_byte)
 {
-  parser_t *p = get_parser_impl(parser);
-  val secret_token_stream = make_string_byte_input_stream(lit("@\x01" "E"));
+  val secret_token_stream;
+
+  if (hold_byte) {
+      val secret_token_string = format(nil, lit("@\x01" "E~a"),
+                                       chr(hold_byte + 0xDC00), nao);
+      secret_token_stream = make_string_byte_input_stream(secret_token_string);
+  } else {
+    secret_token_stream = make_string_byte_input_stream(lit("@\x01" "E"));
+  }
 
   if (catenated_stream_p(p->stream)) {
     catenated_stream_push(secret_token_stream, p->stream);
   } else {
-    set(mkloc(p->stream, parser),
+    set(mkloc(p->stream, p->parser),
         make_catenated_stream(list(secret_token_stream, p->stream, nao)));
   }
-
-  return parser;
 }
 
 void open_txr_file(val spec_file, val *txr_lisp_p, val *name, val *stream)