Big switch to reentrant lexing and parsing.

* parser.l (YY_INPUT): Stop relying on removed yyin_stream;
refer to stream via yyextra.
(yyin_stream, lineno, errors, spec_file_str,
prepared_error_message): Global variables removed.
(yyget_column, yyset_column): Missing prototypes not generated by flex
in bison bridge mode have to be added by us to avoid
warning.
(yyerror): Takes parser and scanner as parameters.  Prepared error
message is now in the parser context.  Calls to other error handling
functions receive scanner context.
(yyerr): New function.
(yyerrorf, yyerrprepf): Takes scanner argument, chases extra data to
get to parser, and refers to parser variables instead of globals.
(num_esc): Scanner argument added.
(%option reentrant, %option bison-bridge, %option extra-type): New
flex options.
(grammar): yyscanner added everywhere.
(end_of_char): Takes scanner argument.
(parse_init): Removed references to yyin_stream and
prepared_error_message.
(parse_reset): Function renamed to open_txr_file. Returns
results via pointers instead of setting global variables.
(regex_parse, lisp_parse): Use reentrant parser interface.

* parser.y (yyerror): Prototype removed.
(yylex): Prototype moved after grammar, with new arguments.
(sym_helper, define_transform): Take scanner argument.
(make_expr): Takes parser argument.
(rlrec): New static function.
(rl): Function turned into macro.
(mkexp, symhlpr): New macros.
(%purse-parser, %parse-param, %lex-param): New Yacc options.
(grammar): Actions re-worked for reentrance. Parser and scanner
contexts are passed down to helper functions, in some cases
via the three new macros. The result of the parse is stored
in the syntax_tree member of the parser_t structure instead
of a global. The yylex function receives the scanner instance.
(get_spec): Function removed.
(parse): New function.

* parser.h (lineno, errors, yyin_stream, spec_file_str):
Declarations removed.
(parser_t): New struct.
(yyerr): New function declared.
(yyparse, yyerror, yyerrorf, end_of_regex, end_of_char,
yylex, yylex_destroy): Declarations updated.

1 files changed, 262 insertions, 260 deletions

diff --git a/parser.l b/parser.l
index 4edb90ee..550e1db9 100644
--- a/parser.l
+++ b/parser.l
@@ -52,25 +52,17 @@
 
 #define YY_INPUT(buf, result, max_size)           \
   do {                                            \
-    val c = get_byte(yyin_stream);                \
+    val c = get_byte(yyextra->stream);            \
     int n = 0;                                    \
     if (c)                                        \
       buf[n++] = (char) c_num(c);                 \
     result = n;                                   \
   } while (0)
 
-val yyin_stream;
-
-cnum lineno = 1;
 int opt_loglevel = 1;   /* 0 - quiet; 1 - normal; 2 - verbose */
 
-int errors;
-val spec_file_str;
-
 val form_to_ln_hash;
 
-static val prepared_error_message;
-
 #define FLEX_NUM_VERSION 10000*YY_FLEX_MAJOR_VERSION + \
                          100*YY_FLEX_MINOR_VERSION + \
                          YY_FLEX_SUBMINOR_VERSION
@@ -82,35 +74,48 @@ int yylex_destroy(void)
 }
 #endif
 
-void yyerror(const char *s)
+/* Missing prototypes not generated by flex. */
+int yyget_column(void *);
+void yyset_column (int  column_no , yyscan_t yyscanner);
+
+void yyerror(parser_t *parser, void *scanner, const char *s)
 {
-  yyerrorf(lit("~a"), string_utf8(s), nao);
-  if (prepared_error_message) {
-    yyerrorf(lit("~a"), prepared_error_message, nao);
-    prepared_error_message = nil;
+  yyerrorf(scanner, lit("~a"), string_utf8(s), nao);
+  if (parser->prepared_msg) {
+    yyerrorf(scanner, lit("~a"), parser->prepared_msg, nao);
+    parser->prepared_msg = nil;
   }
 }
 
-void yyerrorf(val fmt, ...)
+void yyerr(void *scanner, const char *s)
 {
+  yyerror(yyget_extra(scanner), scanner, s);
+}
+
+void yyerrorf(void *scanner, val fmt, ...)
+{
+  parser_t *parser = yyget_extra(scanner);
+
   if (opt_loglevel >= 1) {
     va_list vl;
     va_start (vl, fmt);
     format(std_error, lit("~a: (~a:~a): "), prog_string,
-           spec_file_str, num(lineno), nao);
+           parser->name, num(parser->lineno), nao);
     vformat(std_error, fmt, vl);
     put_char(chr('\n'), std_error);
     va_end (vl);
   }
-  errors++;
+  parser->errors++;
 }
 
-static void yyerrprepf(val fmt, ...)
+static void yyerrprepf(void *scanner, val fmt, ...)
 {
+  parser_t *parser = yyget_extra(scanner);
+
   if (opt_loglevel >= 1) {
     va_list vl;
     va_start (vl, fmt);
-    prepared_error_message = vformat_to_string(fmt, vl);
+    parser->prepared_msg = vformat_to_string(fmt, vl);
     va_end (vl);
   }
 }
@@ -137,26 +142,24 @@ static wchar_t char_esc(int letter)
   internal_error("unhandled escape character");
 }
 
-static wchar_t num_esc(char *num)
+static wchar_t num_esc(void *scn, char *num)
 {
   if (num[0] == 'x') {
     if (strlen(num) > 7)
-      yyerror("too many digits in hex character escape");
+      yyerror(yyget_extra(scn), scn, "too many digits in hex character escape");
     return strtol(num + 1, 0, 16);
   } else {
     if (num[0] == 'o')
       num++;
     if (strlen(num) > 8)
-      yyerror("too many digits in octal character escape");
+      yyerror(yyget_extra(scn), scn, "too many digits in octal character escape");
     return strtol(num, 0, 8);
   }
 }
 
 %}
 
-%option stack
-%option nounput
-%option noinput
+%option stack nounput noinput reentrant bison-bridge extra-type="parser_t *"
 
 SYM     [a-zA-Z0-9_]+
 SGN     [+\-]
@@ -208,72 +211,72 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 <SPECIAL,QSPECIAL,NESTED,BRACED>{NUM} {
   val str = string_own(utf8_dup_from(yytext));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = int_str(str, num(10));
+  yylval->val = int_str(str, num(10));
   return NUMBER;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>{XNUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = int_str(str, num(16));
+  yylval->val = int_str(str, num(16));
   return NUMBER;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>{ONUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = int_str(str, num(8));
+  yylval->val = int_str(str, num(8));
   return NUMBER;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>{BNUM} {
   val str = string_own(utf8_dup_from(yytext + 2));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = int_str(str, num(2));
+  yylval->val = int_str(str, num(2));
   return NUMBER;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>{FLO} {
   val str = string_own(utf8_dup_from(yytext));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = flo_str(str);
+  yylval->val = flo_str(str);
   return NUMBER;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>{FLODOT}/[^.] {
   val str = string_own(utf8_dup_from(yytext));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = flo_str(str);
+  yylval->val = flo_str(str);
   return NUMBER;
 }
 
@@ -282,301 +285,301 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 <NESTED>({FLO}|{FLODOT}){NTOK} {
   val str = string_utf8(yytext);
 
-  yyerrorf(lit("trailing junk in floating-point literal: ~a"), str, nao);
+  yyerrorf(yyscanner, lit("trailing junk in floating-point literal: ~a"), str, nao);
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.val = flo_str(str);
+  yylval->val = flo_str(str);
   return NUMBER;
 }
 
 <NESTED,QSILIT,QWLIT>@{NUM} {
   val str = string_own(utf8_dup_from(yytext + 1));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
-  yylval.val = int_str(str, num(10));
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
+  yylval->val = int_str(str, num(10));
   return METANUM;
 }
 
 <NESTED,QSILIT,QWLIT>@{XNUM} {
   val str = string_own(utf8_dup_from(yytext + 3));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
-  yylval.val = int_str(str, num(16));
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
+  yylval->val = int_str(str, num(16));
   return METANUM;
 }
 
 <NESTED,QSILIT,QWLIT>@{ONUM} {
   val str = string_own(utf8_dup_from(yytext + 3));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
-  yylval.val = int_str(str, num(8));
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
+  yylval->val = int_str(str, num(8));
   return METANUM;
 }
 
 <NESTED,QSILIT,QWLIT>@{BNUM} {
   val str = string_own(utf8_dup_from(yytext + 3));
 
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
-  yylval.val = int_str(str, num(2));
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
+  yylval->val = int_str(str, num(2));
   return METANUM;
 }
 
 <SPECIAL,QSPECIAL>{TOK} |
 <BRACED>{BTOK} |
 <NESTED>{NTOK} {
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
 
-  yylval.lexeme = utf8_dup_from(yytext);
+  yylval->lexeme = utf8_dup_from(yytext);
   return SYMTOK;
 }
 
 <SPECIAL>\({WS}all{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return ALL;
 }
 
 <SPECIAL>\({WS}some/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return SOME;
 }
 
 <SPECIAL>\({WS}none{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return NONE;
 }
 
 <SPECIAL>\({WS}maybe{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return MAYBE;
 }
 
 <SPECIAL>\({WS}cases{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return CASES;
 }
 
 <SPECIAL>\({WS}block/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return BLOCK;
 }
 
 <SPECIAL>\({WS}choose/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return CHOOSE;
 }
 
 <SPECIAL>\({WS}gather/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return GATHER;
 }
 
 <SPECIAL>\({WS}and{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return AND;
 }
 
 <SPECIAL>\({WS}or{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return OR;
 }
 
 <SPECIAL>\({WS}end{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return END;
 }
 
 <SPECIAL>\({WS}collect/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return COLLECT;
 }
 
 <SPECIAL>\({WS}coll/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return COLL;
 }
 
 <SPECIAL>\({WS}until{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return UNTIL;
 }
 
 <SPECIAL>\({WS}output/{ID_END}  {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return OUTPUT;
 }
 
 <SPECIAL>\({WS}repeat/{ID_END}  {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return REPEAT;
 }
 
 
 <SPECIAL>\({WS}rep/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return REP;
 }
 
 <SPECIAL>\({WS}single{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return SINGLE;
 }
 
 <SPECIAL>\({WS}first{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return FIRST;
 }
 
 <SPECIAL>\({WS}last{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return LAST;
 }
 
 <SPECIAL>\({WS}empty{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return EMPTY;
 }
 
 <SPECIAL>\({WS}mod/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return MOD;
 }
 
 <SPECIAL>\({WS}modlast/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return MODLAST;
 }
 
 <SPECIAL>\({WS}define/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return DEFINE;
 }
 
 <SPECIAL>\({WS}try{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return TRY;
 }
 
 <SPECIAL>\({WS}catch/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return CATCH;
 }
 
 <SPECIAL>\({WS}finally{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return FINALLY;
 }
 
 <SPECIAL>\({WS}if/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return IF;
 }
 
 <SPECIAL>\({WS}elif/{ID_END} {
-  yy_push_state(NESTED);
-  yylval.lineno = lineno;
+  yy_push_state(NESTED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return ELIF;
 }
 
 <SPECIAL>\({WS}else{WS}\) {
-  yy_pop_state();
-  yylval.lineno = lineno;
+  yy_pop_state(yyscanner);
+  yylval->lineno = yyextra->lineno;
   return ELSE;
 }
 
 <SPECIAL,QSPECIAL>[{] {
-  yy_push_state(BRACED);
-  yylval.lineno = lineno;
+  yy_push_state(BRACED, yyscanner);
+  yylval->lineno = yyextra->lineno;
   return yytext[0];
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>[(\[] {
- yy_push_state(NESTED);
- yylval.lineno = lineno;
+ yy_push_state(NESTED, yyscanner);
+ yylval->lineno = yyextra->lineno;
  return yytext[0];
 }
 
 <NESTED,BRACED>@ {
-  yylval.lineno = lineno;
+  yylval->lineno = yyextra->lineno;
   return yytext[0];
 }
 
 <NESTED,QSPECIAL,BRACED>,[*] {
-  yylval.chr = '*';
+  yylval->chr = '*';
   return SPLICE;
 }
 
 <NESTED>[,'^] {
-  yylval.chr = yytext[0];
+  yylval->chr = yytext[0];
   return yytext[0];
 }
 
 <QSPECIAL,BRACED>[,'] {
-  yylval.chr = yytext[0];
+  yylval->chr = yytext[0];
   return yytext[0];
 }
 
 <BRACED>[}] {
-  yy_pop_state();
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  yy_pop_state(yyscanner);
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
   return yytext[0];
 }
 
 <SPECIAL,QSPECIAL,NESTED>[)\]] {
-  yy_pop_state();
-  if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-    yy_pop_state();
+  yy_pop_state(yyscanner);
+  if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+    yy_pop_state(yyscanner);
   return yytext[0];
 }
 
@@ -585,42 +588,42 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>\" {
-  yy_push_state(STRLIT);
+  yy_push_state(STRLIT, yyscanner);
   return '"';
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#\\ {
-  yy_push_state(CHRLIT);
+  yy_push_state(CHRLIT, yyscanner);
   return HASH_BACKSLASH;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#[/] {
-  yy_push_state(REGEX);
+  yy_push_state(REGEX, yyscanner);
   return HASH_SLASH;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>` {
-  yy_push_state(QSILIT);
+  yy_push_state(QSILIT, yyscanner);
   return '`';
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#\" {
-  yy_push_state(WLIT);
+  yy_push_state(WLIT, yyscanner);
   return WORDS;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#\*\" {
-  yy_push_state(WLIT);
+  yy_push_state(WLIT, yyscanner);
   return WSPLICE;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#\` {
-  yy_push_state(QWLIT);
+  yy_push_state(QWLIT, yyscanner);
   return QWORDS;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>#\*\` {
-  yy_push_state(QWLIT);
+  yy_push_state(QWLIT, yyscanner);
   return QWSPLICE;
 }
 
@@ -629,61 +632,61 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <NESTED,BRACED>#H {
-  yylval.lineno = lineno;
+  yylval->lineno = yyextra->lineno;
   return HASH_H;
 }
 
 <NESTED>\.\. {
-  yylval.lineno = lineno;
+  yylval->lineno = yyextra->lineno;
   return DOTDOT;
 }
 
 <SPECIAL>@ {
-  yy_pop_state();
-  yylval.lexeme = chk_strdup(L"@");
+  yy_pop_state(yyscanner);
+  yylval->lexeme = chk_strdup(L"@");
   return TEXT;
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>\n {
-  lineno++;
+  yyextra->lineno++;
 }
 
 <SPECIAL,BRACED>[/] {
-  yy_push_state(REGEX);
+  yy_push_state(REGEX, yyscanner);
   return '/';
 }
 
 <SPECIAL,QSPECIAL,NESTED>\. {
-  yylval.chr = '.';
+  yylval->chr = '.';
   return '.';
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>[\\]\n{WS} {
   if (YYSTATE == SPECIAL)
-    yy_pop_state();  /* @\ continuation */
-  lineno++;
+    yy_pop_state(yyscanner);  /* @\ continuation */
+  yyextra->lineno++;
 }
 
 <SPECIAL>[\\][abtnvfre ] {
   wchar_t lexeme[2];
   lexeme[0] = char_esc(yytext[1]);
   lexeme[1] = 0;
-  yylval.lexeme = chk_strdup(lexeme);
-  yy_pop_state();
+  yylval->lexeme = chk_strdup(lexeme);
+  yy_pop_state(yyscanner);
   return TEXT;
 }
 
 <SPECIAL>[\\](x{HEX}+|{OCT}+) {
   wchar_t lexeme[2];
-  lexeme[0] = num_esc(yytext + 1);
+  lexeme[0] = num_esc(yyscanner, yytext + 1);
   lexeme[1] = 0;
-  yylval.lexeme = chk_strdup(lexeme);
-  yy_pop_state();
+  yylval->lexeme = chk_strdup(lexeme);
+  yy_pop_state(yyscanner);
   return TEXT;
 }
 
 <SPECIAL>[\\]. {
-  yyerrorf(lit("unrecognized escape: \\~a"), chr(yytext[1]), nao);
+  yyerrorf(yyscanner, lit("unrecognized escape: \\~a"), chr(yytext[1]), nao);
 }
 
 <SPECIAL,QSPECIAL,NESTED,BRACED>[;].* {
@@ -704,53 +707,53 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <REGEX>[/] {
-  yylval.chr = '/';
+  yylval->chr = '/';
   return '/';
 }
 
 
 <REGEX>[\\][abtnvfre\\ ] {
-  yylval.chr = char_esc(yytext[1]);
+  yylval->chr = char_esc(yytext[1]);
   return REGCHAR;
 }
 
 <REGEX>[\\](x{HEX}+|{OCT}+);? {
-  yylval.chr = num_esc(yytext + 1);
+  yylval->chr = num_esc(yyscanner, yytext + 1);
   return REGCHAR;
 }
 
 <REGEX>[\\][sSdDwW] {
-  yylval.chr = yytext[1];
+  yylval->chr = yytext[1];
   return REGTOKEN;
 }
 
 <REGEX>{WS}[\\]\n{WS} {
-  lineno++;
+  yyextra->lineno++;
 }
 
 <REGEX>\n {
-  lineno++;
+  yyextra->lineno++;
   yyerrprepf(lit("newline in regex"), nao);
   return ERRTOK;
 }
 
 <REGEX>[.*?+~&%] {
-  yylval.chr = yytext[0];
+  yylval->chr = yytext[0];
   return yytext[0];
 }
 
 <REGEX>[\[\]\-] {
-  yylval.chr = yytext[0];
+  yylval->chr = yytext[0];
   return yytext[0];
 }
 
 <REGEX>[()|] {
-  yylval.chr = yytext[0];
+  yylval->chr = yytext[0];
   return yytext[0];
 }
 
 <REGEX>[\\]. {
-  yylval.chr = yytext[1];
+  yylval->chr = yytext[1];
   return REGCHAR;
 }
 
@@ -762,7 +765,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 <REGEX>{UANYN}  {
   wchar_t buf[8];
   utf8_from(buf, yytext);
-  yylval.chr = buf[0];
+  yylval->chr = buf[0];
   return REGCHAR;
 }
 
@@ -773,43 +776,43 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <INITIAL>[ ]+ {
-  yylval.lexeme = utf8_dup_from(yytext);
+  yylval->lexeme = utf8_dup_from(yytext);
   return SPACE;
 }
 
 <INITIAL>({UONLY}|[^@\n ])+ {
-  yylval.lexeme = utf8_dup_from(yytext);
+  yylval->lexeme = utf8_dup_from(yytext);
   return TEXT;
 }
 
 <INITIAL>\n {
-  lineno++;
+  yyextra->lineno++;
   return '\n';
 }
 
 <INITIAL>@{WS}\* {
-  yy_push_state(SPECIAL);
+  yy_push_state(SPECIAL, yyscanner);
   return '*';
 }
 
 <INITIAL>@ {
-  yy_push_state(SPECIAL);
+  yy_push_state(SPECIAL, yyscanner);
 }
 
 <INITIAL>@\x01R {
-  yy_push_state(REGEX);
+  yy_push_state(REGEX, yyscanner);
   return SECRET_ESCAPE_R;
 }
 
 <INITIAL>@\x01E {
-  yy_push_state(SPECIAL);
-  yy_push_state(NESTED);
+  yy_push_state(SPECIAL, yyscanner);
+  yy_push_state(NESTED, yyscanner);
   return SECRET_ESCAPE_E;
 }
 
 <INITIAL>^@[#;].*\n {
   /* eat whole line comment */
-  lineno++;
+  yyextra->lineno++;
 }
 
 <INITIAL>@[#;].* {
@@ -817,76 +820,76 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 }
 
 <STRLIT,WLIT>\" {
-  yy_pop_state();
+  yy_pop_state(yyscanner);
   return yytext[0];
 }
 
 <QSILIT,QWLIT>\` {
-  yy_pop_state();
+  yy_pop_state(yyscanner);
   return yytext[0];
 }
 
 <STRLIT,QSILIT,WLIT,QWLIT>[\\][abtnvfre "`'\\ ] {
-  yylval.chr = char_esc(yytext[1]);
+  yylval->chr = char_esc(yytext[1]);
   return LITCHAR;
 }
 
 <STRLIT,QSILIT,WLIT,QWLIT>{WS}[\\]\n{WS} {
-  lineno++;
+  yyextra->lineno++;
 }
                                 
 <STRLIT,QSILIT,WLIT,QWLIT>[\\](x{HEX}+|{OCT}+);?  {
-  yylval.chr = num_esc(yytext+1);
+  yylval->chr = num_esc(yyscanner, yytext+1);
   return LITCHAR;
 }
 
 <STRLIT,QSILIT,WLIT,QWLIT>[\\]. {
-  yyerrorf(lit("unrecognized escape: \\~a"), chr(yytext[1]), nao);
+  yyerrorf(yyscanner, lit("unrecognized escape: \\~a"), chr(yytext[1]), nao);
 }
 
 <CHRLIT>(x{HEX}+|o{OCT}+) {
-  yylval.chr = num_esc(yytext);
+  yylval->chr = num_esc(yyscanner, yytext);
   return LITCHAR;
 }
 
 <CHRLIT>{SYM} {
-  yylval.lexeme = utf8_dup_from(yytext);
+  yylval->lexeme = utf8_dup_from(yytext);
   return SYMTOK;
 }
 
 <CHRLIT>[^ \t\n] {
-  yylval.lexeme = utf8_dup_from(yytext);
+  yylval->lexeme = utf8_dup_from(yytext);
   return SYMTOK; /* hack */
 }
 
 <STRLIT>\n {
   yyerrprepf(lit("newline in string literal"), nao);
-  lineno++;
-  yylval.chr = yytext[0];
+  yyextra->lineno++;
+  yylval->chr = yytext[0];
   return ERRTOK;
 }
 
 <CHRLIT>\n {
   yyerrprepf(lit("newline in character literal"), nao);
-  lineno++;
-  yylval.chr = yytext[0];
+  yyextra->lineno++;
+  yylval->chr = yytext[0];
   return ERRTOK;
 }
 
 <QSILIT>\n {
   yyerrprepf(lit("newline in string quasiliteral"), nao);
-  lineno++;
-  yylval.chr = yytext[0];
+  yyextra->lineno++;
+  yylval->chr = yytext[0];
   return ERRTOK;
 }
 
 <WLIT,QWLIT>\n {
-  lineno++;
+  yyextra->lineno++;
   return ' ';
 }
 
 <QSILIT,QWLIT>@ {
-  yy_push_state(QSPECIAL);
+  yy_push_state(QSPECIAL, yyscanner);
 }
 
 <WLIT,QWLIT>{WS} {
@@ -896,7 +899,7 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 <STRLIT,CHRLIT,QSILIT,WLIT,QWLIT>{UANYN} {
   wchar_t buf[8];
   utf8_from(buf, yytext);
-  yylval.chr = buf[0];
+  yylval->chr = buf[0];
   return LITCHAR;
 }
 
@@ -908,27 +911,31 @@ UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
 
 %%
 
-void end_of_regex(void)
+void end_of_regex(yyscan_t yyscanner)
 {
+  struct yyguts_t *yyg = (struct yyguts_t *) yyscanner;
+
   if (YYSTATE != REGEX)
     internal_error("end_of_regex called in wrong scanner state");
 
-  yy_pop_state();
+  yy_pop_state(yyscanner);
 
   if (YYSTATE != INITIAL) {
-    if (yy_top_state() == INITIAL
-      || yy_top_state() == QSILIT
-      || yy_top_state() == QWLIT)
-      yy_pop_state();
+    if (yy_top_state(yyscanner) == INITIAL
+      || yy_top_state(yyscanner) == QSILIT
+      || yy_top_state(yyscanner) == QWLIT)
+      yy_pop_state(yyscanner);
   }
 }
 
-void end_of_char(void)
+void end_of_char(yyscan_t yyscanner)
 {
+  struct yyguts_t *yyg = (struct yyguts_t *) yyscanner;
+
   if (YYSTATE != CHRLIT)
     internal_error("end_of_char called in wrong scanner state");
 
-  yy_pop_state();
+  yy_pop_state(yyscanner);
 }
 
 val source_loc(val form)
@@ -946,26 +953,22 @@ val source_loc_str(val form)
 
 void parse_init(void)
 {
-  protect(&yyin_stream, &prepared_error_message,
-          &form_to_ln_hash, (val *) 0);
-
+  prot1(&form_to_ln_hash);
   form_to_ln_hash = make_hash(t, nil, nil);
 }
 
-void parse_reset(val spec_file)
+void open_txr_file(val spec_file, val *name, val *stream)
 {
-  errors = 0;
-  lineno = 1;
-  spec_file_str = spec_file;
   {
-    FILE *in = w_fopen(c_str(spec_file_str), L"r");
+    FILE *in = w_fopen(c_str(spec_file), L"r");
     if (in == 0) {
-      spec_file_str = cat_str(list(spec_file_str, lit("txr"), nao), lit("."));
-      in = w_fopen(c_str(spec_file_str), L"r");
+      spec_file = cat_str(list(spec_file, lit("txr"), nao), lit("."));
+      in = w_fopen(c_str(spec_file), L"r");
       if (in == 0)
         uw_throwf(file_error_s, lit("unable to open ~a"), spec_file, nao);
     }
-    yyin_stream = make_stdio_stream(in, spec_file_str);
+    *stream = make_stdio_stream(in, spec_file);
+    *name = spec_file;
   }
 }
 
@@ -974,22 +977,22 @@ val regex_parse(val string, val error_stream)
   uses_or2;
   val parse_string = cat_str(list(lit("@\x01R"), string, nao), nil);
   val save_stream = std_error;
-  yyin_stream = make_string_byte_input_stream(parse_string);
-  errors = 0;
-  lineno = 1;
+  val stream = make_string_byte_input_stream(parse_string);
+  parser_t parser;
+
   error_stream = default_bool_arg(error_stream);
   std_error = if3(error_stream == t, std_output, or2(error_stream, std_null));
+
   {
     int gc = gc_state(0);
-    spec_file_str = if3(std_error != std_null,
-                        format(nil, lit("regex --> ~a"), string, nao),
-                        lit(""));
-    yyparse();
-    yylex_destroy();
+    val name = if3(std_error != std_null,
+                   format(nil, lit("regex --> ~a"), string, nao),
+                   lit(""));
+    parse(stream, name, &parser);
     gc_state(gc);
   }
   std_error = save_stream;
-  return errors ? nil : get_spec();
+  return parser.errors ? nil : parser.syntax_tree;
 }
 
 val lisp_parse(val source_in, val error_stream)
@@ -1004,18 +1007,17 @@ val lisp_parse(val source_in, val error_stream)
                  format(nil, lit("expr --> ~a"), source, nao),
                  stream_get_prop(input_stream, name_k));
   val save_stream = std_error;
-  yyin_stream = make_catenated_stream(list(secret_token_stream, input_stream, nao));
-  errors = 0;
-  lineno = 1;
+  val stream = make_catenated_stream(list(secret_token_stream, input_stream, nao));
+  parser_t parser;
+
   error_stream = default_bool_arg(error_stream);
   std_error = if3(error_stream == t, std_output, or2(error_stream, std_null));
   {
     int gc = gc_state(0);
-    spec_file_str = if3(std_error != std_null, name, lit(""));
-    yyparse();
-    yylex_destroy();
+    name = if3(std_error != std_null, name, lit(""));
+    parse(stream, name, &parser);
     gc_state(gc);
   }
   std_error = save_stream;
-  return errors ? nil : get_spec();
+  return parser.errors ? nil : parser.syntax_tree;
 }