/* Copyright 2011
 * Kaz Kylheku <kaz@kylheku.com>
 * Vancouver, Canada
 * All rights reserved.
 *
 * BSD License:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in
 *      the documentation and/or other materials provided with the
 *      distribution.
 *   3. The name of the author may not be used to endorse or promote
 *      products derived from this software without specific prior
 *      written permission.
 *
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <dirent.h>
#include <setjmp.h>
#include <stdarg.h>
#include <wchar.h>
#include "config.h"
#include "lib.h"
#include "gc.h"
#include "unwind.h"
#include "regex.h"
#include "stream.h"
#include "parser.h"
#include "txr.h"
#include "utf8.h"
#include "filter.h"
#include "hash.h"
#include "match.h"

int output_produced;

val decline_k, next_spec_k, repeat_spec_k;
val mingap_k, maxgap_k, gap_k, mintimes_k, maxtimes_k, times_k;
val lines_k, chars_k;
val choose_s, longest_k, shortest_k, greedy_k;
val vars_k;

static val h_directive_table, v_directive_table;

static void debugf(val fmt, ...)
{
  if (opt_loglevel >= 2) {
    va_list vl;
    va_start (vl, fmt);
    format(std_error, lit("~a: "), prog_string, nao);
    vformat(std_error, fmt, vl);
    put_char(std_error, chr('\n'));
    va_end (vl);
  }
}

static void debuglf(val line, val fmt, ...)
{
  if (opt_loglevel >= 2) {
    va_list vl;
    va_start (vl, fmt);
    format(std_error, lit("~a: (~a:~a) "), prog_string,
           spec_file_str, line, nao);
    vformat(std_error, fmt, vl);
    put_char(std_error, chr('\n'));
    va_end (vl);
  }
}

static void sem_error(val line, val fmt, ...)
{
  va_list vl;
  val stream = make_string_output_stream();

  va_start (vl, fmt);
  if (line)
    format(stream, lit("(~a:~a) "), spec_file_str, line, nao);
  (void) vformat(stream, fmt, vl);
  va_end (vl);

  uw_throw(query_error_s, get_string_from_stream(stream));
  abort();
}

static void file_err(val line, val fmt, ...)
{
  va_list vl;
  val stream = make_string_output_stream();

  va_start (vl, fmt);
  if (line)
    format(stream, lit("(~a:~a) "), spec_file_str, line, nao);
  (void) vformat(stream, fmt, vl);
  va_end (vl);

  uw_throw(file_error_s, get_string_from_stream(stream));
  abort();
}


static void dump_shell_string(const wchar_t *str)
{
  int ch;

  put_char(std_output, chr('"'));
  while ((ch = *str++) != 0) {
    switch (ch) {
    case '"': case '`': case '$': case '\\': case '\n':
      put_char(std_output, chr('\\'));
      /* fallthrough */
    default:
      put_char(std_output, chr(ch));
    }
  }
  put_char(std_output, chr('"'));
}

static void dump_byte_string(const char *str)
{
  while (*str)
    put_char(std_output, chr(*str++));
}


static void dump_var(val var, char *pfx1, size_t len1,
              char *pfx2, size_t len2, val value, int level)
{
  if (len1 >= 112 || len2 >= 112)
    internal_error("too much depth in bindings");

  if (listp(value)) {
    val iter;
    int i;
    size_t add1 = 0, add2 = 0;

    for (i = 0, iter = value; iter; iter = cdr(iter), i++) {
      if (level < opt_arraydims) {
        add2 = sprintf(pfx2 + len2, "[%d]", i);
        add1 = 0;
      } else {
        add1 = sprintf(pfx1 + len1, "_%d", i);
        add2 = 0;
      }

      dump_var(var, pfx1, len1 + add1, pfx2, len2 + add2, car(iter), level + 1);
    }
  } else {
    val ss = make_string_output_stream();
    val str;

    obj_pprint(value, ss);
    str = get_string_from_stream(ss);

    put_string(std_output, var);
    dump_byte_string(pfx1);
    dump_byte_string(pfx2);
    put_char(std_output, chr('='));
    dump_shell_string(c_str(str));
    put_char(std_output, chr('\n'));
  }
}

static void dump_bindings(val bindings)
{
  if (opt_loglevel >= 2) {
    put_line(std_error, lit("raw_bindings:"));
    dump(bindings, std_error);
  }

  while (bindings) {
    char pfx1[128], pfx2[128];
    val var = car(car(bindings));
    val value = cdr(car(bindings));
    *pfx1 = 0; *pfx2 = 0;
    dump_var(var, pfx1, 0, pfx2, 0, value, 0);
    bindings = cdr(bindings);
  }
}

static val depth(val obj)
{
  val dep = zero;

  if (obj == nil)
    return one;

  if (atom(obj))
    return zero;

  while (obj) {
    dep = max2(dep, depth(first(obj)));
    obj = rest(obj);
  }

  return plus(dep, one);
}

static val weird_merge(val left, val right)
{
  val left_depth = depth(left);
  val right_depth = depth(right);

  while (lt(left_depth, right_depth) || zerop(left_depth)) {
    left = cons(left, nil);
    left_depth = plus(left_depth, one);
  }

  while (lt(right_depth, left_depth) || zerop(right_depth)) {
    right = cons(right, nil);
    right_depth = plus(right_depth, one);
  }

  return append2(left, right);
}

static val bindable(val obj)
{
  return (obj && symbolp(obj) && obj != t && !keywordp(obj)) ? t : nil;
}

static val dest_bind(val linenum, val bindings, val pattern, val value)
{
  if (symbolp(pattern)) {
    if (bindable(pattern)) {
      val existing = assoc(bindings, pattern);
      if (existing) {
        if (tree_find(value, cdr(existing)))
          return bindings;
        if (tree_find(cdr(existing), value))
          return bindings;
        debugf(lit("bind variable mismatch: ~a"), pattern, nao);
        return t;
      }
      return cons(cons(pattern, value), bindings);
    } else {
      return equal(pattern, value) ? bindings : t;
    }
  } else if (consp(pattern)) {
    val piter = pattern, viter = value;

    if (first(pattern) == var_s) {
      uw_throwf(query_error_s, 
                lit("metavariable @~a syntax cannot be used here"),
                second(pattern), nao);
    }

    if (first(pattern) == expr_s) {
      uw_throwf(query_error_s, 
                lit("the @~s syntax cannot be used here"),
                rest(pattern), nao);
    }


    while (consp(piter) && consp(viter))
    {
      bindings = dest_bind(linenum, bindings, car(piter), car(viter));
      if (bindings == t)
        return t;
      piter = cdr(piter);
      viter = cdr(viter);
    }

    if (bindable(piter)) {
      bindings = dest_bind(linenum, bindings, piter, viter);
      if (bindings == t)
        return t;
    } else {
      return equal(piter, viter) ? bindings : t;
    }
    return bindings;
  } else if (tree_find(value, pattern)) {
    return bindings;
  }

  return t;
}

static val eval_form(val lineno, val form, val bindings);

static val vars_to_bindings(val lineno, val vars, val bindings)
{
  val iter;
  list_collect_decl (fixed_vars, tail);

  if (vars && !consp(vars)) 
    sem_error(lineno, lit("not a valid variable list: ~a"), vars, nao);

  for (iter = vars; iter; iter = cdr(iter)) {
    val item = car(iter);
    if (bindable(item)) {
      list_collect (tail, cons(item, nil));
    } else if (consp(item) && bindable(first(item))) {
      list_collect (tail, cons(first(item), 
                               cdr(eval_form(lineno, second(item), bindings))));
    } else { 
      sem_error(lineno, lit("not a variable spec: ~a"), item, nao);
    }
  }
  return fixed_vars;
}

typedef struct {
  val bindings, specline, dataline, pos, spec_lineno, data_lineno, file;
} match_line_ctx;

static match_line_ctx ml_all(val bindings, val specline, val dataline,
                             val pos, val spec_lineno, val data_lineno,
                             val file)
{
  match_line_ctx c = { bindings, specline, dataline, 
                       pos, spec_lineno, data_lineno, file };
  return c;
}

static match_line_ctx ml_specline(match_line_ctx c, val specline)
{
  match_line_ctx nc = c;
  nc.specline = specline;
  return nc;
}

static match_line_ctx ml_bindings_specline(match_line_ctx c, val bindings, val specline)
{
  match_line_ctx nc = c;
  nc.bindings = bindings;
  nc.specline = specline;
  return nc;
}

static val match_line(match_line_ctx c);

typedef val (*h_match_func)(match_line_ctx c, match_line_ctx *cout);

#define LOG_MISMATCH(KIND)                                              \
  debuglf(c.spec_lineno, lit(KIND " mismatch, position ~a (~a:~a)"),    \
          c.pos, c.file, c.data_lineno, nao);                           \
  debuglf(c.spec_lineno, lit("  ~a"), c.dataline, nao);                 \
  if (c_num(c.pos) < 77)                                                \
    debuglf(c.spec_lineno, lit("  ~*~a^"), c.pos, lit(""), nao)

#define LOG_MATCH(KIND, EXTENT)                                         \
  debuglf(c.spec_lineno, lit(KIND " matched, position ~a-~a (~a:~a)"),  \
          c.pos, EXTENT, c.file, c.data_lineno, nao);                   \
  debuglf(c.spec_lineno, lit("  ~a"), c.dataline, nao);                 \
  if (c_num(EXTENT) < 77)                                               \
    debuglf(c.spec_lineno, lit("  ~*~a~-*~a^"), c.pos, lit(""),         \
              minus(EXTENT, c.pos), lit("^"), nao)

#define elem_bind(elem_var, directive_var, specline)    \
  val elem_var = first(specline);                       \
  val directive_var = first(elem_var)

static val h_var(match_line_ctx c, match_line_ctx *cout)
{
  val elem = first(c.specline);
  val sym = second(elem);
  val pat = third(elem);
  val modifier = fourth(elem);
  val pair = assoc(c.bindings, sym); /* var exists already? */

  if (gt(length(modifier), one)) {
    sem_error(c.spec_lineno, lit("multiple modifiers on variable ~s"),
              sym, nao);
  }

  modifier = car(modifier);

  if (pair) {
    /* If the variable already has a binding, we replace
       it with its value, and treat it as a string match.
       The spec looks like ((var <sym> <pat>) ...)
       and it must be transformed into
       (<sym-substituted> <pat> ...) */
    if (pat) {
      c.specline = cons(cdr(pair), cons(pat, rest(c.specline)));
    } else if (nump(modifier)) {
      val past = plus(c.pos, modifier);

      if (length_str_lt(c.dataline, past) || lt(past, c.pos))
      {
        LOG_MISMATCH("fixed field size");
        return nil;
      }

      if (!tree_find(trim_str(sub_str(c.dataline, c.pos, past)),
                     cdr(pair)))
      {
        LOG_MISMATCH("fixed field contents");
        return nil;
      }

      LOG_MATCH("fixed field", past);
      c.pos = past;
      c.specline = cdr(c.specline);
    } else {
      c.specline = cons(cdr(pair), rest(c.specline));
    }
    goto repeat;
  } else if (consp(modifier)) { /* regex variable */
    val past = match_regex(c.dataline, car(modifier), c.pos);
    if (nullp(past)) {
      LOG_MISMATCH("var positive regex");
      return nil;
    }
    LOG_MATCH("var positive regex", past);
    c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, past));
    c.pos = past;
    /* This may have another variable attached */
    if (pat) {
      c.specline = cons(pat, rest(c.specline));
      goto repeat;
    }
  } else if (nump(modifier)) { /* fixed field */
    val past = plus(c.pos, modifier);
    if (length_str_lt(c.dataline, past) || lt(past, c.pos))
    {
      LOG_MISMATCH("count based var");
      return nil;
    }
    LOG_MATCH("count based var", past);
    c.bindings = acons(c.bindings, sym, trim_str(sub_str(c.dataline, c.pos, past)));
    c.pos = past;
    /* This may have another variable attached */
    if (pat) {
      c.specline = cons(pat, rest(c.specline));
      goto repeat;
    }
  } else if (modifier && modifier != t) {
    sem_error(c.spec_lineno, lit("invalid modifier ~s on variable ~s"),
              modifier, sym, nao);
  } else if (pat == nil) { /* no modifier, no elem -> to end of line */
    c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, nil));
    c.pos = length_str(c.dataline);
  } else if (type(pat) == STR) {
    val find = search_str(c.dataline, pat, c.pos, modifier);
    if (!find) {
      LOG_MISMATCH("var delimiting string");
      return nil;
    }
    LOG_MATCH("var delimiting string", find);
    c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, find));
    c.pos = plus(find, length_str(pat));
  } else if (consp(pat) && regexp(first(pat))) {
    val find = search_regex(c.dataline, first(pat), c.pos, modifier);
    val fpos = car(find);
    val flen = cdr(find);
    if (!find) {
      LOG_MISMATCH("var delimiting regex");
      return nil;
    }
    LOG_MATCH("var delimiting regex", fpos);
    c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, fpos));
    c.pos = plus(fpos, flen);
  } else if (consp(pat) && first(pat) == var_s) {
    /* Unbound var followed by var: the following one must either
       be bound, or must specify a regex. */
    val second_sym = second(pat);
    val next_pat = third(pat);
    val next_modifier = fourth(pat);
    val pair = assoc(c.bindings, second_sym); /* var exists already? */

    if (gt(length(next_modifier), one)) {
      sem_error(c.spec_lineno, lit("multiple modifiers on variable ~s"),
                second_sym, nao);
    }

    next_modifier = car(next_modifier);

    if (!pair && consp(next_modifier)) {
      val find = search_regex(c.dataline, first(next_modifier), c.pos, modifier);
      val fpos = car(find);
      val flen = cdr(find);

      if (!find) {
        LOG_MISMATCH("double var regex");
        return nil;
      }

      /* Text from here to start of regex match goes to this
         variable. */
      c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, fpos));
      /* Text from start of regex match to end goes to the
         second variable */
      c.bindings = acons(c.bindings, second_sym, sub_str(c.dataline, fpos, plus(fpos, flen)));
      LOG_MATCH("double var regex (first var)", fpos);
      c.pos = fpos;
      LOG_MATCH("double var regex (second var)", plus(fpos, flen));
      c.pos = plus(fpos, flen);
      if (next_pat) {
        c.specline = cons(next_pat, rest(c.specline));
        goto repeat;
      }
    } else if (!pair) {
      sem_error(c.spec_lineno, lit("consecutive unbound variables"), nao);
    } else {
    /* Re-generate a new spec with an edited version of
       the element we just processed, and repeat. */
      val new_elem = list(var_s, sym, cdr(pair), modifier, nao);

      if (next_pat)
         c.specline = cons(new_elem, cons(next_pat, rest(c.specline)));
      else
         c.specline = cons(new_elem, rest(c.specline));
      goto repeat;
    }
  } else if (consp(pat) && (consp(first(pat)) || stringp(first(pat)))) {
    cons_bind (find, len, search_str(c.dataline, pat, c.pos, modifier));
    if (!find) {
      LOG_MISMATCH("string");
      return nil;
    }
    c.bindings = acons(c.bindings, sym, sub_str(c.dataline, c.pos, find));
    c.pos = plus(find, len);
  } else {
    sem_error(c.spec_lineno,
              lit("variable followed by invalid element"), nao);
  }

  *cout = c;
  return next_spec_k;

repeat:
  *cout = c;
  return repeat_spec_k;
}

static val h_skip(match_line_ctx c, match_line_ctx *cout)
{
  val elem = first(c.specline);
  val max = second(elem);
  val min = third(elem);
  cnum cmax = nump(max) ? c_num(max) : 0;
  cnum cmin = nump(min) ? c_num(min) : 0;
  val greedy = eq(max, greedy_k);
  val last_good_result = nil, last_good_pos = nil;

  if (!rest(c.specline)) {
    debuglf(lit("skip to end of line ~a:~a"), c.file, c.data_lineno);
    return cons(c.bindings, t);
  }

  {
    cnum reps_max = 0, reps_min = 0;

    while (length_str_gt(c.dataline, c.pos) && min && reps_min < cmin) {
      c.pos = plus(c.pos, one);
      reps_min++;
    }

    if (min) {
      if (reps_min != cmin) {
        debuglf(c.spec_lineno,
                lit("skipped only ~a/~a chars to ~a:~a:~a"),
                num(reps_min), num(cmin),
                c.file, c.data_lineno, c.pos, nao);
        return nil;
      }

      debuglf(c.spec_lineno, lit("skipped ~a chars to ~a:~a:~a"),
              num(reps_min), c.file, c.data_lineno, c.pos, nao);
    }

    while (greedy || !max || reps_max++ < cmax) {
      val result = match_line(ml_specline(c, rest(c.specline)));

      if (result) {
        if (greedy) {
          last_good_result = result;
          last_good_pos = c.pos;
        } else {
          LOG_MATCH("skip", c.pos);
          return result;
        }
      }

      if (length_str_le(c.dataline, c.pos))  {
        if (last_good_result) {
          LOG_MATCH("greedy skip", last_good_pos);
          return last_good_result;
        }
        break;
      }

      c.pos = plus(c.pos, one);
    }
  }

  LOG_MISMATCH("skip");
  return nil;
}

static val h_coll(match_line_ctx c, match_line_ctx *cout)
{
  val elem = first(c.specline);
  val coll_specline = second(elem);
  val until_last_specline = third(elem);
  val args = fourth(elem);
  val bindings_coll = nil;
  val last_bindings = nil;
  val max = getplist(args, maxgap_k);
  val min = getplist(args, mingap_k);
  val gap = getplist(args, gap_k);
  val times = getplist(args, times_k);
  val mintimes = getplist(args, mintimes_k);
  val maxtimes = getplist(args, maxtimes_k);
  val chars = getplist(args, chars_k);
  val vars = getplist(args, vars_k);
  cnum cmax = nump(gap) ? c_num(gap) : (nump(max) ? c_num(max) : 0);
  cnum cmin = nump(gap) ? c_num(gap) : (nump(min) ? c_num(min) : 0);
  cnum mincounter = cmin, maxcounter = 0;
  cnum ctimax = nump(times) ? c_num(times) 
                            : (nump(maxtimes) ? c_num(maxtimes) : 0);
  cnum ctimin = nump(times) ? c_num(times) 
                            : (nump(mintimes) ? c_num(mintimes) : 0);
  cnum cchars = nump(chars) ? c_num(chars) : 0;
  cnum timescounter = 0, charscounter = 0;
  val iter;

  vars = vars_to_bindings(c.spec_lineno, vars, c.bindings);

  if (((times || maxtimes) && ctimax == 0) || (chars && cchars == 0)) {
    *cout = c;
    return next_spec_k;
  }

  for (;;) {
    val new_bindings = nil, new_pos = nil;

    if ((gap || min) && mincounter < cmin)
      goto next_coll;

    if (chars && charscounter++ >= cchars)
      break;

    {
      cons_set (new_bindings, new_pos,
                match_line(ml_specline(c, coll_specline)));

      if (until_last_specline) {
        cons_bind (sym, spec, until_last_specline);
        cons_bind (until_last_bindings, until_pos,
                   match_line(ml_bindings_specline(c, new_bindings, spec)));

        if (until_pos) {
          LOG_MATCH("until/last", until_pos);
          if (sym == last_s) {
            last_bindings = set_diff(until_last_bindings,
                                     new_bindings, eq_f, nil);
            c.pos = until_pos;
          }
          break;
        } else {
          LOG_MISMATCH("until/last");
        }
      }

      if (new_pos) {
        val strictly_new_bindings = set_diff(new_bindings,
                                             c.bindings, eq_f, nil);
        LOG_MATCH("coll", new_pos);

        for (iter = vars; iter; iter = cdr(iter)) {
          cons_bind (var, dfl, car(iter));
          val exists = assoc(new_bindings, var);

          if (!exists) {
            if (!dfl) 
              sem_error(c.spec_lineno, lit("coll failed to bind ~a"),
                        var, nao);
            else
              strictly_new_bindings = acons(strictly_new_bindings, 
                                            var, dfl);
          }
        }

        for (iter = strictly_new_bindings; iter; iter = cdr(iter))
        {
          val binding = car(iter);
          val vars_binding = assoc(vars, car(binding));

          if (!vars || vars_binding) {
            val existing = assoc(bindings_coll, car(binding));
            bindings_coll = acons_new(bindings_coll, car(binding),
                                      cons(cdr(binding), cdr(existing)));
          }
        }
      }

      if (new_pos && !equal(new_pos, c.pos)) {
        c.pos = new_pos;
        bug_unless (length_str_ge(c.dataline, c.pos));

        timescounter++;

        if ((times || maxtimes) && timescounter >= ctimax)
          break;

        mincounter = 0;
        maxcounter = 0;
      } else {
next_coll:
        mincounter++;
        if ((gap || max) && ++maxcounter > cmax)
          break;
        c.pos = plus(c.pos, one);
      }

      if (length_str_le(c.dataline, c.pos))
        break;
    }
  }

  if ((times || mintimes) && timescounter < ctimin) {
    debuglf(c.spec_lineno, lit("fewer than ~a iterations collected"),
            num(ctimin), nao);
    return nil;
  }

  if (!bindings_coll)
    debuglf(c.spec_lineno, lit("nothing was collected"), nao);

  for (iter = bindings_coll; iter; iter = cdr(iter)) {
    val pair = car(iter);
    val rev = cons(car(pair), nreverse(cdr(pair)));
    c.bindings = cons(rev, c.bindings);
  }

  if (last_bindings) {
    c.bindings = set_diff(c.bindings, last_bindings, eq_f, car_f);
    c.bindings = nappend2(last_bindings, c.bindings);
  }

  *cout = c;
  return next_spec_k;
}

static val h_parallel(match_line_ctx c, match_line_ctx *cout)
{
  elem_bind(elem, directive, c.specline);
  val specs = third(elem);
  val plist = fourth(elem);
  val all_match = t;
  val some_match = nil;
  val max_pos = c.pos;
  val choose_shortest = getplist(plist, shortest_k);
  val choose_longest = getplist(plist, longest_k);
  val choose_sym = or2(choose_longest, choose_shortest);
  val choose_bindings = c.bindings, choose_pos = c.pos;
  val choose_minmax = choose_longest ? num(-1) : num(NUM_MAX);
  val iter;

  if (choose_longest && choose_shortest)
    sem_error(c.spec_lineno, lit("choose: both :shortest and :longest specified"), nao);

  if (directive == choose_s && !choose_sym)
    sem_error(c.spec_lineno, lit("choose: criterion not specified"), nao);
  for (iter = specs; iter != nil; iter = cdr(iter)) {
    val nested_spec = first(iter);
    cons_bind (new_bindings, new_pos,
               match_line(ml_specline(c, nested_spec)));

    if (new_pos) {
      some_match = t;
      if (gt(new_pos, max_pos))
        max_pos = new_pos;
      if (directive == choose_s) {
        val binding = choose_sym ? assoc(new_bindings, choose_sym) : nil;
        val value = cdr(binding);

        if (value) {
          val len = length_str(value);

          if ((choose_longest && gt(len, choose_minmax)) ||
              (choose_shortest && lt(len, choose_minmax)))
          {
            choose_minmax = len;
            choose_bindings = new_bindings;
            choose_pos = new_pos;
          }
        }
      } else {
        c.bindings = new_bindings;
      }
      if (directive == cases_s || directive == none_s)
        break;
    } else {
      all_match = nil;
      if (directive == all_s)
        break;
    }
  }

  if (directive == all_s && !all_match) {
    debuglf(c.spec_lineno, lit("all: some clauses didn't match"), nao);
    return nil;
  }

  if ((directive == some_s || directive == cases_s) && !some_match) {
    debuglf(c.spec_lineno, lit("some/cases: no clauses matched"), nao);
    return nil;
  }

  if (directive == none_s && some_match) {
    debuglf(c.spec_lineno, lit("none: some clauses matched"), nao);
    return nil;
  }

  /* No check for maybe, since it always succeeds. */

  if (directive == choose_s) {
    c.bindings = choose_bindings;
    c.pos = choose_pos;
  } else {
    c.pos = max_pos;
  }

  *cout = c;
  return next_spec_k;
}

static val h_trailer(match_line_ctx c, match_line_ctx *cout)
{
  val result = match_line(ml_specline(c, rest(c.specline)));
  val new_pos = cdr(result);

  if (!new_pos) {
    LOG_MISMATCH("trailer");
    return nil;
  }

  LOG_MATCH("trailer", new_pos);
  return cons(c.bindings, c.pos);
}

static val h_eol(match_line_ctx c, match_line_ctx *cout)
{
  if (length_str_le(c.dataline, c.pos)) {
    LOG_MATCH("eol", c.pos);
    return cons(c.bindings, t);
  }
  LOG_MISMATCH("eol");
  return nil;
}

static val match_line(match_line_ctx c)
{
  for (;;) {
    val elem;

    if (c.specline == nil)
      break;

    elem = first(c.specline);

    switch (elem ? type(elem) : 0) {
    case CONS: /* directive */
      {
        val directive = first(elem);
        val entry = gethash(h_directive_table, directive);

        if (entry) {
          h_match_func hmf = (h_match_func) cptr_get(entry);
          match_line_ctx nc;
          val result = hmf(c, &nc);

          if (result == next_spec_k) {
            c = nc;
            break;
          } else if (result == repeat_spec_k) {
            c = nc;
            continue;
          } else {
            return result;
          }
        } else if (regexp(directive)) {
          val past = match_regex(c.dataline, directive, c.pos);
          if (nullp(past)) {
            LOG_MISMATCH("regex");
            return nil;
          }
          LOG_MATCH("regex", past);
          c.pos = past;
        } else if (consp(directive) || stringp(directive)) {
          cons_bind (find, len, search_str_tree(c.dataline, elem, c.pos, nil));
          val newpos;

          if (find == nil || !equal(find, c.pos)) {
            LOG_MISMATCH("string tree");
            return nil;
          }

          newpos = plus(find, len);
          LOG_MATCH("string tree", newpos);
          c.pos = newpos;
        } else {
          sem_error(c.spec_lineno, lit("unknown directive: ~a"), directive, nao);
        }
      }
      break;
    case STR:
      {
        val find = search_str(c.dataline, elem, c.pos, nil);
        val newpos;
        if (find == nil || !equal(find, c.pos)) {
          LOG_MISMATCH("string");
          return nil;
        }
        newpos = plus(find, length_str(elem));
        LOG_MATCH("string", newpos);
        c.pos = newpos;
        break;
      }
    default:
      sem_error(c.spec_lineno, lit("unsupported object in spec: ~s"), elem, nao);
    }

    c.specline = cdr(c.specline);
  }

  return cons(c.bindings, c.pos);
}

static val format_field(val string_or_list, val modifier, val filter)
{
  val n = zero;
  val plist = nil;

  if (!stringp(string_or_list))
    return string_or_list;

  for (; modifier; pop(&modifier)) {
    val item = first(modifier);
    if (nump(item))
      n = item;
    if (regexp(item))
      uw_throw(query_error_s, lit("format_field: regex modifier in output"));
    if (keywordp(item)) {
      plist = modifier;
      break;
    }
  }

  {
    val filter_sym = getplist(plist, filter_k);

    if (filter_sym) {
      filter = get_filter_trie(filter_sym);

      if (!filter) {
        uw_throwf(query_error_s, lit("format_field: filter ~s not known"),
                  filter_sym, nao);
      }
    }

    if (filter)
      string_or_list = filter_string(filter, cat_str(list(string_or_list, nao),
                                                     nil));
  }

  {
    val right = lt(n, zero);
    val width = if3(lt(n, zero), neg(n), n);
    val diff = minus(width, length_str(string_or_list));

    if (le(diff, zero))
      return string_or_list;

    if (ge(length_str(string_or_list), width))
      return string_or_list;

    {
      val padding = mkstring(diff, chr(' '));

      return if3(right,
                 cat_str(list(padding, string_or_list, nao), nil),
                 cat_str(list(string_or_list, padding, nao), nil));
    }
  }
}

static val subst_vars(val spec, val bindings, val filter)
{
  list_collect_decl(out, iter);

  while (spec) {
    val elem = first(spec);

    if (consp(elem)) {
      if (first(elem) == var_s) {
        val sym = second(elem);
        val pat = third(elem);
        val modifiers = fourth(elem);
        val pair = assoc(bindings, sym);

        if (pair) {
          if (pat)
            spec = cons(filter_string(filter, cdr(pair)), cons(pat, rest(spec)));
          else if (modifiers)
            spec = cons(format_field(cdr(pair), modifiers, filter), rest(spec));
          else
            spec = cons(filter_string(filter, cdr(pair)), rest(spec));
          continue;
        }
        uw_throwf(query_error_s, lit("unbound variable ~a"),
                                 sym, nao);
      } else if (first(elem) == quasi_s) {
        val nested = subst_vars(rest(elem), bindings, filter);
        list_collect_append(iter, nested);
        spec = cdr(spec);
        continue;
      } else {
        val nested = subst_vars(elem, bindings, filter);
        list_collect_append(iter, nested);
        spec = cdr(spec);
        continue;
      }
    }

    list_collect(iter, elem);
    spec = cdr(spec);
  }

  return out;
}

static val eval_form(val lineno, val form, val bindings)
{
  val ret = nil;

  uw_catch_begin (cons(query_error_s, nil), exc_sym, exc);
  {
    if (!form) {
      ret = cons(t, form);
    } else if (bindable(form)) {
      ret = assoc(bindings, form);
    } else if (consp(form)) {
      if (first(form) == quasi_s) {
        ret = cons(t, cat_str(subst_vars(rest(form), bindings, nil), nil));
      } else if (regexp(car(form))) {
        ret = cons(t, form);
      } else if (first(form) == var_s) {
        sem_error(lineno, lit("metavariable @~a syntax cannot be used here"),
                  second(form), nao);
      } else if (first(form) == expr_s) {
        sem_error(lineno, lit("the @~s syntax cannot be used here"),
                  rest(form), nao);
      } else {
        val subforms = mapcar(curry_123_2(func_n3(eval_form), 
                                          lineno, bindings), form);

        if (all_satisfy(subforms, identity_f, nil))
          ret = cons(t, mapcar(func_n1(cdr), subforms));
      }
    } else if (stringp(form)) {
      ret = cons(t, form);
    } else {
      ret = cons(t, form);
    }

    uw_catch (exc_sym, exc) {
      if (stringp(exc) && !equal(exc, lit("")) && 
          chr_str(exc, zero) == chr('('))
      {
        uw_throw (exc_sym, exc);
      }

      sem_error(lineno, lit("~a"), exc, nao);
    }
  }
  uw_catch_end;

  if (!ret)
    sem_error(lineno, lit("unbound variable in form ~s"), form, nao);

  return ret;
}

enum fpip_close { fpip_fclose, fpip_pclose, fpip_closedir };

typedef struct fpip {
  FILE *f;
  DIR *d;
  enum fpip_close close;
} fpip_t;

static fpip_t complex_open(val name, val output)
{
  fpip_t ret = { 0, 0 };

  const wchar_t *namestr = c_str(name);
  cnum len = c_num(length_str(name));

  if (len == 0)
    return ret;

  if (!wcscmp(namestr, L"-")) {
    ret.close = fpip_fclose;
    ret.f = output ? stdout : stdin;
    output_produced = output ? 1 : 0;
  } else if (namestr[0] == '!') {
    ret.close = fpip_pclose;
    ret.f = w_popen(namestr+1, output ? L"w" : L"r");
  } else if (namestr[0] == '$') {
    char *name;
    if (output)
      return ret;
    name = (char *) utf8_dup_to(namestr+1);
    ret.close = fpip_closedir;
    ret.d = opendir(name);
    free(name);
  } else {
    ret.close = fpip_fclose;
    ret.f = w_fopen(namestr, output ? L"w" : L"r");
  }

  return ret;
}

static int complex_open_failed(fpip_t fp)
{
  return fp.f == 0 && fp.d == 0;
}

static val complex_snarf(fpip_t fp, val name)
{
  switch (fp.close) {
  case fpip_fclose:
    return lazy_stream_cons(make_stdio_stream(fp.f, name, t, nil));
  case fpip_pclose:
    return lazy_stream_cons(make_pipe_stream(fp.f, name, t, nil));
  case fpip_closedir:
    return lazy_stream_cons(make_dir_stream(fp.d));
  }

  internal_error("bad input source type");
}

static val complex_stream(fpip_t fp, val name)
{
  switch (fp.close) {
  case fpip_fclose:
    return make_stdio_stream(fp.f, name, t, nil);
  case fpip_pclose:
    return make_pipe_stream(fp.f, name, t, nil);
  case fpip_closedir:
    uw_throwf(query_error_s, lit("cannot output to directory: ~a"), name, nao);
  }

  internal_error("bad input source type");
}


static val robust_length(val obj)
{
  if (obj == nil)
    return zero;
  if (atom(obj))
    return negone;
  return length(obj);
}

static val bind_car(val bind_cons)
{
  return if3(consp(cdr(bind_cons)),
               cons(car(bind_cons), car(cdr(bind_cons))),
               bind_cons);
}

static val bind_cdr(val bind_cons)
{
  return if3(consp(cdr(bind_cons)),
               cons(car(bind_cons), cdr(cdr(bind_cons))),
               bind_cons);
}

static val extract_vars(val output_spec)
{
  list_collect_decl (vars, tai);

  if (consp(output_spec)) {
    if (first(output_spec) == var_s) {
      list_collect (tai, second(output_spec));
    } else {
      for (; output_spec; output_spec = cdr(output_spec))
        list_collect_nconc(tai, extract_vars(car(output_spec)));
    }
  }

  return vars;
}

static val extract_bindings(val bindings, val output_spec)
{
  list_collect_decl (bindings_out, tail);
  val var_list = extract_vars(output_spec);

  for (; bindings; bindings = cdr(bindings))
    if (memq(car(car(bindings)), var_list))
      list_collect(tail, car(bindings));

  return bindings_out;
}

static void do_output_line(val bindings, val specline,
                           val spec_lineno, val filter, val out)
{
  for (; specline; specline = rest(specline)) {
    val elem = first(specline);

    switch (elem ? type(elem) : 0) {
    case CONS:
      {
        val directive = first(elem);

        if (directive == var_s) {
          val str = cat_str(subst_vars(cons(elem, nil),
                                       bindings, filter), nil);
          if (str == nil)
            sem_error(spec_lineno, lit("bad substitution: ~a"),
                      second(elem), nao);
          put_string(out, str);
        } else if (directive == rep_s) {
          val main_clauses = second(elem);
          val single_clauses = third(elem);
          val first_clauses = fourth(elem);
          val last_clauses = fifth(elem);
          val empty_clauses = sixth(elem);
          val bind_cp = extract_bindings(bindings, elem);
          val max_depth = reduce_left(func_n2(max2),
                                      bind_cp, zero,
                                      chain(func_n1(cdr),
                                            func_n1(robust_length),
                                            nao));

          if (equal(max_depth, zero) && empty_clauses) {
            do_output_line(bindings, empty_clauses, spec_lineno, filter, out);
          } else if (equal(max_depth, one) && single_clauses) {
            val bind_a = mapcar(func_n1(bind_car), bind_cp);
            do_output_line(bind_a, single_clauses, spec_lineno, filter, out);
          } else if (!zerop(max_depth)) {
            cnum i;

            for (i = 0; i < c_num(max_depth); i++) {
              val bind_a = mapcar(func_n1(bind_car), bind_cp);
              val bind_d = mapcar(func_n1(bind_cdr), bind_cp);

              if (i == 0 && first_clauses) {
                do_output_line(bind_a, first_clauses, spec_lineno, filter, out);
              } else if (i == c_num(max_depth) - 1 && last_clauses) {
                do_output_line(bind_a, last_clauses, spec_lineno, filter, out);
              } else {
                do_output_line(bind_a, main_clauses, spec_lineno, filter, out);
              }

              bind_cp = bind_d;
            }
          }

        } else {
          sem_error(spec_lineno, lit("unknown directive: ~a"), directive, nao);
        }
      }
      break;
    case STR:
      put_string(out, elem);
      break;
    case 0:
      break;
    default:
      sem_error(spec_lineno,
                lit("unsupported object in output spec: ~s"), elem, nao);
    }
  }
}

static void do_output(val bindings, val specs, val filter, val out)
{
  if (equal(specs, null_list))
    return;

  for (; specs; specs = cdr(specs)) {
    cons_bind (spec_lineno, specline, first(specs));
    val first_elem = first(specline);

    if (consp(first_elem)) {
      val sym = first(first_elem);

      if (sym == repeat_s) {
        val main_clauses = second(first_elem);
        val single_clauses = third(first_elem);
        val first_clauses = fourth(first_elem);
        val last_clauses = fifth(first_elem);
        val empty_clauses = sixth(first_elem);
        val bind_cp = extract_bindings(bindings, first_elem);
        val max_depth = reduce_left(func_n2(max2),
                                    bind_cp, zero,
                                    chain(func_n1(cdr),
                                          func_n1(robust_length),
                                          nao));

        if (equal(max_depth, zero) && empty_clauses) {
          do_output(bind_cp, empty_clauses, filter, out);
        } else if (equal(max_depth, one) && single_clauses) {
          val bind_a = mapcar(func_n1(bind_car), bind_cp);
          do_output(bind_a, single_clauses, filter, out);
        } else if (!zerop(max_depth)) {
          cnum i;

          for (i = 0; i < c_num(max_depth); i++) {
            val bind_a = mapcar(func_n1(bind_car), bind_cp);
            val bind_d = mapcar(func_n1(bind_cdr), bind_cp);

            if (i == 0 && first_clauses) {
              do_output(bind_a, first_clauses, filter, out);
            } else if (i == c_num(max_depth) - 1 && last_clauses) {
              do_output(bind_a, last_clauses, filter, out);
            } else {
              do_output(bind_a, main_clauses, filter, out);
            }

            bind_cp = bind_d;
          }
        }
        continue;
      }
    }

    do_output_line(bindings, specline, spec_lineno, filter, out);
    put_char(out, chr('\n'));
  }
}

typedef struct {
  val spec, files, bindings, data, data_lineno;
} match_files_ctx;

static match_files_ctx mf_all(val spec, val files, val bindings,
                              val data, val data_lineno)
{
  match_files_ctx c = { spec, files, bindings, data, data_lineno };
  return c;
}

static match_files_ctx mf_args(match_files_ctx c)
{
  match_files_ctx nc = c;
  nc.files = cons(string(L"args"), c.files);
  nc.data = c.files;
  nc.data_lineno = num(1);
  return nc;
}

static match_files_ctx mf_data(match_files_ctx c, val data, val data_lineno)
{
  match_files_ctx nc = c;
  nc.data = data;
  nc.data_lineno = data_lineno;
  return nc;
}

static match_files_ctx mf_spec(match_files_ctx c, val spec)
{
  match_files_ctx nc = c;
  nc.spec = spec;
  return nc;
}

static match_files_ctx mf_spec_bindings(match_files_ctx c, val spec,
                                        val bindings)
{
  match_files_ctx nc = c;
  nc.spec = spec;
  nc.bindings = bindings;
  return nc;
}

static val match_files(match_files_ctx a);

typedef val (*v_match_func)(match_files_ctx c, match_files_ctx *cout);

#define spec_bind(specline_var, spec_linenum_var, first_spec_var, spec)  \
  val s_p_ ## specline_var = first(spec);                                \
  val specline_var = rest(s_p_ ## specline_var);                         \
  val spec_linenum_var = first(s_p_ ## specline_var);                    \
  val first_spec_var = first(specline_var)

static val v_skip(match_files_ctx c, match_files_ctx *cout)
{
  val specline = rest(first(c.spec));

  if (rest(specline))
    return decline_k;

  c.spec = rest(c.spec);

  if (!c.spec)
    return cons(c.bindings, cons(c.data, c.data_lineno));

  {
    val spec_linenum = first(first(c.spec));
    val first_spec = first(specline);
    val args = rest(first_spec);
    val max = first(args);
    val min = second(args);
    cnum cmax = nump(max) ? c_num(max) : 0;
    cnum cmin = nump(min) ? c_num(min) : 0;
    val greedy = eq(max, greedy_k);
    val last_good_result = nil;
    val last_good_line = num(0);

    {
      cnum reps_max = 0, reps_min = 0;
      uw_block_begin(nil, result);

      while (c.data && min && reps_min < cmin) {
        c.data = rest(c.data);
        c.data_lineno = plus(c.data_lineno, num(1));
        reps_min++;
      }

      if (min) {
        if (reps_min != cmin) {
          debuglf(spec_linenum, lit("skipped only ~a/~a lines to ~a:~a"),
                  num(reps_min), num(cmin),
                  first(c.files), c.data_lineno, nao);
          uw_block_return(nil, nil);
        }

        debuglf(spec_linenum, lit("skipped ~a lines to ~a:~a"),
                num(reps_min), first(c.files),
                c.data_lineno, nao);
      }

      while (greedy || !max || reps_max++ < cmax) {
        result = match_files(c);

        if (result) {
          if (greedy) {
            last_good_result = result;
            last_good_line = c.data_lineno;
          } else {
            debuglf(spec_linenum, lit("skip matched ~a:~a"), first(c.files),
                    c.data_lineno, nao);
            break;
          }
        } else {
          debuglf(spec_linenum, lit("skip didn't match ~a:~a"),
                  first(c.files), c.data_lineno, nao);
        }

        if (!c.data)
          break;

        debuglf(spec_linenum, lit("skip didn't match ~a:~a"), first(c.files),
                c.data_lineno, nao);

        c.data = rest(c.data);
        c.data_lineno = plus(c.data_lineno, num(1));
      }

      uw_block_end;

      if (result)
        return result;
      if (last_good_result) {
        debuglf(spec_linenum, lit("greedy skip matched ~a:~a"), 
                first(c.files), last_good_line, nao);
        return last_good_result;
      }
    }

    debuglf(spec_linenum, lit("skip failed"), nao);
    return nil;
  }
}

static val v_trailer(match_files_ctx c, match_files_ctx *cout)
{
  if (rest(rest(first(c.spec))))
    return decline_k;

  c.spec = rest(c.spec);

  if (!c.spec)  {
    return cons(c.bindings, cons(c.data, c.data_lineno));
  } else {
    cons_bind (new_bindings, success, match_files(c));
    return success ? cons(new_bindings, cons(c.data, c.data_lineno)) : nil;
  }
}

static val v_freeform(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);

  val args = rest(first_spec);
  val vals = mapcar(func_n1(cdr),
                       mapcar(curry_123_2(func_n3(eval_form),
                                          spec_linenum, c.bindings), args));

  if ((c.spec = rest(c.spec)) == nil) {
    sem_error(spec_linenum,
              lit("freeform must be followed by a query line"), nao);
  } else {
    val limit = or2(if2(nump(first(vals)), first(vals)),
                    if2(nump(second(vals)), second(vals)));
    val term = or2(if2(stringp(first(vals)), first(vals)),
                   if2(stringp(second(vals)), second(vals)));
    val ff_specline = rest(first(c.spec));
    val ff_dataline = lazy_str(c.data, term, limit);

    cons_bind (new_bindings, success,
               match_line(ml_all(c.bindings, ff_specline, ff_dataline, zero,
                                 spec_linenum, c.data_lineno, first(c.files))));

    if (!success) {
      debuglf(spec_linenum, lit("freeform match failure"), nao);
      return nil;
    }

    if (nump(success)) {
      c.data = lazy_str_get_trailing_list(ff_dataline, success);
      c.data_lineno = plus(c.data_lineno, num(1));
    }

    c.bindings = new_bindings;
  }

  *cout = c;
  return next_spec_k;
}

static val v_block(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);

  val name = first(rest(first_spec));

  if (rest(specline))
    sem_error(spec_linenum,
              lit("unexpected material after block directive"), nao);

  if (rest(c.spec))
  {
    uw_block_begin(name, result);
    result = match_files(c);
    uw_block_end;
    return result;
  }

  *cout = c;
  return next_spec_k;
}

static val v_accept_fail(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val sym = first(first_spec);
  val target = first(rest(first_spec));

  if (rest(specline))
    sem_error(spec_linenum, lit("unexpected material after ~a"), sym, nao);

  uw_block_return(target,
                  if2(sym == accept_s,
                      cons(c.bindings,
                           if3(c.data, cons(c.data, c.data_lineno), t))));
  /* TODO: uw_block_return could just throw this */
  if (target)
    sem_error(spec_linenum, lit("~a: no block named ~a in scope"),
              sym, target, nao);
  else
    sem_error(spec_linenum, lit("%~a: no anonymous block in scope"),
              sym, nao);
  return nil;
}

static val v_next(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  
  if (rest(first_spec) && rest(specline))
    sem_error(spec_linenum, lit("invalid combination of old "
                                "and new next syntax"), nao);
  if (rest(specline)) {
    sem_error(spec_linenum, lit("obsolete next syntax: trailing material"), nao);
  }

  if ((c.spec = rest(c.spec)) == nil)
    return cons(c.bindings, cons(c.data, c.data_lineno));

  if (rest(first_spec)) {
    val source = rest(first_spec);
    val keyword = first(source);
    val arg = keyword;

    if (keywordp(keyword)) {
      if (eq(keyword, nothrow_k)) {
        sem_error(spec_linenum, lit("misplaced :nothrow"), nao);
      } else if (eq(keyword, args_k)) {
        cons_bind (new_bindings, success,
                   match_files(mf_args(c)));

        if (success)
          return cons(new_bindings,
                      if3(c.data, cons(c.data, c.data_lineno), t));
        return nil;
      }
      arg = second(source);
    }

    {
      val eval = eval_form(spec_linenum, arg, c.bindings);
      val str = cdr(eval);

      if (eq(second(source), nothrow_k)) {
        if (str) {
          c.files = cons(cons(nothrow_k, str), c.files);
        } else {
          c.files = rest(c.files);
          if (!c.files) {
            debuglf(spec_linenum, lit("next: out of arguments"), nao);
            return nil;
          }
          c.files = cons(cons(nothrow_k, first(c.files)), rest(c.files));
        }
      } else {
        if (str) {
          c.files = cons(str, c.files);
        } else {
          c.files = rest(c.files);
          if (!c.files)
            sem_error(spec_linenum, lit("next: out of arguments"), nao);
          c.files = cons(cons(nothrow_k, first(c.files)), rest(c.files));
        }
      }
    }
  } else {
    c.files = rest(c.files);
    if (!c.files)
      sem_error(spec_linenum, lit("next: out of arguments"), nao);
  }

  /* We recursively process the file list, but the new
     data position we return to the caller must be in the
     original file we we were called with. Hence, we can't
     make a straight tail call here. */
  {
    cons_bind (new_bindings, success, match_files(mf_data(c, t, nil)));

    if (success)
      return cons(new_bindings,
                  if3(c.data, cons(c.data, c.data_lineno), t));
    return nil;
  }
}

static val v_parallel(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);

  if (second(first_spec) == t) {
    return decline_k;
  } else {
    val sym = first(first_spec);
    val all_match = t;
    val some_match = nil;
    val max_line = zero;
    val max_data = nil;
    val specs = second(first_spec);
    val plist = third(first_spec);
    val choose_shortest = getplist(plist, shortest_k);
    val choose_longest = getplist(plist, longest_k);
    val choose_sym = or2(choose_longest, choose_shortest);
    val choose_bindings = c.bindings, choose_line = zero, choose_data = nil;
    val choose_minmax = choose_longest ? num(-1) : num(NUM_MAX);
    val iter;

    if (choose_longest && choose_shortest)
      sem_error(spec_linenum, lit("choose: both :shortest and :longest specified"), nao);

    if (sym == choose_s && !choose_sym)
      sem_error(spec_linenum, lit("choose: criterion not specified"), nao);

    for (iter = specs; iter != nil; iter = rest(iter))
    {
      val nested_spec = first(iter);
      cons_bind (new_bindings, success, 
                 match_files(mf_spec(c, nested_spec)));

      if (success) {
        some_match = t;

        if (sym == choose_s) {
          val binding = choose_sym ? assoc(new_bindings, choose_sym) : nil;
          val value = cdr(binding);

          if (value) {
            val len = length_str(value);

            if ((choose_longest && gt(len, choose_minmax)) ||
                (choose_shortest && lt(len, choose_minmax)))
            {
              choose_minmax = len;
              choose_bindings = new_bindings;

              if (success == t) {
                choose_data = t;
              } else {
                cons_bind (new_data, new_line, success);
                choose_data = new_data;
                choose_line = new_line;
              }
            }
          }
        } else {
          /* choose does not propagate bindings between clauses! */
          c.bindings = new_bindings;
        }


        if (success == t) {
          max_data = t;
        } else if (consp(success) && max_data != t) {
          cons_bind (new_data, new_line, success);
          if (gt(new_line, max_line)) {
            max_line = new_line;
            max_data = new_data;
          }
        }
        if (sym == cases_s || sym == none_s)
          break;
      } else {
        all_match = nil;
        if (sym == all_s)
          break;
      }
    }

    if (sym == all_s && !all_match) {
      debuglf(spec_linenum, lit("all: some clauses didn't match"), nao);
      return nil;
    }

    if ((sym == some_s || sym == cases_s) && !some_match) {
      debuglf(spec_linenum, lit("some/cases: no clauses matched"), nao);
      return nil;
    }

    if (sym == none_s && some_match) {
      debuglf(spec_linenum, lit("none: some clauses matched"), nao);
      return nil;
    }

    /* No check for maybe, since it always succeeds. */

    if (choose_sym) {
      if (consp(choose_data)) {
        c.data_lineno = choose_line;
        c.data = choose_data;
      } else if (choose_data == t) {
        c.data = nil;
      }
      c.bindings = choose_bindings;
    } else if (consp(max_data)) {
      c.data_lineno = max_line;
      c.data = max_data;
    } else if (max_data == t) {
      c.data = nil;
    }

    *cout = c;
    return next_spec_k;
  }
}

static val v_collect(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val coll_spec = second(first_spec);
  val until_last_spec = third(first_spec);
  val args = fourth(first_spec);
  val bindings_coll = nil;
  val last_bindings = nil;
  val max = getplist(args, maxgap_k);
  val min = getplist(args, mingap_k);
  val gap = getplist(args, gap_k);
  val times = getplist(args, times_k);
  val mintimes = getplist(args, mintimes_k);
  val maxtimes = getplist(args, maxtimes_k);
  val lines = getplist(args, lines_k);
  val vars = getplist(args, vars_k);
  cnum cmax = nump(gap) ? c_num(gap) : (nump(max) ? c_num(max) : 0);
  cnum cmin = nump(gap) ? c_num(gap) : (nump(min) ? c_num(min) : 0);
  cnum mincounter = cmin, maxcounter = 0;
  cnum ctimax = nump(times) ? c_num(times) 
                            : (nump(maxtimes) ? c_num(maxtimes) : 0);
  cnum ctimin = nump(times) ? c_num(times) 
                            : (nump(mintimes) ? c_num(mintimes) : 0);
  cnum timescounter = 0, linescounter = 0;
  cnum ctimes = nump(times) ? c_num(times) : 0;
  cnum clines = nump(lines) ? c_num(lines) : 0;
  val iter;

  if (gap && (max || min))
    sem_error(spec_linenum, lit("collect: cannot mix :gap with :mingap or :maxgap"), nao);

  vars = vars_to_bindings(spec_linenum, vars, c.bindings);

  if ((times && ctimes == 0) || (lines && clines == 0)) {
    *cout = c;
    return next_spec_k;
  }

  uw_block_begin(nil, result);

  result = t;

  while (c.data) {
    val new_bindings = nil, success = nil;

    if ((gap || min) && mincounter < cmin)
      goto next_collect;

    if (lines && linescounter++ >= clines)
      break;

    {
      cons_set (new_bindings, success,
                match_files(mf_spec(c, coll_spec)));

      /* Until/last clause sees un-collated bindings from collect. */
      if (until_last_spec)
      {
        cons_bind (sym, ul_spec, until_last_spec);
        cons_bind (until_last_bindings, success,
                   match_files(mf_spec(c, ul_spec)));

        if (success) {
          debuglf(spec_linenum, lit("until/last matched ~a:~a"),
                  first(c.files), c.data_lineno, nao);
          /* Until discards bindings and position, last keeps them. */
          if (sym == last_s) {
            last_bindings = set_diff(until_last_bindings,
                                     new_bindings, eq_f, nil);
            if (success == t) {
              c.data = t;
            } else {
              cons_bind (new_data, new_line, success);
              c.data = new_data;
              c.data_lineno = new_line;
            }
          }
          break;
        }
      }

      if (success) {
        val strictly_new_bindings = set_diff(new_bindings,
                                             c.bindings, eq_f, nil);

        debuglf(spec_linenum, lit("collect matched ~a:~a"),
                first(c.files), c.data_lineno, nao);

        for (iter = vars; iter; iter = cdr(iter)) {
          cons_bind (var, dfl, car(iter));
          val exists = assoc(new_bindings, var);

          if (!exists) {
            if (!dfl) 
              sem_error(spec_linenum, lit("collect failed to bind ~a"),
                        var, nao);
            else
              strictly_new_bindings = acons(strictly_new_bindings, 
                                            var, dfl);
          }
        }

        for (iter = strictly_new_bindings; iter; iter = cdr(iter))
        {
          val binding = car(iter);
          val vars_binding = assoc(vars, car(binding));

          if (!vars || vars_binding) {
            val existing = assoc(bindings_coll, car(binding));

            bindings_coll = acons_new(bindings_coll, car(binding),
                                      cons(cdr(binding), cdr(existing)));
          }
        }
      }

      if (success) {
        if (consp(success)) {
          cons_bind (new_data, new_line, success);

          bug_unless (ge(new_line, c.data_lineno));

          if (new_line == c.data_lineno) {
            new_data = cdr(new_data);
            new_line = plus(new_line, num(1));
          }

          debuglf(spec_linenum, lit("collect advancing from line ~a to ~a"),
                  c.data_lineno, new_line, nao);

          c.data = new_data;
          c.data_lineno = new_line;
          *car_l(success) = nil;
        } else {
          debuglf(spec_linenum, lit("collect consumed entire file"), nao);
          c.data = nil;
        }
        mincounter = 0;
        maxcounter = 0;

        timescounter++;

        if ((times || maxtimes) && timescounter >= ctimax)
          break;
      } else {
next_collect:
        mincounter++;
        if ((gap || max) && ++maxcounter > cmax)
          break;
        c.data_lineno = plus(c.data_lineno, num(1));
        c.data = rest(c.data);
      }
    }
  }

  uw_block_end;

  if (!result) {
    debuglf(spec_linenum, lit("collect explicitly failed"), nao);
    return nil;
  }

  if ((times || mintimes) && timescounter < ctimin) {
    debuglf(spec_linenum, lit("fewer than ~a iterations collected"),
            num(ctimin), nao);
    return nil;
  }

  if (!bindings_coll)
    debuglf(spec_linenum, lit("nothing was collected"), nao);

  c.bindings = set_diff(c.bindings, bindings_coll, eq_f, car_f);

  for (iter = bindings_coll; iter; iter = cdr(iter)) {
    val pair = car(iter);
    val rev = cons(car(pair), nreverse(cdr(pair)));
    c.bindings = cons(rev, c.bindings);
  }

  if (last_bindings) {
    c.bindings = set_diff(c.bindings, last_bindings, eq_f, car_f);
    c.bindings = nappend2(last_bindings, c.bindings);
  }

  *cout = c;
  return next_spec_k;
}

static val v_flatten(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val iter;

  for (iter = rest(first_spec); iter; iter = rest(iter)) {
    val sym = first(iter);

    if (!bindable(sym)) {
      sem_error(spec_linenum,
                lit("flatten: ~s is not a bindable symbol"), sym, nao);
    } else {
      val existing = assoc(c.bindings, sym);

      if (existing)
        *cdr_l(existing) = flatten(cdr(existing));
    }
  }

  *cout = c;
  return next_spec_k;
}

static val v_forget_local(match_files_ctx c, match_files_ctx *cout)
{
  val specline = rest(first(c.spec));
  val first_spec = first(specline);
  c.bindings = alist_remove(c.bindings, rest(first_spec));
  *cout = c;
  return next_spec_k;
}

static val v_merge(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val target = first(rest(first_spec));
  val args = rest(rest(first_spec));
  val merged = nil;

  if (!bindable(target))
    sem_error(spec_linenum, lit("merge: ~s is not a bindable symbol"),
              target, nao);

  for (; args; args = rest(args)) {
    val arg = first(args);

    if (arg) {
      val arg_eval = eval_form(spec_linenum, arg, c.bindings);

      if (merged)
        merged = weird_merge(merged, cdr(arg_eval));
      else
        merged = cdr(arg_eval);
    }
  }

  c.bindings = acons_new(c.bindings, target, merged);

  *cout = c;
  return next_spec_k;
}

static val v_bind(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val args = rest(first_spec);
  val pattern = first(args);
  val form = second(args);
  val val = eval_form(spec_linenum, form, c.bindings);

  c.bindings = dest_bind(spec_linenum, c.bindings, pattern, cdr(val));

  if (c.bindings == t)
    return nil;

  *cout = c;
  return next_spec_k;
}

static val v_cat(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val iter;

  for (iter = rest(first_spec); iter; iter = rest(iter)) {
    val sym = first(iter);

    if (!bindable(sym)) {
      sem_error(spec_linenum,
                lit("cat: ~s is not a bindable symbol"), sym, nao);
    } else {
      val existing = assoc(c.bindings, sym);
      val sep = nil;

      if (rest(specline)) {
        val sub = subst_vars(rest(specline), c.bindings, nil);
        sep = cat_str(sub, nil);
      }

      if (existing)
        *cdr_l(existing) = cat_str(flatten(cdr(existing)), sep);
    }
  }

  *cout = c;
  return next_spec_k;
}

static val v_output(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val specs = second(first_spec);
  val dest_spec = third(first_spec);
  val nothrow = nil;
  val dest = lit("-");
  val filter = nil;
  fpip_t fp;

  if (eq(first(dest_spec), nothrow_k)) {
    if (rest(dest_spec))
      sem_error(spec_linenum, lit("material after :nothrow in output"), nao);
  } else if (!keywordp(first(dest_spec))) {
    val form = first(dest_spec);
    val val = eval_form(spec_linenum, form, c.bindings);
    dest = or2(cdr(val), dest);
    pop(&dest_spec);
  }

  if (eq(first(dest_spec), nothrow_k)) {
    nothrow = t;
    pop(&dest_spec);
  }

  if (keywordp(first(dest_spec))) {
    val filter_sym = getplist(dest_spec, filter_k);

    if (filter_sym) {
      filter = get_filter_trie(filter_sym);

      if (!filter)
        sem_error(spec_linenum, lit("unknown filter ~s"), filter_sym, nao);
    }
  }

  fp = (errno = 0, complex_open(dest, t));

  debugf(lit("opening data sink ~a"), dest, nao);

  if (complex_open_failed(fp)) {
    if (nothrow) {
      debugf(lit("could not open ~a: "
                 "treating as failed match due to nothrow"), dest, nao);
      return nil;
    } else if (errno != 0) {
      file_err(nil, lit("could not open ~a (error ~a/~a)"), dest,
               num(errno), string_utf8(strerror(errno)), nao);
    } else {
      file_err(nil, lit("could not open ~a"), dest, nao);
    }
  } else {
    val stream = complex_stream(fp, dest);
    do_output(c.bindings, specs, filter, stream);
    close_stream(stream, t);
  }

  *cout = c;
  return next_spec_k;
}

static val v_try(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val catch_syms = second(first_spec);
  val try_clause = third(first_spec);
  val catch_fin = fourth(first_spec);
  val finally_clause = nil;

  {
    uw_block_begin(nil, result);
    uw_catch_begin(catch_syms, exsym, exvals);

    result = match_files(mf_spec(c, try_clause));
    uw_do_unwind;

    uw_catch(exsym, exvals) {
      {
        val iter;

        for (iter = catch_fin; iter; iter = cdr(iter)) {
          val clause = car(iter);
          val type = first(second(clause));
          val params = second(second(clause));
          val body = third(clause);
          val vals = if3(listp(exvals),
                         exvals,
                         cons(cons(t, exvals), nil));

          if (first(clause) == catch_s) {
            if (uw_exception_subtype_p(exsym, type)) {
              val all_bind = t;
              val piter, viter;

              for (piter = params, viter = vals;
                   piter && viter;
                   piter = cdr(piter), viter = cdr(viter))
              {
                val param = car(piter);
                val value = car(viter);

                if (value) {
                  c.bindings = dest_bind(spec_linenum, c.bindings, 
                                       param, cdr(value));

                  if (c.bindings == t) {
                    all_bind = nil;
                    break;
                  }
                }
              }

              if (all_bind) {
                cons_bind (new_bindings, success,
                           match_files(mf_spec(c, body)));
                if (success) {
                  c.bindings = new_bindings;
                  result = t; /* catch succeeded, so try succeeds */
                  if (consp(success)) {
                    c.data = car(success);
                    c.data_lineno = cdr(success);
                  } else {
                    c.data = nil;
                  }
                }
              }
              break;
            }
          } else if (car(clause) == finally_s) {
            finally_clause = body;
          }
        }
      }
      uw_do_unwind;
    }

    uw_unwind {
      val iter;

      /* result may be t, from catch above. */
      if (consp(result)) {
        /* We process it before finally, as part of the unwinding, so
           finally can accumulate more bindings over top of any bindings
           produced by the main clause. */
        cons_bind (new_bindings, success, result);
        if (consp(success)) {
          c.data = car(success);
          c.data_lineno = cdr(success);
        } else {
          c.data = nil;
        }
        c.bindings = new_bindings;
      }

      if (!finally_clause) {
        for (iter = catch_fin; iter; iter = cdr(iter)) {
          val clause = car(iter);
          if (first(clause) == finally_s) {
            finally_clause = third(clause);
            break;
          }
        }
      }

      if (finally_clause) {
        cons_bind (new_bindings, success,
                   match_files(mf_spec(c, finally_clause)));

        if (success) {
          c.bindings = new_bindings;
          result = t; /* finally succeeds, so try block succeeds */
          if (consp(success)) {
            c.data = car(success);
            c.data_lineno = cdr(success);
          } else {
            c.data = nil;
          }
        }
      }
    }

    uw_catch_end;
    uw_block_end;

    if (!result)
      return nil;

    *cout = c;
    return next_spec_k;
  }
}

static val v_define(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val args = second(first_spec);
  val body = third(first_spec);
  val name = first(args);
  val params = second(args);

  if (rest(specline))
    sem_error(spec_linenum,
              lit("unexpected material after define"), nao);

  uw_set_func(name, cons(params, body));

  *cout = c;
  return next_spec_k;
}

static val v_defex(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val types = rest(first_spec);
  if (!all_satisfy(types, func_n1(symbolp), nil))
    sem_error(spec_linenum, lit("defex arguments must all be symbols"),
              nao);
  (void) reduce_left(func_n2(uw_register_subtype), types, nil, nil);

  *cout = c;
  return next_spec_k;
}

static val v_throw(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val type = second(first_spec);
  val args = rest(rest(first_spec));
  if (!symbolp(type))
    sem_error(spec_linenum, lit("throw: ~a is not a type symbol"),
              type, nao);
  {
    val values = mapcar(curry_123_2(func_n3(eval_form), 
                                    spec_linenum, c.bindings), args);
    uw_throw(type, values);
  }
}

static val v_deffilter(match_files_ctx c, match_files_ctx *cout)
{
  spec_bind (specline, spec_linenum, first_spec, c.spec);
  val sym = second(first_spec);
  val table = rest(rest(first_spec));

  if (!symbolp(sym))
    sem_error(spec_linenum, lit("deffilter: ~a is not a symbol"),
              first(first_spec), nao);

  if (!all_satisfy(table, andf(func_n1(listp), 
                               chain(func_n1(length),
                                     curry_12_2(func_n2(eq), two),
                                     nao),
                               chain(func_n1(first),
                                     func_n1(stringp), 
                                     nao),
                               chain(func_n1(second),
                                     func_n1(stringp), 
                                     nao), 
                               nao),
                            nil))
    sem_error(spec_linenum, 
              lit("deffilter arguments must be string pairs"),
              nao);
  register_filter(sym, table);
  /* TODO: warn about replaced filter. */
  *cout = c;
  return next_spec_k;
}

static val v_eof(match_files_ctx c, match_files_ctx *cout)
{
  if (c.data) {
    debugf(lit("eof failed to match at ~a"), c.data_lineno, nao);
    return nil;
  }
  *cout = c;
  return next_spec_k;
}

static val match_files(match_files_ctx c)
{
  gc_hint(c.data);

  if (listp(c.data)) { /* recursive call with lazy list */
    ; /* no specia initialization */
  } else if (c.files) { /* c.data == t: toplevel call with file list */
    val source_spec = first(c.files);
    val name = consp(source_spec) ? cdr(source_spec) : source_spec;
    fpip_t fp = (errno = 0, complex_open(name, nil));
    val first_spec_item = second(first(c.spec));

    if (consp(first_spec_item) && eq(first(first_spec_item), next_s)) {
      debugf(lit("not opening source ~a "
                 "since query starts with next directive"), name, nao);
    } else {
      debugf(lit("opening data source ~a"), name, nao);

      if (complex_open_failed(fp)) {
        if (consp(source_spec) && car(source_spec) == nothrow_k) {
          debugf(lit("could not open ~a: "
                     "treating as failed match due to nothrow"), name, nao);
          return nil;
        } else if (errno != 0)
          file_err(nil, lit("could not open ~a (error ~a/~a)"), name,
                   num(errno), string_utf8(strerror(errno)), nao);
        else
          file_err(nil, lit("could not open ~a"), name, nao);
        return nil;
      }

      c.files = cons(name, cdr(c.files)); /* Get rid of cons and nothrow */

      if ((c.data = complex_snarf(fp, name)) != nil)
        c.data_lineno = num(1);
    }
  } else { /* toplevel call with no data or file list */
    c.data = nil;
  }

  for (; c.spec; c.spec = rest(c.spec), 
                 c.data = rest(c.data),
                 c.data_lineno = plus(c.data_lineno, num(1)))
repeat_spec_same_data:
  {
    spec_bind (specline, spec_linenum, first_spec, c.spec);

    if (consp(first_spec)) {
      val sym = first(first_spec);
      val entry = gethash(v_directive_table, sym);

      if (entry) {
        v_match_func vmf = (v_match_func) cptr_get(entry);
        match_files_ctx nc;
        val result = vmf(c, &nc);

        if (result == next_spec_k) {
          c = nc;  
          if ((c.spec = rest(c.spec)) == nil)
            break;
          goto repeat_spec_same_data;
        } else if (result == decline_k) {
          /* go on to other processing below */
        } else {
          return result;
        }
      } else {
        val func = uw_get_func(sym);

        if (func) {
          val args = rest(first_spec);
          val params = car(func);
          val ub_p_a_pairs = nil;
          val body = cdr(func);
          val piter, aiter;
          val bindings_cp = copy_alist(c.bindings);

          if (!equal(length(args), length(params)))
            sem_error(spec_linenum, lit("function ~a takes ~a argument(s)"),
                      sym, length(params), nao);

          for (piter = params, aiter = args; piter;
               piter = cdr(piter), aiter = cdr(aiter))
          {
            val param = car(piter);
            val arg = car(aiter);

            if (arg && bindable(arg)) {
              val val = assoc(c.bindings, arg);
              if (val) {
                bindings_cp = acons_new(bindings_cp,
                                        param,
                                        cdr(val));
              } else {
                bindings_cp = alist_nremove1(bindings_cp, param);
                ub_p_a_pairs = cons(cons(param, arg), ub_p_a_pairs);
              }
            } else {
              val val = eval_form(spec_linenum, arg, c.bindings);
              bindings_cp = acons_new(bindings_cp, param, cdr(val));
            }
          }

          {
            uw_block_begin(nil, result);
            uw_env_begin;
            result = match_files(mf_spec_bindings(c, body, bindings_cp));
            uw_env_end;
            uw_block_end;

            if (!result) {
              debuglf(spec_linenum, lit("function failed"), nao);
              return nil;
            }

            {
              cons_bind (new_bindings, success, result);

              for (piter = ub_p_a_pairs; piter; piter = cdr(piter))
              {
                cons_bind (param, arg, car(piter));

                if (symbolp(arg)) {
                  val newbind = assoc(new_bindings, param);
                  if (newbind) {
                    c.bindings = dest_bind(spec_linenum, c.bindings, 
                                         arg, cdr(newbind));
                    if (c.bindings == t) {
                      debuglf(spec_linenum,
                              lit("binding mismatch on ~a "
                                  "when returning from ~a"), arg, sym, nao);
                      return nil;
                    }
                  }
                }
              }

              if (consp(success)) {
                debuglf(spec_linenum,
                        lit("function matched; "
                            "advancing from line ~a to ~a"),
                        c.data_lineno, cdr(success), nao);
                c.data = car(success);
                c.data_lineno = cdr(success);
              } else {
                debuglf(spec_linenum, lit("function consumed entire file"),
                        nao);
                c.data = nil;
              }
            }
          }

          if ((c.spec = rest(c.spec)) == nil)
            break;

          goto repeat_spec_same_data;
        }
      }
    }

    if (c.data)
    {
      val dataline = first(c.data);
      cons_bind (new_bindings, success,
                 match_line(ml_all(c.bindings, specline, dataline, zero,
                                   spec_linenum, c.data_lineno, first(c.files))));

      if (nump(success) && c_num(success) < c_num(length_str(dataline))) {
        debuglf(spec_linenum, lit("spec only matches line to position ~a: ~a"),
                success, dataline, nao);
        return nil;
      }

      if (!success)
        return nil;

      c.bindings = new_bindings;
    } else {
      debuglf(spec_linenum, lit("spec ran out of data"), nao);
      return nil;
    }
  }

  return cons(c.bindings, if3(c.data, cons(c.data, c.data_lineno), t));
}

int extract(val spec, val files, val predefined_bindings)
{
  cons_bind (bindings, success, match_files(mf_all(spec, files, 
                                                   predefined_bindings,
                                                   t, nil)));

  if (!output_produced) {
    if (!opt_nobindings) {
      if (bindings) {
        bindings = nreverse(bindings);
        dump_bindings(bindings);
      }
    }

    if (!success)
      put_line(std_output, lit("false"));
  }

  return success ? 0 : EXIT_FAILURE;
}

static void syms_init(void)
{
  decline_k = intern(lit("decline"), keyword_package);
  next_spec_k = intern(lit("next-spec"), keyword_package);
  repeat_spec_k = intern(lit("repeat-spec"), keyword_package);
  mingap_k = intern(lit("mingap"), keyword_package);
  maxgap_k = intern(lit("maxgap"), keyword_package);
  gap_k = intern(lit("gap"), keyword_package);
  mintimes_k = intern(lit("mintimes"), keyword_package);
  maxtimes_k = intern(lit("maxtimes"), keyword_package);
  times_k = intern(lit("times"), keyword_package);
  lines_k = intern(lit("lines"), keyword_package);
  chars_k = intern(lit("chars"), keyword_package);
  choose_s = intern(lit("choose"), user_package);
  longest_k = intern(lit("longest"), keyword_package);
  shortest_k = intern(lit("shortest"), keyword_package);
  greedy_k = intern(lit("greedy"), keyword_package);
  vars_k = intern(lit("vars"), keyword_package);
}

static void dir_tables_init(void)
{
  h_directive_table = make_hash(nil, nil);
  v_directive_table = make_hash(nil, nil);

  protect(&h_directive_table, &v_directive_table, (val *) 0);

  sethash(v_directive_table, skip_s, cptr((mem_t *) v_skip));
  sethash(v_directive_table, trailer_s, cptr((mem_t *) v_trailer));
  sethash(v_directive_table, freeform_s, cptr((mem_t *) v_freeform));
  sethash(v_directive_table, block_s, cptr((mem_t *) v_block));
  sethash(v_directive_table, accept_s, cptr((mem_t *) v_accept_fail));
  sethash(v_directive_table, fail_s, cptr((mem_t *) v_accept_fail));
  sethash(v_directive_table, next_s, cptr((mem_t *) v_next));
  sethash(v_directive_table, some_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, all_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, none_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, maybe_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, cases_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, choose_s, cptr((mem_t *) v_parallel));
  sethash(v_directive_table, collect_s, cptr((mem_t *) v_collect));
  sethash(v_directive_table, flatten_s, cptr((mem_t *) v_flatten));
  sethash(v_directive_table, forget_s, cptr((mem_t *) v_forget_local));
  sethash(v_directive_table, local_s, cptr((mem_t *) v_forget_local));
  sethash(v_directive_table, merge_s, cptr((mem_t *) v_merge));
  sethash(v_directive_table, bind_s, cptr((mem_t *) v_bind));
  sethash(v_directive_table, cat_s, cptr((mem_t *) v_cat));
  sethash(v_directive_table, output_s, cptr((mem_t *) v_output));
  sethash(v_directive_table, define_s, cptr((mem_t *) v_define));
  sethash(v_directive_table, try_s, cptr((mem_t *) v_try));
  sethash(v_directive_table, defex_s, cptr((mem_t *) v_defex));
  sethash(v_directive_table, throw_s, cptr((mem_t *) v_throw));
  sethash(v_directive_table, deffilter_s, cptr((mem_t *) v_deffilter));
  sethash(v_directive_table, eof_s, cptr((mem_t *) v_eof));

  sethash(h_directive_table, var_s, cptr((mem_t *) h_var));
  sethash(h_directive_table, skip_s, cptr((mem_t *) h_skip));
  sethash(h_directive_table, coll_s, cptr((mem_t *) h_coll));
  sethash(h_directive_table, all_s, cptr((mem_t *) h_parallel));
  sethash(h_directive_table, none_s, cptr((mem_t *) h_parallel));
  sethash(h_directive_table, maybe_s, cptr((mem_t *) h_parallel));
  sethash(h_directive_table, cases_s, cptr((mem_t *) h_parallel));
  sethash(h_directive_table, choose_s, cptr((mem_t *) h_parallel));
  sethash(h_directive_table, trailer_s, cptr((mem_t *) h_trailer));
  sethash(h_directive_table, eol_s, cptr((mem_t *) h_eol));
}

void match_init(void)
{
  syms_init();
  dir_tables_init();
}